@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5c342570d7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

Add marlin unit tests and marlin benchmark script (#4815)

alexm-nm committed May 16, 2024 at 09:36 UTC 5c342570d7e4c73bfaa4c4057174b92117d322bb
6 files changed +736
benchmarks/kernels/benchmark_marlin.py new
+183
@@ -0,0 +1,183 @@
1 +import argparse
2 +
3 +import torch
4 +import torch.utils.benchmark as benchmark
5 +from benchmark_shapes import WEIGHT_SHAPES
6 +
7 +from vllm import _custom_ops as ops
8 +from vllm.model_executor.layers.quantization.gptq_marlin import (
9 + GPTQ_MARLIN_SUPPORTED_GROUP_SIZES, GPTQ_MARLIN_SUPPORTED_NUM_BITS)
10 +from vllm.model_executor.layers.quantization.utils.marlin_utils import (
11 + MarlinWorkspace, marlin_quantize)
12 +from vllm.model_executor.layers.quantization.utils.quant_utils import (
13 + gptq_pack, quantize_weights, sort_weights)
14 +
15 +DEFAULT_MODELS = ["meta-llama/Llama-2-7b-hf/TP1"]
16 +DEFAULT_BATCH_SIZES = [1, 16, 32, 64, 128, 256, 512]
17 +
18 +ACT_ORDER_OPTS = [False, True]
19 +K_FULL_OPTS = [False, True]
20 +
21 +
22 +def bench_run(results, model, act_order, is_k_full, num_bits, group_size,
23 + size_m, size_k, size_n):
24 + label = "Quant Matmul"
25 +
26 + sub_label = ("{}, act={} k_full={}, b={}, g={}, "
27 + "MKN=({}x{}x{})".format(model, act_order, is_k_full, num_bits,
28 + group_size, size_m, size_k, size_n))
29 +
30 + print(f"Testing: {sub_label}")
31 +
32 + a = torch.randn(size_m, size_k).to(torch.half).cuda()
33 + b = torch.rand(size_k, size_n).to(torch.half).cuda()
34 +
35 + a_tmp = (torch.zeros(size_m, size_k).to(torch.half).cuda())
36 +
37 + # Marlin quant
38 + (
39 + marlin_w_ref,
40 + marlin_q_w,
41 + marlin_s,
42 + marlin_g_idx,
43 + marlin_sort_indices,
44 + marlin_rand_perm,
45 + ) = marlin_quantize(b, num_bits, group_size, act_order)
46 +
47 + # GPTQ quant
48 + (w_ref, q_w, s, g_idx,
49 + rand_perm) = quantize_weights(b, num_bits, group_size, act_order)
50 + q_w_gptq = gptq_pack(q_w, num_bits, size_k, size_n)
51 +
52 + # For act_order, sort the "weights" and "g_idx"
53 + # so that group ids are increasing
54 + repack_sort_indices = torch.empty(0, dtype=torch.int, device=b.device)
55 + if act_order:
56 + (q_w, g_idx, repack_sort_indices) = sort_weights(q_w, g_idx)
57 +
58 + # Prepare
59 + marlin_workspace = MarlinWorkspace(size_n)
60 +
61 + globals = {
62 + "marlin_w_ref": marlin_w_ref,
63 + "marlin_q_w": marlin_q_w,
64 + "marlin_s": marlin_s,
65 + "marlin_g_idx": marlin_g_idx,
66 + "marlin_sort_indices": marlin_sort_indices,
67 + "marlin_rand_perm": marlin_rand_perm,
68 + "q_w_gptq": q_w_gptq,
69 + "repack_sort_indices": repack_sort_indices,
70 + "num_bits": num_bits,
71 + "group_size": group_size,
72 + "size_m": size_m,
73 + "size_n": size_n,
74 + "size_k": size_k,
75 + "is_k_full": is_k_full,
76 + "a": a,
77 + "a_tmp": a_tmp,
78 + "gptq_marlin_gemm": ops.gptq_marlin_gemm,
79 + "gptq_marlin_repack": ops.gptq_marlin_repack,
80 + "marlin_workspace": marlin_workspace,
81 + }
82 +
83 + min_run_time = 1
84 +
85 + # Warmup pytorch
86 + for i in range(5):
87 + torch.matmul(a, marlin_w_ref)
88 +
89 + results.append(
90 + benchmark.Timer(
91 + stmt="torch.matmul(a, marlin_w_ref)",
92 + globals=globals,
93 + label=label,
94 + sub_label=sub_label,
95 + description="pytorch_gemm",
96 + ).blocked_autorange(min_run_time=min_run_time))
97 +
98 + results.append(
99 + benchmark.Timer(
100 + stmt=
101 + "output = gptq_marlin_gemm(a, marlin_q_w, marlin_s, marlin_g_idx, marlin_sort_indices, marlin_workspace.scratch, num_bits, size_m, size_n, size_k, is_k_full)", # noqa: E501
102 + globals=globals,
103 + label=label,
104 + sub_label=sub_label,
105 + description="gptq_marlin_gemm",
106 + ).blocked_autorange(min_run_time=min_run_time))
107 +
108 + results.append(
109 + benchmark.Timer(
110 + stmt=
111 + "q_res = gptq_marlin_repack(q_w_gptq, repack_sort_indices, size_k, size_n, num_bits)", # noqa: E501
112 + globals=globals,
113 + label=label,
114 + sub_label=sub_label,
115 + description="gptq_marlin_repack",
116 + ).blocked_autorange(min_run_time=min_run_time))
117 +
118 +
119 +def main(args):
120 + print("Benchmarking models:")
121 + for i, model in enumerate(args.models):
122 + print(f"[{i}] {model}")
123 +
124 + results = []
125 +
126 + for model in args.models:
127 + for layer in WEIGHT_SHAPES[model]:
128 + size_k = layer[0]
129 + size_n = layer[1]
130 +
131 + if len(args.limit_k) > 0 and size_k not in args.limit_k:
132 + continue
133 +
134 + if len(args.limit_n) > 0 and size_n not in args.limit_n:
135 + continue
136 +
137 + for act_order in ACT_ORDER_OPTS:
138 + for is_k_full in K_FULL_OPTS:
139 + for num_bits in GPTQ_MARLIN_SUPPORTED_NUM_BITS:
140 + for group_size in GPTQ_MARLIN_SUPPORTED_GROUP_SIZES:
141 + if len(
142 + args.limit_group_size
143 + ) > 0 and group_size not in args.limit_group_size:
144 + continue
145 +
146 + # For act_order, the group_size must be less than
147 + # size_k
148 + if act_order and (group_size == size_k
149 + or group_size == -1):
150 + continue
151 +
152 + for size_m in args.batch_sizes:
153 + bench_run(results, model, act_order, is_k_full,
154 + num_bits, group_size, size_m, size_k,
155 + size_n)
156 +
157 + compare = benchmark.Compare(results)
158 + compare.print()
159 +
160 +
161 +# For quick benchmarking use:
162 +# python benchmark_marlin.py --batch-sizes 1 16 32 --limit-k 4096 --limit-n 4096 --limit-group-size 128 # noqa E501
163 +#
164 +if __name__ == "__main__":
165 + parser = argparse.ArgumentParser(
166 + description="Benchmark Marlin across specified models/shapes/batches")
167 + parser.add_argument(
168 + "--models",
169 + nargs="+",
170 + type=str,
171 + default=DEFAULT_MODELS,
172 + choices=WEIGHT_SHAPES.keys(),
173 + )
174 + parser.add_argument("--batch-sizes",
175 + nargs="+",
176 + type=int,
177 + default=DEFAULT_BATCH_SIZES)
178 + parser.add_argument("--limit-k", nargs="+", type=int, default=[])
179 + parser.add_argument("--limit-n", nargs="+", type=int, default=[])
180 + parser.add_argument("--limit-group-size", nargs="+", type=int, default=[])
181 +
182 + args = parser.parse_args()
183 + main(args)
benchmarks/kernels/benchmark_shapes.py new
+75
@@ -0,0 +1,75 @@
1 +WEIGHT_SHAPES = {
2 + "ideal": [[4 * 256 * 32, 256 * 32]],
3 + "mistralai/Mistral-7B-v0.1/TP1": [
4 + [4096, 6144],
5 + [4096, 4096],
6 + [4096, 28672],
7 + [14336, 4096],
8 + ],
9 + "mistralai/Mistral-7B-v0.1/TP2": [
10 + [4096, 3072],
11 + [2048, 4096],
12 + [4096, 14336],
13 + [7168, 4096],
14 + ],
15 + "mistralai/Mistral-7B-v0.1/TP4": [
16 + [4096, 1536],
17 + [1024, 4096],
18 + [4096, 7168],
19 + [3584, 4096],
20 + ],
21 + "meta-llama/Llama-2-7b-hf/TP1": [
22 + [4096, 12288],
23 + [4096, 4096],
24 + [4096, 22016],
25 + [11008, 4096],
26 + ],
27 + "meta-llama/Llama-2-7b-hf/TP2": [
28 + [4096, 6144],
29 + [2048, 4096],
30 + [4096, 11008],
31 + [5504, 4096],
32 + ],
33 + "meta-llama/Llama-2-7b-hf/TP4": [
34 + [4096, 3072],
35 + [1024, 4096],
36 + [4096, 5504],
37 + [2752, 4096],
38 + ],
39 + "meta-llama/Llama-2-13b-hf/TP1": [
40 + [5120, 15360],
41 + [5120, 5120],
42 + [5120, 27648],
43 + [13824, 5120],
44 + ],
45 + "meta-llama/Llama-2-13b-hf/TP2": [
46 + [5120, 7680],
47 + [2560, 5120],
48 + [5120, 13824],
49 + [6912, 5120],
50 + ],
51 + "meta-llama/Llama-2-13b-hf/TP4": [
52 + [5120, 3840],
53 + [1280, 5120],
54 + [5120, 6912],
55 + [3456, 5120],
56 + ],
57 + "meta-llama/Llama-2-70b-hf/TP1": [
58 + [8192, 10240],
59 + [8192, 8192],
60 + [8192, 57344],
61 + [28672, 8192],
62 + ],
63 + "meta-llama/Llama-2-70b-hf/TP2": [
64 + [8192, 5120],
65 + [4096, 8192],
66 + [8192, 28672],
67 + [14336, 8192],
68 + ],
69 + "meta-llama/Llama-2-70b-hf/TP4": [
70 + [8192, 2560],
71 + [2048, 8192],
72 + [8192, 14336],
73 + [7168, 8192],
74 + ],
75 +}
tests/kernels/test_marlin_gemm.py new
+158
@@ -0,0 +1,158 @@
1 +"""Tests for the marlin kernel.
2 +
3 +Run `pytest tests/kernels/marlin/test_marlin_gemm.py`.
4 +"""
5 +import pytest
6 +import torch
7 +
8 +from vllm import _custom_ops as ops
9 +from vllm.model_executor.layers.quantization.gptq_marlin import (
10 + GPTQ_MARLIN_SUPPORTED_GROUP_SIZES, GPTQ_MARLIN_SUPPORTED_NUM_BITS)
11 +from vllm.model_executor.layers.quantization.utils.marlin_utils import (
12 + MarlinWorkspace, is_marlin_supported, marlin_quantize, marlin_weights)
13 +from vllm.model_executor.layers.quantization.utils.quant_utils import (
14 + gptq_pack, quantize_weights, sort_weights)
15 +
16 +ACT_ORDER_OPTS = [False, True]
17 +K_FULL_OPTS = [False, True]
18 +
19 +K_CHUNKS = [128, 256]
20 +N_CHUNKS = [64, 128, 256]
21 +
22 +MNK_FACTORS = [
23 + (1, 1, 1),
24 + (1, 4, 8),
25 + (1, 7, 5),
26 + (1, 7 * 4, 5 * 1),
27 + (13, 17, 67),
28 + (26, 37, 13),
29 + (67, 13, 11),
30 +]
31 +
32 +
33 +def rand_data(shape):
34 + data = torch.rand(shape).to(torch.half).cuda()
35 + return data
36 +
37 +
38 +@pytest.mark.skipif(not is_marlin_supported(),
39 + reason="Marlin is not supported on this GPU type.")
40 +@pytest.mark.parametrize("k_chunk", K_CHUNKS)
41 +@pytest.mark.parametrize("n_chunk", N_CHUNKS)
42 +@pytest.mark.parametrize("num_bits", GPTQ_MARLIN_SUPPORTED_NUM_BITS)
43 +@pytest.mark.parametrize("group_size", GPTQ_MARLIN_SUPPORTED_GROUP_SIZES)
44 +@pytest.mark.parametrize("act_order", ACT_ORDER_OPTS)
45 +@pytest.mark.parametrize("mnk_factors", MNK_FACTORS)
46 +def test_marlin_repack(k_chunk, n_chunk, num_bits, group_size, act_order,
47 + mnk_factors):
48 + m_factor, n_factor, k_factor = mnk_factors
49 +
50 + size_m = m_factor
51 + size_k = k_chunk * k_factor
52 + size_n = n_chunk * n_factor
53 +
54 + print(f"MNK = {size_m} {size_n} {size_k}")
55 +
56 + # Filter act_order
57 + if act_order:
58 + if group_size == -1:
59 + return
60 + if group_size == size_k:
61 + return
62 +
63 + # Normalize group_size
64 + if group_size == -1:
65 + group_size = size_k
66 + assert group_size <= size_k
67 +
68 + # Create input
69 + b_weight = rand_data((size_k, size_n))
70 +
71 + # Quantize (and apply act_order if provided)
72 + w_ref, q_w, s, g_idx, rand_perm = quantize_weights(b_weight, num_bits,
73 + group_size, act_order)
74 +
75 + # Pack to GPTQ format
76 + q_w_gptq = gptq_pack(q_w, num_bits, size_k, size_n)
77 +
78 + # For act_order, sort the "weights" and "g_idx" so that group ids are
79 + # increasing
80 + sort_indices = torch.empty(0, dtype=torch.int, device=b_weight.device)
81 + if act_order:
82 + q_w, g_idx, sort_indices = sort_weights(q_w, g_idx)
83 +
84 + # Pack to Marlin format
85 + marlin_q_w_1 = marlin_weights(q_w, size_k, size_n, num_bits)
86 +
87 + # Run Marlin repack GPU kernel
88 + marlin_q_w_2 = ops.gptq_marlin_repack(
89 + q_w_gptq,
90 + sort_indices,
91 + size_k,
92 + size_n,
93 + num_bits,
94 + )
95 + torch.cuda.synchronize()
96 +
97 + assert torch.allclose(marlin_q_w_1, marlin_q_w_2)
98 +
99 +
100 +@pytest.mark.skipif(not is_marlin_supported(),
101 + reason="Marlin is not supported on this GPU type.")
102 +@pytest.mark.parametrize("k_chunk", K_CHUNKS)
103 +@pytest.mark.parametrize("n_chunk", N_CHUNKS)
104 +@pytest.mark.parametrize("num_bits", GPTQ_MARLIN_SUPPORTED_NUM_BITS)
105 +@pytest.mark.parametrize("group_size", GPTQ_MARLIN_SUPPORTED_GROUP_SIZES)
106 +@pytest.mark.parametrize("mnk_factors", MNK_FACTORS)
107 +@pytest.mark.parametrize("act_order", ACT_ORDER_OPTS)
108 +@pytest.mark.parametrize("is_k_full", K_FULL_OPTS)
109 +def test_marlin_gemm(
110 + k_chunk,
111 + n_chunk,
112 + num_bits,
113 + group_size,
114 + mnk_factors,
115 + act_order,
116 + is_k_full,
117 +):
118 + m_factor, n_factor, k_factor = mnk_factors
119 +
120 + size_m = m_factor
121 + size_k = k_chunk * k_factor
122 + size_n = n_chunk * n_factor
123 +
124 + print(f"MNK = {size_m} {size_n} {size_k}")
125 + print(f"groupsize = {group_size}")
126 +
127 + if act_order:
128 + if group_size == -1:
129 + return
130 + if group_size == size_k:
131 + return
132 +
133 + a_input = rand_data((size_m, size_k))
134 + b_weight = rand_data((size_k, size_n))
135 +
136 + w_ref, marlin_q_w, marlin_s, g_idx, sort_indices, _ = marlin_quantize(
137 + b_weight, num_bits, group_size, act_order)
138 +
139 + workspace = MarlinWorkspace(size_n)
140 +
141 + output = ops.gptq_marlin_gemm(
142 + a_input,
143 + marlin_q_w,
144 + marlin_s,
145 + g_idx,
146 + sort_indices,
147 + workspace.scratch,
148 + num_bits,
149 + a_input.shape[0],
150 + b_weight.shape[1],
151 + a_input.shape[1],
152 + is_k_full,
153 + )
154 + output_ref = torch.matmul(a_input, w_ref)
155 +
156 + torch.cuda.synchronize()
157 +
158 + assert torch.allclose(output, output_ref, rtol=1e-2)
vllm/model_executor/layers/quantization/utils/__init__.py
vllm/model_executor/layers/quantization/utils/marlin_utils.py new
+174
@@ -0,0 +1,174 @@
1 +"""This file is used for /tests and /benchmarks"""
2 +import numpy
3 +import torch
4 +
5 +from vllm.model_executor.layers.quantization.gptq_marlin import (
6 + GPTQ_MARLIN_MAX_PARALLEL, GPTQ_MARLIN_MIN_THREAD_N, GPTQ_MARLIN_TILE)
7 +from vllm.model_executor.layers.quantization.utils.quant_utils import (
8 + get_pack_factor, quantize_weights, sort_weights)
9 +
10 +__cuda_arch = torch.cuda.get_device_capability()
11 +
12 +
13 +def is_marlin_supported():
14 + return __cuda_arch[0] >= 8
15 +
16 +
17 +# Precompute permutations for Marlin weight and scale shuffling # noqa: E501
18 +#
19 +# Marlin works on [16,64] tiles. The goal of the permutations is to reorder the weight data so that it is compatible noqa: # noqa: E501
20 +# with the tensor-core format that is described here:
21 +# https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#matrix-fragments-for-mma-m16n8k16-with-floating-point-type # noqa: E501
22 +#
23 +# As a result of this reordering, the vector loads inside the kernel will get the data as it is needed for tensor-core # noqa: E501
24 +# (without the need to use ldmatrix instructions) # noqa: E501
25 +def _get_perms(num_bits):
26 + perm_list = []
27 + for i in range(32):
28 + perm1 = []
29 + col = i // 4
30 + for block in [0, 1]:
31 + for row in [
32 + 2 * (i % 4),
33 + 2 * (i % 4) + 1,
34 + 2 * (i % 4 + 4),
35 + 2 * (i % 4 + 4) + 1,
36 + ]:
37 + perm1.append(16 * row + col + 8 * block)
38 + for j in range(4):
39 + perm_list.extend([p + 256 * j for p in perm1])
40 +
41 + perm = numpy.array(perm_list)
42 +
43 + if num_bits == 4:
44 + interleave = numpy.array([0, 2, 4, 6, 1, 3, 5, 7])
45 + elif num_bits == 8:
46 + interleave = numpy.array([0, 2, 1, 3])
47 + else:
48 + raise Exception("num_bits must be 4 or 8, got {}".format(num_bits))
49 +
50 + perm = perm.reshape((-1, len(interleave)))[:, interleave].ravel()
51 + perm = torch.from_numpy(perm)
52 + scale_perm = []
53 + for i in range(8):
54 + scale_perm.extend([i + 8 * j for j in range(8)])
55 + scale_perm_single = []
56 + for i in range(4):
57 + scale_perm_single.extend(
58 + [2 * i + j for j in [0, 1, 8, 9, 16, 17, 24, 25]])
59 + return perm, scale_perm, scale_perm_single
60 +
61 +
62 +_perm = {}
63 +_scale_perm = {}
64 +_scale_perm_single = {}
65 +for num_bits in [4, 8]:
66 + perm, scale_perm, scale_perm_single = _get_perms(num_bits)
67 + _perm[num_bits] = perm
68 + _scale_perm[num_bits] = scale_perm
69 + _scale_perm_single[num_bits] = scale_perm_single
70 +
71 +
72 +def marlin_permute_weights(q_w,
73 + size_k,
74 + size_n,
75 + num_bits,
76 + tile=GPTQ_MARLIN_TILE):
77 + assert q_w.shape == (size_k, size_n)
78 + assert size_k % tile == 0, f"size_k = {size_k}, tile = {tile}"
79 + assert size_n % tile == 0, f"size_k = {size_n}, tile = {tile}"
80 +
81 + # Permute weights to 16x64 marlin tiles
82 + q_w = q_w.reshape((size_k // tile, tile, size_n // tile, tile))
83 + q_w = q_w.permute((0, 2, 1, 3))
84 + q_w = q_w.reshape((size_k // tile, size_n * tile))
85 +
86 + q_w = q_w.reshape(
87 + (-1, _perm[num_bits].numel()))[:, _perm[num_bits]].reshape(q_w.shape)
88 +
89 + return q_w
90 +
91 +
92 +def marlin_weights(q_w, size_k, size_n, num_bits):
93 + # Permute
94 + q_w = marlin_permute_weights(q_w, size_k, size_n, num_bits)
95 +
96 + # Pack
97 + pack_factor = get_pack_factor(num_bits)
98 + orig_device = q_w.device
99 +
100 + q_w = q_w.cpu().numpy().astype(numpy.uint32)
101 +
102 + q_packed = numpy.zeros((q_w.shape[0], q_w.shape[1] // pack_factor),
103 + dtype=numpy.uint32)
104 +
105 + for i in range(pack_factor):
106 + q_packed |= q_w[:, i::pack_factor] << num_bits * i
107 +
108 + q_packed = torch.from_numpy(q_packed.astype(numpy.int32)).to(orig_device)
109 +
110 + return q_packed
111 +
112 +
113 +def marlin_permute_scales(s, size_k, size_n, group_size, num_bits):
114 + if group_size < size_k and group_size != -1:
115 + s = s.reshape((-1, len(_scale_perm[num_bits])))[:,
116 + _scale_perm[num_bits]]
117 + else:
118 + s = s.reshape(
119 + (-1,
120 + len(_scale_perm_single[num_bits])))[:,
121 + _scale_perm_single[num_bits]]
122 + s = s.reshape((-1, size_n)).contiguous()
123 +
124 + return s
125 +
126 +
127 +def marlin_quantize(
128 + w: torch.Tensor,
129 + num_bits: int,
130 + group_size: int,
131 + act_order: bool,
132 +):
133 + size_k, size_n = w.shape
134 +
135 + # Normalize group_size
136 + if group_size == -1:
137 + group_size = size_k
138 + assert group_size <= size_k
139 +
140 + # Quantize (and apply act_order if provided)
141 + w_ref, q_w, s, g_idx, rand_perm = quantize_weights(w, num_bits, group_size,
142 + act_order)
143 +
144 + # For act_order, sort the "weights" and "g_idx" so that group ids are
145 + # increasing
146 + sort_indices = torch.empty(0, dtype=torch.int, device=w.device)
147 + if act_order:
148 + q_w, g_idx, sort_indices = sort_weights(q_w, g_idx)
149 +
150 + # Reformat to marlin
151 + marlin_q_w = marlin_weights(q_w, size_k, size_n, num_bits)
152 + marlin_s = marlin_permute_scales(s, size_k, size_n, group_size, num_bits)
153 +
154 + # Create result
155 + res_list = [w_ref, marlin_q_w, marlin_s, g_idx, sort_indices, rand_perm]
156 + for i in range(len(res_list)):
157 + res_list[i] = res_list[i].to(w.device)
158 +
159 + return res_list
160 +
161 +
162 +class MarlinWorkspace:
163 +
164 + def __init__(self, out_features):
165 + assert (out_features % GPTQ_MARLIN_MIN_THREAD_N == 0), (
166 + "out_features = {} is undivisible by GPTQ_MARLIN_MIN_THREAD_N = {}"
167 + .format(out_features, GPTQ_MARLIN_MIN_THREAD_N))
168 +
169 + max_workspace_size = ((out_features // GPTQ_MARLIN_MIN_THREAD_N) *
170 + GPTQ_MARLIN_MAX_PARALLEL)
171 +
172 + self.scratch = torch.zeros(max_workspace_size,
173 + dtype=torch.int,
174 + device="cuda")
vllm/model_executor/layers/quantization/utils/quant_utils.py new
+146
@@ -0,0 +1,146 @@
1 +"""This file is used for /tests and /benchmarks"""
2 +import numpy
3 +import torch
4 +
5 +SUPPORTED_NUM_BITS = [4, 8]
6 +SUPPORTED_GROUP_SIZES = [-1, 32, 64, 128]
7 +
8 +
9 +def get_pack_factor(num_bits):
10 + assert num_bits in SUPPORTED_NUM_BITS, f"Unsupported num_bits = {num_bits}"
11 + return 32 // num_bits
12 +
13 +
14 +def permute_rows(q_w: torch.Tensor, w_ref: torch.Tensor, group_size: int):
15 + assert q_w.shape == w_ref.shape
16 +
17 + orig_device = q_w.device
18 + k_size, _ = q_w.shape
19 +
20 + g_idx = torch.zeros((k_size, ), dtype=torch.int32)
21 + for i in range(k_size):
22 + g_idx[i] = i // group_size
23 +
24 + # Simulate act_order by doing a random permutation on K
25 + rand_perm = torch.randperm(k_size)
26 +
27 + g_idx = g_idx[rand_perm].contiguous()
28 + q_w = q_w[rand_perm, :].contiguous()
29 + w_ref = w_ref[rand_perm, :].contiguous()
30 +
31 + return (
32 + w_ref.to(device=orig_device),
33 + q_w.to(device=orig_device),
34 + g_idx.to(device=orig_device),
35 + rand_perm.to(device=orig_device),
36 + )
37 +
38 +
39 +def quantize_weights(w: torch.Tensor, num_bits: int, group_size: int,
40 + act_order: bool):
41 + orig_device = w.device
42 + size_k, size_n = w.shape
43 +
44 + assert w.is_floating_point(), "w must be float"
45 + assert num_bits in SUPPORTED_NUM_BITS, f"Unsupported num_bits = {num_bits}"
46 + assert group_size in SUPPORTED_GROUP_SIZES + [
47 + size_k
48 + ], f"Unsupported groupsize = {group_size}"
49 +
50 + if group_size == -1:
51 + group_size = size_k
52 + assert group_size <= size_k
53 +
54 + max_q_val = 2**num_bits - 1
55 + half_q_val = (max_q_val + 1) // 2
56 +
57 + # Reshape to [groupsize, -1]
58 + if group_size < size_k:
59 + w = w.reshape((-1, group_size, size_n))
60 + w = w.permute(1, 0, 2)
61 + w = w.reshape((group_size, -1))
62 +
63 + # Compute scale for each group
64 + s = torch.max(torch.abs(w), 0, keepdim=True)[0]
65 + s *= 2 / max_q_val # 2 => symmetric
66 +
67 + # Quantize
68 + q_w = torch.round(w / s).int()
69 + q_w += half_q_val
70 + q_w = torch.clamp(q_w, 0, max_q_val)
71 +
72 + # Compute ref (dequantized)
73 + w_ref = (q_w - half_q_val).half() * s
74 +
75 + # Restore original shapes
76 + if group_size < size_k:
77 +
78 + def reshape_w(w):
79 + w = w.reshape((group_size, -1, size_n))
80 + w = w.permute(1, 0, 2)
81 + w = w.reshape((size_k, size_n)).contiguous()
82 + return w
83 +
84 + q_w = reshape_w(q_w)
85 + w_ref = reshape_w(w_ref)
86 +
87 + s = s.reshape((-1, size_n)).contiguous()
88 +
89 + # Apply act_order
90 + g_idx = torch.empty(0, dtype=torch.int, device=w.device)
91 + rand_perm = torch.empty(0, dtype=torch.int, device=w.device)
92 + if act_order:
93 + assert (
94 + group_size < size_k
95 + ), "For act_order, groupsize = {} must be less than size_k = {}".format(
96 + group_size, size_k)
97 +
98 + w_ref, q_w, g_idx, rand_perm = permute_rows(q_w, w_ref, group_size)
99 +
100 + return (
101 + w_ref.to(device=orig_device),
102 + q_w.to(device=orig_device),
103 + s.to(device=orig_device),
104 + g_idx.to(device=orig_device),
105 + rand_perm.to(device=orig_device),
106 + )
107 +
108 +
109 +def sort_weights(q_w: torch.Tensor, g_idx: torch.Tensor):
110 + orig_device = q_w.device
111 +
112 + sort_indices = torch.argsort(g_idx).to(
113 + dtype=torch.int32) # Sort based on g_idx
114 +
115 + g_idx = g_idx[sort_indices].contiguous()
116 + q_w = q_w[sort_indices, :].contiguous()
117 +
118 + return (
119 + q_w.to(device=orig_device),
120 + g_idx.to(device=orig_device),
121 + sort_indices.to(device=orig_device),
122 + )
123 +
124 +
125 +def gptq_pack(
126 + q_w: torch.Tensor,
127 + num_bits: int,
128 + size_k: int,
129 + size_n: int,
130 +):
131 + assert q_w.shape == (size_k, size_n)
132 +
133 + pack_factor = get_pack_factor(num_bits)
134 + assert size_k % pack_factor == 0
135 +
136 + orig_device = q_w.device
137 +
138 + q_w = q_w.cpu().numpy().astype(numpy.uint32)
139 +
140 + q_res = numpy.zeros((size_k // pack_factor, size_n), dtype=numpy.uint32)
141 +
142 + for i in range(pack_factor):
143 + q_res |= q_w[i::pack_factor, :] << num_bits * i
144 +
145 + q_res = torch.from_numpy(q_res.astype(numpy.int32)).to(orig_device)
146 + return q_res