@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6066253296
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

Marlin 24 prefill performance improvement (about 25% better on average) (#4983)

Alexander Matveev committed May 23, 2024 at 02:39 UTC 606625329648e6eff1883e23040adfad82f219cf
4 files changed +107 -32
benchmarks/kernels/benchmark_marlin.py
+62 -12
@@ -6,9 +6,13 @@ from benchmark_shapes import WEIGHT_SHAPES
6
7 from vllm import _custom_ops as ops
8 from vllm.model_executor.layers.quantization.gptq_marlin import (
9 + GPTQ_MARLIN_MAX_PARALLEL, GPTQ_MARLIN_MIN_THREAD_N,
10 GPTQ_MARLIN_SUPPORTED_GROUP_SIZES, GPTQ_MARLIN_SUPPORTED_NUM_BITS)
11 +from vllm.model_executor.layers.quantization.gptq_marlin_24 import (
12 + GPTQ_MARLIN_24_MAX_PARALLEL, GPTQ_MARLIN_24_MIN_THREAD_N,
13 + GPTQ_MARLIN_24_SUPPORTED_GROUP_SIZES, GPTQ_MARLIN_24_SUPPORTED_NUM_BITS)
14 from vllm.model_executor.layers.quantization.utils.marlin_utils import (
11 - MarlinWorkspace, marlin_quantize)
15 + MarlinWorkspace, marlin_24_quantize, marlin_quantize)
16 from vllm.model_executor.layers.quantization.utils.quant_utils import (
17 gptq_pack, quantize_weights, sort_weights)
18
@@ -44,6 +48,10 @@ def bench_run(results, model, act_order, is_k_full, num_bits, group_size,
48 marlin_rand_perm,
49 ) = marlin_quantize(b, num_bits, group_size, act_order)
50
51 + # Marlin_24 quant
52 + (marlin_24_w_ref, marlin_24_q_w_comp, marlin_24_meta,
53 + marlin_24_s) = marlin_24_quantize(b, num_bits, group_size)
54 +
55 # GPTQ quant
56 (w_ref, q_w, s, g_idx,
57 rand_perm) = quantize_weights(b, num_bits, group_size, act_order)
@@ -56,28 +64,43 @@ def bench_run(results, model, act_order, is_k_full, num_bits, group_size,
64 (q_w, g_idx, repack_sort_indices) = sort_weights(q_w, g_idx)
65
66 # Prepare
59 - marlin_workspace = MarlinWorkspace(size_n)
67 + marlin_workspace = MarlinWorkspace(size_n, GPTQ_MARLIN_MIN_THREAD_N,
68 + GPTQ_MARLIN_MAX_PARALLEL)
69 +
70 + marlin_24_workspace = MarlinWorkspace(size_n, GPTQ_MARLIN_24_MIN_THREAD_N,
71 + GPTQ_MARLIN_24_MAX_PARALLEL)
72
73 globals = {
74 + # Gen params
75 + "num_bits": num_bits,
76 + "group_size": group_size,
77 + "size_m": size_m,
78 + "size_n": size_n,
79 + "size_k": size_k,
80 + "a": a,
81 + "a_tmp": a_tmp,
82 + # Marlin params
83 "marlin_w_ref": marlin_w_ref,
84 "marlin_q_w": marlin_q_w,
85 "marlin_s": marlin_s,
86 "marlin_g_idx": marlin_g_idx,
87 "marlin_sort_indices": marlin_sort_indices,
88 "marlin_rand_perm": marlin_rand_perm,
89 + "marlin_workspace": marlin_workspace,
90 + "is_k_full": is_k_full,
91 + # Marlin_24 params
92 + "marlin_24_w_ref": marlin_24_w_ref,
93 + "marlin_24_q_w_comp": marlin_24_q_w_comp,
94 + "marlin_24_meta": marlin_24_meta,
95 + "marlin_24_s": marlin_24_s,
96 + "marlin_24_workspace": marlin_24_workspace,
97 + # GPTQ params
98 "q_w_gptq": q_w_gptq,
99 "repack_sort_indices": repack_sort_indices,
70 - "num_bits": num_bits,
71 - "group_size": group_size,
72 - "size_m": size_m,
73 - "size_n": size_n,
74 - "size_k": size_k,
75 - "is_k_full": is_k_full,
76 - "a": a,
77 - "a_tmp": a_tmp,
100 + # Kernels
101 "gptq_marlin_gemm": ops.gptq_marlin_gemm,
102 + "gptq_marlin_24_gemm": ops.gptq_marlin_24_gemm,
103 "gptq_marlin_repack": ops.gptq_marlin_repack,
80 - "marlin_workspace": marlin_workspace,
104 }
105
106 min_run_time = 1
@@ -105,6 +128,18 @@ def bench_run(results, model, act_order, is_k_full, num_bits, group_size,
128 description="gptq_marlin_gemm",
129 ).blocked_autorange(min_run_time=min_run_time))
130
131 + if (num_bits in GPTQ_MARLIN_24_SUPPORTED_NUM_BITS
132 + and group_size in GPTQ_MARLIN_24_SUPPORTED_GROUP_SIZES):
133 + results.append(
134 + benchmark.Timer(
135 + stmt=
136 + "output = gptq_marlin_24_gemm(a, marlin_24_q_w_comp, marlin_24_meta, marlin_24_s, marlin_24_workspace.scratch, num_bits, size_m, size_n, size_k)", # noqa: E501
137 + globals=globals,
138 + label=label,
139 + sub_label=sub_label,
140 + description="gptq_marlin_24_gemm",
141 + ).blocked_autorange(min_run_time=min_run_time))
142 +
143 results.append(
144 benchmark.Timer(
145 stmt=
@@ -135,8 +170,20 @@ def main(args):
170 continue
171
172 for act_order in ACT_ORDER_OPTS:
173 + if len(args.limit_act_order
174 + ) > 0 and act_order not in args.limit_act_order:
175 + continue
176 +
177 for is_k_full in K_FULL_OPTS:
178 + if len(args.limit_k_full
179 + ) > 0 and is_k_full not in args.limit_k_full:
180 + continue
181 +
182 for num_bits in GPTQ_MARLIN_SUPPORTED_NUM_BITS:
183 + if len(args.limit_num_bits
184 + ) > 0 and num_bits not in args.limit_num_bits:
185 + continue
186 +
187 for group_size in GPTQ_MARLIN_SUPPORTED_GROUP_SIZES:
188 if len(
189 args.limit_group_size
@@ -159,7 +206,7 @@ def main(args):
206
207
208 # For quick benchmarking use:
162 -# python benchmark_marlin.py --batch-sizes 1 16 32 --limit-k 4096 --limit-n 4096 --limit-group-size 128 # noqa E501
209 +# python benchmark_marlin.py --batch-sizes 1 16 32 --limit-k 4096 --limit-n 4096 --limit-group-size 128 --limit-num-bits 4 --limit-act-order 0 --limit-k-full 1 # noqa E501
210 #
211 if __name__ == "__main__":
212 parser = argparse.ArgumentParser(
@@ -178,6 +225,9 @@ if __name__ == "__main__":
225 parser.add_argument("--limit-k", nargs="+", type=int, default=[])
226 parser.add_argument("--limit-n", nargs="+", type=int, default=[])
227 parser.add_argument("--limit-group-size", nargs="+", type=int, default=[])
228 + parser.add_argument("--limit-num-bits", nargs="+", type=int, default=[])
229 + parser.add_argument("--limit-act-order", nargs="+", type=int, default=[])
230 + parser.add_argument("--limit-k-full", nargs="+", type=int, default=[])
231
232 args = parser.parse_args()
233 main(args)
csrc/quantization/marlin/sparse/marlin_24_cuda_kernel.cu
+40 -15
@@ -48,12 +48,12 @@ namespace marlin_24 {
48 // than 1 warp per schedule allows some more latency hiding. At the same time,
49 // we want relatively few warps to have many registers per warp and small tiles.
50 static constexpr int THREADS = 256;
51 -static constexpr int STAGES = 4; // 4 pipeline stages fit into shared memory
51 +static constexpr int STAGES = 4;
52
53 static constexpr int min_thread_n = 128;
54
55 static constexpr int tile_size = 16;
56 -static constexpr int max_par = 16;
56 +static constexpr int max_par = 64;
57
58 #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
59
@@ -736,10 +736,10 @@ __global__ void Marlin_24(
736 for (int pipe = 0; pipe < stages;) {
737 fetch_to_shared((pipe + stages - 1) % stages, pipe,
738 slice_iters >= stages);
739 + matmul(pipe);
740 wait_for_stage();
741
742 fetch_to_registers(pipe + 1, (pipe + 1) % stages);
742 - matmul(pipe);
743
744 pipe++;
745 slice_iters--;
@@ -899,9 +899,12 @@ void marlin_cuda_2_4(const void* A, const void* B, const void* meta, void* C,
899 // than better compute utilization
900 thread_k = 128;
901 thread_m = 128;
902 - } else {
902 + } else if (prob_n <= 256) {
903 thread_k = 64;
904 thread_m = 256;
905 + } else {
906 + thread_k = 32;
907 + thread_m = 512;
908 }
909 }
910
@@ -928,19 +931,21 @@ void marlin_cuda_2_4(const void* A, const void* B, const void* meta, void* C,
931 int4* C_ptr = (int4*)C;
932 const int4* s_ptr = (const int4*)s;
933
934 + constexpr int max_m_blocks = 4;
935 +
936 int* locks = (int*)workspace;
932 - for (int i = 0; i < tot_n_blocks; i += 4) {
937 + for (int i = 0; i < tot_n_blocks; i += max_m_blocks) {
938 int thread_n_blocks = tot_n_blocks - i;
939 prob_n = tot_n - 16 * i;
940 int par = 1;
936 - if (thread_n_blocks > 4) {
941 + if (thread_n_blocks > max_m_blocks) {
942 // Note that parallel > 1 currently only works for inputs without any
943 // padding
939 - par = (16 * thread_n_blocks - pad) / 64;
944 + par = (16 * thread_n_blocks - pad) / (max_m_blocks * 16);
945 if (par > max_par) par = max_par;
941 - prob_n = 64 * par;
942 - i += 4 * (par - 1);
943 - thread_n_blocks = 4;
946 + prob_n = (max_m_blocks * 16) * par;
947 + i += max_m_blocks * (par - 1);
948 + thread_n_blocks = max_m_blocks;
949 }
950
951 // For compilation speed, we only define the kernel configurations that have
@@ -951,8 +956,9 @@ void marlin_cuda_2_4(const void* A, const void* B, const void* meta, void* C,
956 if (false) {
957 } // BMxBNxBK, group
958 // 4-bit
954 - CALL_IF_2_4(4, 8, 1, 4, -1) // e.g., 16x128x128
955 - CALL_IF_2_4(4, 8, 1, 4, 4) // e.g., 16x128x128, 64
959 + CALL_IF_2_4(4, 8, 1, 4, -1) // e.g., 16x128x128
960 + CALL_IF_2_4(4, 8, 1, 4, 4) // e.g., 16x128x128, 64
961 +
962 CALL_IF_2_4(4, 16, 1, 2, -1) // e.g., 16x256x64
963 CALL_IF_2_4(4, 16, 1, 2, 4) // e.g., 16x256x64, 64
964 CALL_IF_2_4(4, 16, 2, 2, -1) // e.g.. 32x256x64
@@ -962,9 +968,19 @@ void marlin_cuda_2_4(const void* A, const void* B, const void* meta, void* C,
968 CALL_IF_2_4(4, 16, 4, 2, -1)
969 CALL_IF_2_4(4, 16, 4, 2, 4)
970
971 + CALL_IF_2_4(4, 32, 1, 1, -1) // e.g., 16x256x64
972 + CALL_IF_2_4(4, 32, 1, 1, 4) // e.g., 16x256x64, 64
973 + CALL_IF_2_4(4, 32, 2, 1, -1) // e.g.. 32x256x64
974 + CALL_IF_2_4(4, 32, 2, 1, 4)
975 + CALL_IF_2_4(4, 32, 3, 1, -1)
976 + CALL_IF_2_4(4, 32, 3, 1, 4)
977 + CALL_IF_2_4(4, 32, 4, 1, -1)
978 + CALL_IF_2_4(4, 32, 4, 1, 4)
979 +
980 // 8-bit
966 - CALL_IF_2_4(8, 8, 1, 4, -1) // e.g., 16x128x128
967 - CALL_IF_2_4(8, 8, 1, 4, 4) // e.g., 16x128x128, 64
981 + CALL_IF_2_4(8, 8, 1, 4, -1) // e.g., 16x128x128
982 + CALL_IF_2_4(8, 8, 1, 4, 4) // e.g., 16x128x128, 64
983 +
984 CALL_IF_2_4(8, 16, 1, 2, -1) // e.g., 16x256x64
985 CALL_IF_2_4(8, 16, 1, 2, 4) // e.g., 16x256x64, 64
986 CALL_IF_2_4(8, 16, 2, 2, -1) // e.g.. 32x256x64
@@ -973,6 +989,15 @@ void marlin_cuda_2_4(const void* A, const void* B, const void* meta, void* C,
989 CALL_IF_2_4(8, 16, 3, 2, 4)
990 CALL_IF_2_4(8, 16, 4, 2, -1)
991 CALL_IF_2_4(8, 16, 4, 2, 4)
992 +
993 + CALL_IF_2_4(8, 32, 1, 1, -1) // e.g., 16x256x64
994 + CALL_IF_2_4(8, 32, 1, 1, 4) // e.g., 16x256x64, 64
995 + CALL_IF_2_4(8, 32, 2, 1, -1) // e.g.. 32x256x64
996 + CALL_IF_2_4(8, 32, 2, 1, 4)
997 + CALL_IF_2_4(8, 32, 3, 1, -1)
998 + CALL_IF_2_4(8, 32, 3, 1, 4)
999 + CALL_IF_2_4(8, 32, 4, 1, -1)
1000 + CALL_IF_2_4(8, 32, 4, 1, 4)
1001 else {
1002 throw std::runtime_error("Unsupported shapes: MKN = [" + str(prob_m) +
1003 ", " + str(prob_k) + ", " + str(prob_n) + "]" +
@@ -1062,7 +1087,7 @@ torch::Tensor gptq_marlin_24_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
1087 int thread_k = -1;
1088 int thread_m = -1;
1089 int sms = -1;
1065 - int max_par = 16;
1090 + int max_par = marlin_24::max_par;
1091
1092 int groupsize = -1;
1093 if (b_scales.size(0) > 1) {
tests/kernels/test_marlin_gemm.py
+1 -1
@@ -27,7 +27,7 @@ MARLIN_K_CHUNKS = [128]
27 MARLIN_N_CHUNKS = [64, 128, 256]
28
29 MARLIN_24_K_CHUNKS = [128]
30 -MARLIN_24_N_CHUNKS = [256]
30 +MARLIN_24_N_CHUNKS = [512]
31
32 MNK_FACTORS = [
33 (1, 1, 1),
vllm/model_executor/layers/quantization/gptq_marlin_24.py
+4 -4
@@ -15,7 +15,7 @@ logger = init_logger(__name__)
15 GPTQ_MARLIN_24_TILE = 16
16 GPTQ_MARLIN_24_MIN_THREAD_N = 128
17 GPTQ_MARLIN_24_MIN_THREAD_K = 128
18 -GPTQ_MARLIN_24_MAX_PARALLEL = 16
18 +GPTQ_MARLIN_24_MAX_PARALLEL = 64
19
20 GPTQ_MARLIN_24_SUPPORTED_NUM_BITS = [4, 8]
21 GPTQ_MARLIN_24_SUPPORTED_GROUP_SIZES = [-1, 128]
@@ -53,14 +53,14 @@ class GPTQMarlin24Config(QuantizationConfig):
53 self.tile_size = 16
54
55 # Min out_features dim
56 - self.min_n_threads = 128
56 + self.min_n_threads = GPTQ_MARLIN_24_MIN_THREAD_N
57
58 # Min in_features dim
59 - self.min_k_threads = 128
59 + self.min_k_threads = GPTQ_MARLIN_24_MIN_THREAD_K
60
61 # Max parallel problems to solve at once (improves large
62 # batch performance)
63 - self.max_parallel = 16
63 + self.max_parallel = GPTQ_MARLIN_24_MAX_PARALLEL
64
65 # Permutation length used by the marlin kernels.
66 self.perm_len = 1024