A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
Marlin 24 prefill performance improvement (about 25% better on average) (#4983)
Alexander Matveev committed
May 23, 2024 at 02:39 UTC
606625329648e6eff1883e23040adfad82f219cf
4 files changed
+107
-32
benchmarks/kernels/benchmark_marlin.py
+62
-12
@@ -6,9 +6,13 @@ from benchmark_shapes import WEIGHT_SHAPES
6
7
from vllm import _custom_ops as ops
8
from vllm.model_executor.layers.quantization.gptq_marlin import (
9
+ GPTQ_MARLIN_MAX_PARALLEL, GPTQ_MARLIN_MIN_THREAD_N,
10
GPTQ_MARLIN_SUPPORTED_GROUP_SIZES, GPTQ_MARLIN_SUPPORTED_NUM_BITS)
11
+from vllm.model_executor.layers.quantization.gptq_marlin_24 import (
12
+ GPTQ_MARLIN_24_MAX_PARALLEL, GPTQ_MARLIN_24_MIN_THREAD_N,
13
+ GPTQ_MARLIN_24_SUPPORTED_GROUP_SIZES, GPTQ_MARLIN_24_SUPPORTED_NUM_BITS)
14
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
11
- MarlinWorkspace, marlin_quantize)
15
+ MarlinWorkspace, marlin_24_quantize, marlin_quantize)
16
from vllm.model_executor.layers.quantization.utils.quant_utils import (
17
gptq_pack, quantize_weights, sort_weights)
18
@@ -44,6 +48,10 @@ def bench_run(results, model, act_order, is_k_full, num_bits, group_size,
48
marlin_rand_perm,
49
) = marlin_quantize(b, num_bits, group_size, act_order)
50
51
+ # Marlin_24 quant
52
+ (marlin_24_w_ref, marlin_24_q_w_comp, marlin_24_meta,
53
+ marlin_24_s) = marlin_24_quantize(b, num_bits, group_size)
54
+
55
# GPTQ quant
56
(w_ref, q_w, s, g_idx,
57
rand_perm) = quantize_weights(b, num_bits, group_size, act_order)
@@ -56,28 +64,43 @@ def bench_run(results, model, act_order, is_k_full, num_bits, group_size,
64
(q_w, g_idx, repack_sort_indices) = sort_weights(q_w, g_idx)
65
66
# Prepare
59
- marlin_workspace = MarlinWorkspace(size_n)
67
+ marlin_workspace = MarlinWorkspace(size_n, GPTQ_MARLIN_MIN_THREAD_N,
68
+ GPTQ_MARLIN_MAX_PARALLEL)
69
+
70
+ marlin_24_workspace = MarlinWorkspace(size_n, GPTQ_MARLIN_24_MIN_THREAD_N,
71
+ GPTQ_MARLIN_24_MAX_PARALLEL)
72
73
globals = {
74
+ # Gen params
75
+ "num_bits": num_bits,
76
+ "group_size": group_size,
77
+ "size_m": size_m,
78
+ "size_n": size_n,
79
+ "size_k": size_k,
80
+ "a": a,
81
+ "a_tmp": a_tmp,
82
+ # Marlin params
83
"marlin_w_ref": marlin_w_ref,
84
"marlin_q_w": marlin_q_w,
85
"marlin_s": marlin_s,
86
"marlin_g_idx": marlin_g_idx,
87
"marlin_sort_indices": marlin_sort_indices,
88
"marlin_rand_perm": marlin_rand_perm,
89
+ "marlin_workspace": marlin_workspace,
90
+ "is_k_full": is_k_full,
91
+ # Marlin_24 params
92
+ "marlin_24_w_ref": marlin_24_w_ref,
93
+ "marlin_24_q_w_comp": marlin_24_q_w_comp,
94
+ "marlin_24_meta": marlin_24_meta,
95
+ "marlin_24_s": marlin_24_s,
96
+ "marlin_24_workspace": marlin_24_workspace,
97
+ # GPTQ params
98
"q_w_gptq": q_w_gptq,
99
"repack_sort_indices": repack_sort_indices,
70
- "num_bits": num_bits,
71
- "group_size": group_size,
72
- "size_m": size_m,
73
- "size_n": size_n,
74
- "size_k": size_k,
75
- "is_k_full": is_k_full,
76
- "a": a,
77
- "a_tmp": a_tmp,
100
+ # Kernels
101
"gptq_marlin_gemm": ops.gptq_marlin_gemm,
102
+ "gptq_marlin_24_gemm": ops.gptq_marlin_24_gemm,
103
"gptq_marlin_repack": ops.gptq_marlin_repack,
80
- "marlin_workspace": marlin_workspace,
104
}
105
106
min_run_time = 1
@@ -105,6 +128,18 @@ def bench_run(results, model, act_order, is_k_full, num_bits, group_size,
128
description="gptq_marlin_gemm",
129
).blocked_autorange(min_run_time=min_run_time))
130
131
+ if (num_bits in GPTQ_MARLIN_24_SUPPORTED_NUM_BITS
132
+ and group_size in GPTQ_MARLIN_24_SUPPORTED_GROUP_SIZES):
133
+ results.append(
134
+ benchmark.Timer(
135
+ stmt=
136
+ "output = gptq_marlin_24_gemm(a, marlin_24_q_w_comp, marlin_24_meta, marlin_24_s, marlin_24_workspace.scratch, num_bits, size_m, size_n, size_k)", # noqa: E501
137
+ globals=globals,
138
+ label=label,
139
+ sub_label=sub_label,
140
+ description="gptq_marlin_24_gemm",
141
+ ).blocked_autorange(min_run_time=min_run_time))
142
+
143
results.append(
144
benchmark.Timer(
145
stmt=
@@ -135,8 +170,20 @@ def main(args):
170
continue
171
172
for act_order in ACT_ORDER_OPTS:
173
+ if len(args.limit_act_order
174
+ ) > 0 and act_order not in args.limit_act_order:
175
+ continue
176
+
177
for is_k_full in K_FULL_OPTS:
178
+ if len(args.limit_k_full
179
+ ) > 0 and is_k_full not in args.limit_k_full:
180
+ continue
181
+
182
for num_bits in GPTQ_MARLIN_SUPPORTED_NUM_BITS:
183
+ if len(args.limit_num_bits
184
+ ) > 0 and num_bits not in args.limit_num_bits:
185
+ continue
186
+
187
for group_size in GPTQ_MARLIN_SUPPORTED_GROUP_SIZES:
188
if len(
189
args.limit_group_size
@@ -159,7 +206,7 @@ def main(args):
206
207
208
# For quick benchmarking use:
162
-# python benchmark_marlin.py --batch-sizes 1 16 32 --limit-k 4096 --limit-n 4096 --limit-group-size 128 # noqa E501
209
+# python benchmark_marlin.py --batch-sizes 1 16 32 --limit-k 4096 --limit-n 4096 --limit-group-size 128 --limit-num-bits 4 --limit-act-order 0 --limit-k-full 1 # noqa E501
210
#
211
if __name__ == "__main__":
212
parser = argparse.ArgumentParser(
@@ -178,6 +225,9 @@ if __name__ == "__main__":
225
parser.add_argument("--limit-k", nargs="+", type=int, default=[])
226
parser.add_argument("--limit-n", nargs="+", type=int, default=[])
227
parser.add_argument("--limit-group-size", nargs="+", type=int, default=[])
228
+ parser.add_argument("--limit-num-bits", nargs="+", type=int, default=[])
229
+ parser.add_argument("--limit-act-order", nargs="+", type=int, default=[])
230
+ parser.add_argument("--limit-k-full", nargs="+", type=int, default=[])
231
232
args = parser.parse_args()
233
main(args)
csrc/quantization/marlin/sparse/marlin_24_cuda_kernel.cu
+40
-15
@@ -48,12 +48,12 @@ namespace marlin_24 {
48
// than 1 warp per schedule allows some more latency hiding. At the same time,
49
// we want relatively few warps to have many registers per warp and small tiles.
50
static constexpr int THREADS = 256;
51
-static constexpr int STAGES = 4; // 4 pipeline stages fit into shared memory
51
+static constexpr int STAGES = 4;
52
53
static constexpr int min_thread_n = 128;
54
55
static constexpr int tile_size = 16;
56
-static constexpr int max_par = 16;
56
+static constexpr int max_par = 64;
57
58
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
59
@@ -736,10 +736,10 @@ __global__ void Marlin_24(
736
for (int pipe = 0; pipe < stages;) {
737
fetch_to_shared((pipe + stages - 1) % stages, pipe,
738
slice_iters >= stages);
739
+ matmul(pipe);
740
wait_for_stage();
741
742
fetch_to_registers(pipe + 1, (pipe + 1) % stages);
742
- matmul(pipe);
743
744
pipe++;
745
slice_iters--;
@@ -899,9 +899,12 @@ void marlin_cuda_2_4(const void* A, const void* B, const void* meta, void* C,
899
// than better compute utilization
900
thread_k = 128;
901
thread_m = 128;
902
- } else {
902
+ } else if (prob_n <= 256) {
903
thread_k = 64;
904
thread_m = 256;
905
+ } else {
906
+ thread_k = 32;
907
+ thread_m = 512;
908
}
909
}
910
@@ -928,19 +931,21 @@ void marlin_cuda_2_4(const void* A, const void* B, const void* meta, void* C,
931
int4* C_ptr = (int4*)C;
932
const int4* s_ptr = (const int4*)s;
933
934
+ constexpr int max_m_blocks = 4;
935
+
936
int* locks = (int*)workspace;
932
- for (int i = 0; i < tot_n_blocks; i += 4) {
937
+ for (int i = 0; i < tot_n_blocks; i += max_m_blocks) {
938
int thread_n_blocks = tot_n_blocks - i;
939
prob_n = tot_n - 16 * i;
940
int par = 1;
936
- if (thread_n_blocks > 4) {
941
+ if (thread_n_blocks > max_m_blocks) {
942
// Note that parallel > 1 currently only works for inputs without any
943
// padding
939
- par = (16 * thread_n_blocks - pad) / 64;
944
+ par = (16 * thread_n_blocks - pad) / (max_m_blocks * 16);
945
if (par > max_par) par = max_par;
941
- prob_n = 64 * par;
942
- i += 4 * (par - 1);
943
- thread_n_blocks = 4;
946
+ prob_n = (max_m_blocks * 16) * par;
947
+ i += max_m_blocks * (par - 1);
948
+ thread_n_blocks = max_m_blocks;
949
}
950
951
// For compilation speed, we only define the kernel configurations that have
@@ -951,8 +956,9 @@ void marlin_cuda_2_4(const void* A, const void* B, const void* meta, void* C,
956
if (false) {
957
} // BMxBNxBK, group
958
// 4-bit
954
- CALL_IF_2_4(4, 8, 1, 4, -1) // e.g., 16x128x128
955
- CALL_IF_2_4(4, 8, 1, 4, 4) // e.g., 16x128x128, 64
959
+ CALL_IF_2_4(4, 8, 1, 4, -1) // e.g., 16x128x128
960
+ CALL_IF_2_4(4, 8, 1, 4, 4) // e.g., 16x128x128, 64
961
+
962
CALL_IF_2_4(4, 16, 1, 2, -1) // e.g., 16x256x64
963
CALL_IF_2_4(4, 16, 1, 2, 4) // e.g., 16x256x64, 64
964
CALL_IF_2_4(4, 16, 2, 2, -1) // e.g.. 32x256x64
@@ -962,9 +968,19 @@ void marlin_cuda_2_4(const void* A, const void* B, const void* meta, void* C,
968
CALL_IF_2_4(4, 16, 4, 2, -1)
969
CALL_IF_2_4(4, 16, 4, 2, 4)
970
971
+ CALL_IF_2_4(4, 32, 1, 1, -1) // e.g., 16x256x64
972
+ CALL_IF_2_4(4, 32, 1, 1, 4) // e.g., 16x256x64, 64
973
+ CALL_IF_2_4(4, 32, 2, 1, -1) // e.g.. 32x256x64
974
+ CALL_IF_2_4(4, 32, 2, 1, 4)
975
+ CALL_IF_2_4(4, 32, 3, 1, -1)
976
+ CALL_IF_2_4(4, 32, 3, 1, 4)
977
+ CALL_IF_2_4(4, 32, 4, 1, -1)
978
+ CALL_IF_2_4(4, 32, 4, 1, 4)
979
+
980
// 8-bit
966
- CALL_IF_2_4(8, 8, 1, 4, -1) // e.g., 16x128x128
967
- CALL_IF_2_4(8, 8, 1, 4, 4) // e.g., 16x128x128, 64
981
+ CALL_IF_2_4(8, 8, 1, 4, -1) // e.g., 16x128x128
982
+ CALL_IF_2_4(8, 8, 1, 4, 4) // e.g., 16x128x128, 64
983
+
984
CALL_IF_2_4(8, 16, 1, 2, -1) // e.g., 16x256x64
985
CALL_IF_2_4(8, 16, 1, 2, 4) // e.g., 16x256x64, 64
986
CALL_IF_2_4(8, 16, 2, 2, -1) // e.g.. 32x256x64
@@ -973,6 +989,15 @@ void marlin_cuda_2_4(const void* A, const void* B, const void* meta, void* C,
989
CALL_IF_2_4(8, 16, 3, 2, 4)
990
CALL_IF_2_4(8, 16, 4, 2, -1)
991
CALL_IF_2_4(8, 16, 4, 2, 4)
992
+
993
+ CALL_IF_2_4(8, 32, 1, 1, -1) // e.g., 16x256x64
994
+ CALL_IF_2_4(8, 32, 1, 1, 4) // e.g., 16x256x64, 64
995
+ CALL_IF_2_4(8, 32, 2, 1, -1) // e.g.. 32x256x64
996
+ CALL_IF_2_4(8, 32, 2, 1, 4)
997
+ CALL_IF_2_4(8, 32, 3, 1, -1)
998
+ CALL_IF_2_4(8, 32, 3, 1, 4)
999
+ CALL_IF_2_4(8, 32, 4, 1, -1)
1000
+ CALL_IF_2_4(8, 32, 4, 1, 4)
1001
else {
1002
throw std::runtime_error("Unsupported shapes: MKN = [" + str(prob_m) +
1003
", " + str(prob_k) + ", " + str(prob_n) + "]" +
@@ -1062,7 +1087,7 @@ torch::Tensor gptq_marlin_24_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
1087
int thread_k = -1;
1088
int thread_m = -1;
1089
int sms = -1;
1065
- int max_par = 16;
1090
+ int max_par = marlin_24::max_par;
1091
1092
int groupsize = -1;
1093
if (b_scales.size(0) > 1) {
tests/kernels/test_marlin_gemm.py
+1
-1
@@ -27,7 +27,7 @@ MARLIN_K_CHUNKS = [128]
27
MARLIN_N_CHUNKS = [64, 128, 256]
28
29
MARLIN_24_K_CHUNKS = [128]
30
-MARLIN_24_N_CHUNKS = [256]
30
+MARLIN_24_N_CHUNKS = [512]
31
32
MNK_FACTORS = [
33
(1, 1, 1),
vllm/model_executor/layers/quantization/gptq_marlin_24.py
+4
-4
@@ -15,7 +15,7 @@ logger = init_logger(__name__)
15
GPTQ_MARLIN_24_TILE = 16
16
GPTQ_MARLIN_24_MIN_THREAD_N = 128
17
GPTQ_MARLIN_24_MIN_THREAD_K = 128
18
-GPTQ_MARLIN_24_MAX_PARALLEL = 16
18
+GPTQ_MARLIN_24_MAX_PARALLEL = 64
19
20
GPTQ_MARLIN_24_SUPPORTED_NUM_BITS = [4, 8]
21
GPTQ_MARLIN_24_SUPPORTED_GROUP_SIZES = [-1, 128]
@@ -53,14 +53,14 @@ class GPTQMarlin24Config(QuantizationConfig):
53
self.tile_size = 16
54
55
# Min out_features dim
56
- self.min_n_threads = 128
56
+ self.min_n_threads = GPTQ_MARLIN_24_MIN_THREAD_N
57
58
# Min in_features dim
59
- self.min_k_threads = 128
59
+ self.min_k_threads = GPTQ_MARLIN_24_MIN_THREAD_K
60
61
# Max parallel problems to solve at once (improves large
62
# batch performance)
63
- self.max_parallel = 16
63
+ self.max_parallel = GPTQ_MARLIN_24_MAX_PARALLEL
64
65
# Permutation length used by the marlin kernels.
66
self.perm_len = 1024