A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Kernel] Add punica dimension for Qwen2 LoRA (#5441)
Jinzhen Lin committed
Jun 21, 2024 at 08:55 UTC
1f5674218f968dec625d0995fe5cd5d626db9188
2 files changed
+53
-2
csrc/punica/bgmv/bgmv_config.h
+36
-2
@@ -16,15 +16,20 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
16
f(in_T, out_T, W_T, narrow, 512) \
17
f(in_T, out_T, W_T, narrow, 640) \
18
f(in_T, out_T, W_T, narrow, 768) \
19
+ f(in_T, out_T, W_T, narrow, 896) \
20
f(in_T, out_T, W_T, narrow, 1024) \
21
f(in_T, out_T, W_T, narrow, 1152) \
22
+ f(in_T, out_T, W_T, narrow, 1216) \
23
f(in_T, out_T, W_T, narrow, 1280) \
24
f(in_T, out_T, W_T, narrow, 1536) \
25
f(in_T, out_T, W_T, narrow, 1664) \
26
f(in_T, out_T, W_T, narrow, 1728) \
27
f(in_T, out_T, W_T, narrow, 1792) \
28
f(in_T, out_T, W_T, narrow, 2048) \
29
+ f(in_T, out_T, W_T, narrow, 2240) \
30
f(in_T, out_T, W_T, narrow, 2304) \
31
+ f(in_T, out_T, W_T, narrow, 2368) \
32
+ f(in_T, out_T, W_T, narrow, 2432) \
33
f(in_T, out_T, W_T, narrow, 2560) \
34
f(in_T, out_T, W_T, narrow, 2752) \
35
f(in_T, out_T, W_T, narrow, 2816) \
@@ -32,8 +37,12 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
37
f(in_T, out_T, W_T, narrow, 3328) \
38
f(in_T, out_T, W_T, narrow, 3456) \
39
f(in_T, out_T, W_T, narrow, 3584) \
40
+ f(in_T, out_T, W_T, narrow, 3712) \
41
f(in_T, out_T, W_T, narrow, 4096) \
42
+ f(in_T, out_T, W_T, narrow, 4480) \
43
f(in_T, out_T, W_T, narrow, 4608) \
44
+ f(in_T, out_T, W_T, narrow, 4736) \
45
+ f(in_T, out_T, W_T, narrow, 4864) \
46
f(in_T, out_T, W_T, narrow, 5120) \
47
f(in_T, out_T, W_T, narrow, 5504) \
48
f(in_T, out_T, W_T, narrow, 5632) \
@@ -43,8 +52,11 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
52
f(in_T, out_T, W_T, narrow, 6848) \
53
f(in_T, out_T, W_T, narrow, 6912) \
54
f(in_T, out_T, W_T, narrow, 7168) \
55
+ f(in_T, out_T, W_T, narrow, 7424) \
56
f(in_T, out_T, W_T, narrow, 8192) \
57
+ f(in_T, out_T, W_T, narrow, 8960) \
58
f(in_T, out_T, W_T, narrow, 9216) \
59
+ f(in_T, out_T, W_T, narrow, 9472) \
60
f(in_T, out_T, W_T, narrow, 10240) \
61
f(in_T, out_T, W_T, narrow, 11008) \
62
f(in_T, out_T, W_T, narrow, 11264) \
@@ -52,8 +64,11 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
64
f(in_T, out_T, W_T, narrow, 13696) \
65
f(in_T, out_T, W_T, narrow, 13824) \
66
f(in_T, out_T, W_T, narrow, 14336) \
67
+ f(in_T, out_T, W_T, narrow, 14784) \
68
+ f(in_T, out_T, W_T, narrow, 14848) \
69
f(in_T, out_T, W_T, narrow, 15360) \
70
f(in_T, out_T, W_T, narrow, 16384) \
71
+ f(in_T, out_T, W_T, narrow, 18944) \
72
f(in_T, out_T, W_T, narrow, 20480) \
73
f(in_T, out_T, W_T, narrow, 22016) \
74
f(in_T, out_T, W_T, narrow, 22528) \
@@ -61,6 +76,8 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
76
f(in_T, out_T, W_T, narrow, 27392) \
77
f(in_T, out_T, W_T, narrow, 27648) \
78
f(in_T, out_T, W_T, narrow, 28672) \
79
+ f(in_T, out_T, W_T, narrow, 29568) \
80
+ f(in_T, out_T, W_T, narrow, 29696) \
81
f(in_T, out_T, W_T, narrow, 32000) \
82
f(in_T, out_T, W_T, narrow, 32256) \
83
f(in_T, out_T, W_T, narrow, 32512) \
@@ -85,9 +102,9 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
102
// Keep above in sync with vllm/lora/layers::LogitsProcessorWithLoRA
103
// and vllm/tests/lora/test_punica.py
104
88
-// Used for defining kernels going from the variety of
105
+// Used for defining kernels going from the variety of
106
// dim in to the narrow dim out
90
- // Using it for the fully sharded column
107
+ // Using it for the fully sharded column
108
// parallel LoRA A which splits the rank dim
109
#define FOR_INST_BGMV_NARROW(f, in_T, out_T, W_T, narrow) \
110
f(in_T, out_T, W_T, 128, narrow) \
@@ -95,15 +112,20 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
112
f(in_T, out_T, W_T, 512, narrow) \
113
f(in_T, out_T, W_T, 640, narrow) \
114
f(in_T, out_T, W_T, 768, narrow) \
115
+ f(in_T, out_T, W_T, 896, narrow) \
116
f(in_T, out_T, W_T, 1024, narrow) \
117
f(in_T, out_T, W_T, 1152, narrow) \
118
+ f(in_T, out_T, W_T, 1216, narrow) \
119
f(in_T, out_T, W_T, 1280, narrow) \
120
f(in_T, out_T, W_T, 1536, narrow) \
121
f(in_T, out_T, W_T, 1664, narrow) \
122
f(in_T, out_T, W_T, 1728, narrow) \
123
f(in_T, out_T, W_T, 1792, narrow) \
124
f(in_T, out_T, W_T, 2048, narrow) \
125
+ f(in_T, out_T, W_T, 2240, narrow) \
126
f(in_T, out_T, W_T, 2304, narrow) \
127
+ f(in_T, out_T, W_T, 2368, narrow) \
128
+ f(in_T, out_T, W_T, 2432, narrow) \
129
f(in_T, out_T, W_T, 2560, narrow) \
130
f(in_T, out_T, W_T, 2752, narrow) \
131
f(in_T, out_T, W_T, 2816, narrow) \
@@ -111,8 +133,12 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
133
f(in_T, out_T, W_T, 3328, narrow) \
134
f(in_T, out_T, W_T, 3456, narrow) \
135
f(in_T, out_T, W_T, 3584, narrow) \
136
+ f(in_T, out_T, W_T, 3712, narrow) \
137
f(in_T, out_T, W_T, 4096, narrow) \
138
+ f(in_T, out_T, W_T, 4480, narrow) \
139
f(in_T, out_T, W_T, 4608, narrow) \
140
+ f(in_T, out_T, W_T, 4736, narrow) \
141
+ f(in_T, out_T, W_T, 4864, narrow) \
142
f(in_T, out_T, W_T, 5120, narrow) \
143
f(in_T, out_T, W_T, 5504, narrow) \
144
f(in_T, out_T, W_T, 5632, narrow) \
@@ -122,8 +148,11 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
148
f(in_T, out_T, W_T, 6848, narrow) \
149
f(in_T, out_T, W_T, 6912, narrow) \
150
f(in_T, out_T, W_T, 7168, narrow) \
151
+ f(in_T, out_T, W_T, 7424, narrow) \
152
f(in_T, out_T, W_T, 8192, narrow) \
153
+ f(in_T, out_T, W_T, 8960, narrow) \
154
f(in_T, out_T, W_T, 9216, narrow) \
155
+ f(in_T, out_T, W_T, 9472, narrow) \
156
f(in_T, out_T, W_T, 10240, narrow) \
157
f(in_T, out_T, W_T, 11008, narrow) \
158
f(in_T, out_T, W_T, 11264, narrow) \
@@ -131,8 +160,11 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
160
f(in_T, out_T, W_T, 13696, narrow) \
161
f(in_T, out_T, W_T, 13824, narrow) \
162
f(in_T, out_T, W_T, 14336, narrow) \
163
+ f(in_T, out_T, W_T, 14784, narrow) \
164
+ f(in_T, out_T, W_T, 14848, narrow) \
165
f(in_T, out_T, W_T, 15360, narrow) \
166
f(in_T, out_T, W_T, 16384, narrow) \
167
+ f(in_T, out_T, W_T, 18944, narrow) \
168
f(in_T, out_T, W_T, 20480, narrow) \
169
f(in_T, out_T, W_T, 22016, narrow) \
170
f(in_T, out_T, W_T, 22528, narrow) \
@@ -140,6 +172,8 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
172
f(in_T, out_T, W_T, 27392, narrow) \
173
f(in_T, out_T, W_T, 27648, narrow) \
174
f(in_T, out_T, W_T, 28672, narrow) \
175
+ f(in_T, out_T, W_T, 29568, narrow) \
176
+ f(in_T, out_T, W_T, 29696, narrow) \
177
f(in_T, out_T, W_T, 32000, narrow) \
178
f(in_T, out_T, W_T, 32256, narrow) \
179
f(in_T, out_T, W_T, 32512, narrow) \
tests/lora/test_punica.py
+17
@@ -49,21 +49,30 @@ H1 = H2 = [
49
128,
50
256,
51
512,
52
+ 896,
53
1024,
54
1152,
55
+ 1216,
56
1280,
57
1536,
58
1664,
59
2048,
60
+ 2240,
61
2304,
62
+ 2368,
63
+ 2432,
64
2560,
65
2752,
66
3072,
67
3328,
68
3456,
69
3584,
70
+ 3712,
71
4096,
72
+ 4480,
73
4608,
74
+ 4736,
75
+ 4864,
76
5120,
77
5504,
78
5632,
@@ -73,19 +82,27 @@ H1 = H2 = [
82
6848,
83
6912,
84
7168,
85
+ 7424,
86
8192,
87
+ 8960,
88
9216,
89
+ 9472,
90
10240,
91
11008,
92
11264,
93
13824,
94
14336,
95
+ 14784,
96
+ 14848,
97
15360,
98
+ 18944,
99
22016,
100
22528,
101
24576,
102
27392,
103
27648,
104
+ 29568,
105
+ 29696,
106
32000,
107
32256,
108
32512,