@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1f5674218f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Kernel] Add punica dimension for Qwen2 LoRA (#5441)

Jinzhen Lin committed Jun 21, 2024 at 08:55 UTC 1f5674218f968dec625d0995fe5cd5d626db9188
2 files changed +53 -2
csrc/punica/bgmv/bgmv_config.h
+36 -2
@@ -16,15 +16,20 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
16 f(in_T, out_T, W_T, narrow, 512) \
17 f(in_T, out_T, W_T, narrow, 640) \
18 f(in_T, out_T, W_T, narrow, 768) \
19 + f(in_T, out_T, W_T, narrow, 896) \
20 f(in_T, out_T, W_T, narrow, 1024) \
21 f(in_T, out_T, W_T, narrow, 1152) \
22 + f(in_T, out_T, W_T, narrow, 1216) \
23 f(in_T, out_T, W_T, narrow, 1280) \
24 f(in_T, out_T, W_T, narrow, 1536) \
25 f(in_T, out_T, W_T, narrow, 1664) \
26 f(in_T, out_T, W_T, narrow, 1728) \
27 f(in_T, out_T, W_T, narrow, 1792) \
28 f(in_T, out_T, W_T, narrow, 2048) \
29 + f(in_T, out_T, W_T, narrow, 2240) \
30 f(in_T, out_T, W_T, narrow, 2304) \
31 + f(in_T, out_T, W_T, narrow, 2368) \
32 + f(in_T, out_T, W_T, narrow, 2432) \
33 f(in_T, out_T, W_T, narrow, 2560) \
34 f(in_T, out_T, W_T, narrow, 2752) \
35 f(in_T, out_T, W_T, narrow, 2816) \
@@ -32,8 +37,12 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
37 f(in_T, out_T, W_T, narrow, 3328) \
38 f(in_T, out_T, W_T, narrow, 3456) \
39 f(in_T, out_T, W_T, narrow, 3584) \
40 + f(in_T, out_T, W_T, narrow, 3712) \
41 f(in_T, out_T, W_T, narrow, 4096) \
42 + f(in_T, out_T, W_T, narrow, 4480) \
43 f(in_T, out_T, W_T, narrow, 4608) \
44 + f(in_T, out_T, W_T, narrow, 4736) \
45 + f(in_T, out_T, W_T, narrow, 4864) \
46 f(in_T, out_T, W_T, narrow, 5120) \
47 f(in_T, out_T, W_T, narrow, 5504) \
48 f(in_T, out_T, W_T, narrow, 5632) \
@@ -43,8 +52,11 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
52 f(in_T, out_T, W_T, narrow, 6848) \
53 f(in_T, out_T, W_T, narrow, 6912) \
54 f(in_T, out_T, W_T, narrow, 7168) \
55 + f(in_T, out_T, W_T, narrow, 7424) \
56 f(in_T, out_T, W_T, narrow, 8192) \
57 + f(in_T, out_T, W_T, narrow, 8960) \
58 f(in_T, out_T, W_T, narrow, 9216) \
59 + f(in_T, out_T, W_T, narrow, 9472) \
60 f(in_T, out_T, W_T, narrow, 10240) \
61 f(in_T, out_T, W_T, narrow, 11008) \
62 f(in_T, out_T, W_T, narrow, 11264) \
@@ -52,8 +64,11 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
64 f(in_T, out_T, W_T, narrow, 13696) \
65 f(in_T, out_T, W_T, narrow, 13824) \
66 f(in_T, out_T, W_T, narrow, 14336) \
67 + f(in_T, out_T, W_T, narrow, 14784) \
68 + f(in_T, out_T, W_T, narrow, 14848) \
69 f(in_T, out_T, W_T, narrow, 15360) \
70 f(in_T, out_T, W_T, narrow, 16384) \
71 + f(in_T, out_T, W_T, narrow, 18944) \
72 f(in_T, out_T, W_T, narrow, 20480) \
73 f(in_T, out_T, W_T, narrow, 22016) \
74 f(in_T, out_T, W_T, narrow, 22528) \
@@ -61,6 +76,8 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
76 f(in_T, out_T, W_T, narrow, 27392) \
77 f(in_T, out_T, W_T, narrow, 27648) \
78 f(in_T, out_T, W_T, narrow, 28672) \
79 + f(in_T, out_T, W_T, narrow, 29568) \
80 + f(in_T, out_T, W_T, narrow, 29696) \
81 f(in_T, out_T, W_T, narrow, 32000) \
82 f(in_T, out_T, W_T, narrow, 32256) \
83 f(in_T, out_T, W_T, narrow, 32512) \
@@ -85,9 +102,9 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
102 // Keep above in sync with vllm/lora/layers::LogitsProcessorWithLoRA
103 // and vllm/tests/lora/test_punica.py
104
88 -// Used for defining kernels going from the variety of
105 +// Used for defining kernels going from the variety of
106 // dim in to the narrow dim out
90 - // Using it for the fully sharded column
107 + // Using it for the fully sharded column
108 // parallel LoRA A which splits the rank dim
109 #define FOR_INST_BGMV_NARROW(f, in_T, out_T, W_T, narrow) \
110 f(in_T, out_T, W_T, 128, narrow) \
@@ -95,15 +112,20 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
112 f(in_T, out_T, W_T, 512, narrow) \
113 f(in_T, out_T, W_T, 640, narrow) \
114 f(in_T, out_T, W_T, 768, narrow) \
115 + f(in_T, out_T, W_T, 896, narrow) \
116 f(in_T, out_T, W_T, 1024, narrow) \
117 f(in_T, out_T, W_T, 1152, narrow) \
118 + f(in_T, out_T, W_T, 1216, narrow) \
119 f(in_T, out_T, W_T, 1280, narrow) \
120 f(in_T, out_T, W_T, 1536, narrow) \
121 f(in_T, out_T, W_T, 1664, narrow) \
122 f(in_T, out_T, W_T, 1728, narrow) \
123 f(in_T, out_T, W_T, 1792, narrow) \
124 f(in_T, out_T, W_T, 2048, narrow) \
125 + f(in_T, out_T, W_T, 2240, narrow) \
126 f(in_T, out_T, W_T, 2304, narrow) \
127 + f(in_T, out_T, W_T, 2368, narrow) \
128 + f(in_T, out_T, W_T, 2432, narrow) \
129 f(in_T, out_T, W_T, 2560, narrow) \
130 f(in_T, out_T, W_T, 2752, narrow) \
131 f(in_T, out_T, W_T, 2816, narrow) \
@@ -111,8 +133,12 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
133 f(in_T, out_T, W_T, 3328, narrow) \
134 f(in_T, out_T, W_T, 3456, narrow) \
135 f(in_T, out_T, W_T, 3584, narrow) \
136 + f(in_T, out_T, W_T, 3712, narrow) \
137 f(in_T, out_T, W_T, 4096, narrow) \
138 + f(in_T, out_T, W_T, 4480, narrow) \
139 f(in_T, out_T, W_T, 4608, narrow) \
140 + f(in_T, out_T, W_T, 4736, narrow) \
141 + f(in_T, out_T, W_T, 4864, narrow) \
142 f(in_T, out_T, W_T, 5120, narrow) \
143 f(in_T, out_T, W_T, 5504, narrow) \
144 f(in_T, out_T, W_T, 5632, narrow) \
@@ -122,8 +148,11 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
148 f(in_T, out_T, W_T, 6848, narrow) \
149 f(in_T, out_T, W_T, 6912, narrow) \
150 f(in_T, out_T, W_T, 7168, narrow) \
151 + f(in_T, out_T, W_T, 7424, narrow) \
152 f(in_T, out_T, W_T, 8192, narrow) \
153 + f(in_T, out_T, W_T, 8960, narrow) \
154 f(in_T, out_T, W_T, 9216, narrow) \
155 + f(in_T, out_T, W_T, 9472, narrow) \
156 f(in_T, out_T, W_T, 10240, narrow) \
157 f(in_T, out_T, W_T, 11008, narrow) \
158 f(in_T, out_T, W_T, 11264, narrow) \
@@ -131,8 +160,11 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
160 f(in_T, out_T, W_T, 13696, narrow) \
161 f(in_T, out_T, W_T, 13824, narrow) \
162 f(in_T, out_T, W_T, 14336, narrow) \
163 + f(in_T, out_T, W_T, 14784, narrow) \
164 + f(in_T, out_T, W_T, 14848, narrow) \
165 f(in_T, out_T, W_T, 15360, narrow) \
166 f(in_T, out_T, W_T, 16384, narrow) \
167 + f(in_T, out_T, W_T, 18944, narrow) \
168 f(in_T, out_T, W_T, 20480, narrow) \
169 f(in_T, out_T, W_T, 22016, narrow) \
170 f(in_T, out_T, W_T, 22528, narrow) \
@@ -140,6 +172,8 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
172 f(in_T, out_T, W_T, 27392, narrow) \
173 f(in_T, out_T, W_T, 27648, narrow) \
174 f(in_T, out_T, W_T, 28672, narrow) \
175 + f(in_T, out_T, W_T, 29568, narrow) \
176 + f(in_T, out_T, W_T, 29696, narrow) \
177 f(in_T, out_T, W_T, 32000, narrow) \
178 f(in_T, out_T, W_T, 32256, narrow) \
179 f(in_T, out_T, W_T, 32512, narrow) \
tests/lora/test_punica.py
+17
@@ -49,21 +49,30 @@ H1 = H2 = [
49 128,
50 256,
51 512,
52 + 896,
53 1024,
54 1152,
55 + 1216,
56 1280,
57 1536,
58 1664,
59 2048,
60 + 2240,
61 2304,
62 + 2368,
63 + 2432,
64 2560,
65 2752,
66 3072,
67 3328,
68 3456,
69 3584,
70 + 3712,
71 4096,
72 + 4480,
73 4608,
74 + 4736,
75 + 4864,
76 5120,
77 5504,
78 5632,
@@ -73,19 +82,27 @@ H1 = H2 = [
82 6848,
83 6912,
84 7168,
85 + 7424,
86 8192,
87 + 8960,
88 9216,
89 + 9472,
90 10240,
91 11008,
92 11264,
93 13824,
94 14336,
95 + 14784,
96 + 14848,
97 15360,
98 + 18944,
99 22016,
100 22528,
101 24576,
102 27392,
103 27648,
104 + 29568,
105 + 29696,
106 32000,
107 32256,
108 32512,