A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
[Experimental] Add multi-LoRA support (#1804)
Co-authored-by: Chen Shen <scv119@gmail.com> Co-authored-by: Shreyas Krishnaswamy <shrekris@anyscale.com> Co-authored-by: Avnish Narayan <avnish@anyscale.com>
Antoni Baum committed
Jan 24, 2024 at 00:26 UTC
9b945daaf1ce03b8b02d68b37c59baf28566b535
52 files changed
+8035
-126
.buildkite/test-pipeline.yaml
+3
@@ -41,6 +41,9 @@ steps:
41
- label: Worker Test
42
command: pytest -v -s worker
43
44
+- label: LoRA Test
45
+ command: pytest -v -s lora
46
+
47
- label: Benchmarks
48
working_dir: "/vllm-workspace/.buildkite"
49
commands:
benchmarks/benchmark_latency.py
+5
-5
@@ -65,7 +65,9 @@ def main(args: argparse.Namespace):
65
if args.profile:
66
profile_dir = args.profile_result_dir
67
if not profile_dir:
68
- profile_dir = Path(".") / "vllm_benchmark_result" / f"latency_result_{time.time()}"
68
+ profile_dir = Path(
69
+ "."
70
+ ) / "vllm_benchmark_result" / f"latency_result_{time.time()}"
71
print(f"Profiling (results will be saved to '{profile_dir}')...")
72
run_to_completion(profile_dir=args.profile_result_dir)
73
return
@@ -123,9 +125,7 @@ if __name__ == '__main__':
125
'--profile-result-dir',
126
type=str,
127
default=None,
126
- help=(
127
- 'path to save the pytorch profiler output. Can be visualized '
128
- 'with ui.perfetto.dev or Tensorboard.'
129
- ))
128
+ help=('path to save the pytorch profiler output. Can be visualized '
129
+ 'with ui.perfetto.dev or Tensorboard.'))
130
args = parser.parse_args()
131
main(args)
csrc/punica/LICENSE
new
+217
@@ -0,0 +1,217 @@
1
+Contains code from https://github.com/punica-ai/punica
2
+
3
+ Apache License
4
+ Version 2.0, January 2004
5
+ http://www.apache.org/licenses/
6
+
7
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
8
+
9
+ 1. Definitions.
10
+
11
+ "License" shall mean the terms and conditions for use, reproduction,
12
+ and distribution as defined by Sections 1 through 9 of this document.
13
+
14
+ "Licensor" shall mean the copyright owner or entity authorized by
15
+ the copyright owner that is granting the License.
16
+
17
+ "Legal Entity" shall mean the union of the acting entity and all
18
+ other entities that control, are controlled by, or are under common
19
+ control with that entity. For the purposes of this definition,
20
+ "control" means (i) the power, direct or indirect, to cause the
21
+ direction or management of such entity, whether by contract or
22
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
23
+ outstanding shares, or (iii) beneficial ownership of such entity.
24
+
25
+ "You" (or "Your") shall mean an individual or Legal Entity
26
+ exercising permissions granted by this License.
27
+
28
+ "Source" form shall mean the preferred form for making modifications,
29
+ including but not limited to software source code, documentation
30
+ source, and configuration files.
31
+
32
+ "Object" form shall mean any form resulting from mechanical
33
+ transformation or translation of a Source form, including but
34
+ not limited to compiled object code, generated documentation,
35
+ and conversions to other media types.
36
+
37
+ "Work" shall mean the work of authorship, whether in Source or
38
+ Object form, made available under the License, as indicated by a
39
+ copyright notice that is included in or attached to the work
40
+ (an example is provided in the Appendix below).
41
+
42
+ "Derivative Works" shall mean any work, whether in Source or Object
43
+ form, that is based on (or derived from) the Work and for which the
44
+ editorial revisions, annotations, elaborations, or other modifications
45
+ represent, as a whole, an original work of authorship. For the purposes
46
+ of this License, Derivative Works shall not include works that remain
47
+ separable from, or merely link (or bind by name) to the interfaces of,
48
+ the Work and Derivative Works thereof.
49
+
50
+ "Contribution" shall mean any work of authorship, including
51
+ the original version of the Work and any modifications or additions
52
+ to that Work or Derivative Works thereof, that is intentionally
53
+ submitted to Licensor for inclusion in the Work by the copyright owner
54
+ or by an individual or Legal Entity authorized to submit on behalf of
55
+ the copyright owner. For the purposes of this definition, "submitted"
56
+ means any form of electronic, verbal, or written communication sent
57
+ to the Licensor or its representatives, including but not limited to
58
+ communication on electronic mailing lists, source code control systems,
59
+ and issue tracking systems that are managed by, or on behalf of, the
60
+ Licensor for the purpose of discussing and improving the Work, but
61
+ excluding communication that is conspicuously marked or otherwise
62
+ designated in writing by the copyright owner as "Not a Contribution."
63
+
64
+ "Contributor" shall mean Licensor and any individual or Legal Entity
65
+ on behalf of whom a Contribution has been received by Licensor and
66
+ subsequently incorporated within the Work.
67
+
68
+ 2. Grant of Copyright License. Subject to the terms and conditions of
69
+ this License, each Contributor hereby grants to You a perpetual,
70
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
71
+ copyright license to reproduce, prepare Derivative Works of,
72
+ publicly display, publicly perform, sublicense, and distribute the
73
+ Work and such Derivative Works in Source or Object form.
74
+
75
+ 3. Grant of Patent License. Subject to the terms and conditions of
76
+ this License, each Contributor hereby grants to You a perpetual,
77
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
78
+ (except as stated in this section) patent license to make, have made,
79
+ use, offer to sell, sell, import, and otherwise transfer the Work,
80
+ where such license applies only to those patent claims licensable
81
+ by such Contributor that are necessarily infringed by their
82
+ Contribution(s) alone or by combination of their Contribution(s)
83
+ with the Work to which such Contribution(s) was submitted. If You
84
+ institute patent litigation against any entity (including a
85
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
86
+ or a Contribution incorporated within the Work constitutes direct
87
+ or contributory patent infringement, then any patent licenses
88
+ granted to You under this License for that Work shall terminate
89
+ as of the date such litigation is filed.
90
+
91
+ 4. Redistribution. You may reproduce and distribute copies of the
92
+ Work or Derivative Works thereof in any medium, with or without
93
+ modifications, and in Source or Object form, provided that You
94
+ meet the following conditions:
95
+
96
+ (a) You must give any other recipients of the Work or
97
+ Derivative Works a copy of this License; and
98
+
99
+ (b) You must cause any modified files to carry prominent notices
100
+ stating that You changed the files; and
101
+
102
+ (c) You must retain, in the Source form of any Derivative Works
103
+ that You distribute, all copyright, patent, trademark, and
104
+ attribution notices from the Source form of the Work,
105
+ excluding those notices that do not pertain to any part of
106
+ the Derivative Works; and
107
+
108
+ (d) If the Work includes a "NOTICE" text file as part of its
109
+ distribution, then any Derivative Works that You distribute must
110
+ include a readable copy of the attribution notices contained
111
+ within such NOTICE file, excluding those notices that do not
112
+ pertain to any part of the Derivative Works, in at least one
113
+ of the following places: within a NOTICE text file distributed
114
+ as part of the Derivative Works; within the Source form or
115
+ documentation, if provided along with the Derivative Works; or,
116
+ within a display generated by the Derivative Works, if and
117
+ wherever such third-party notices normally appear. The contents
118
+ of the NOTICE file are for informational purposes only and
119
+ do not modify the License. You may add Your own attribution
120
+ notices within Derivative Works that You distribute, alongside
121
+ or as an addendum to the NOTICE text from the Work, provided
122
+ that such additional attribution notices cannot be construed
123
+ as modifying the License.
124
+
125
+ You may add Your own copyright statement to Your modifications and
126
+ may provide additional or different license terms and conditions
127
+ for use, reproduction, or distribution of Your modifications, or
128
+ for any such Derivative Works as a whole, provided Your use,
129
+ reproduction, and distribution of the Work otherwise complies with
130
+ the conditions stated in this License.
131
+
132
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
133
+ any Contribution intentionally submitted for inclusion in the Work
134
+ by You to the Licensor shall be under the terms and conditions of
135
+ this License, without any additional terms or conditions.
136
+ Notwithstanding the above, nothing herein shall supersede or modify
137
+ the terms of any separate license agreement you may have executed
138
+ with Licensor regarding such Contributions.
139
+
140
+ 6. Trademarks. This License does not grant permission to use the trade
141
+ names, trademarks, service marks, or product names of the Licensor,
142
+ except as required for reasonable and customary use in describing the
143
+ origin of the Work and reproducing the content of the NOTICE file.
144
+
145
+ 7. Disclaimer of Warranty. Unless required by applicable law or
146
+ agreed to in writing, Licensor provides the Work (and each
147
+ Contributor provides its Contributions) on an "AS IS" BASIS,
148
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
149
+ implied, including, without limitation, any warranties or conditions
150
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
151
+ PARTICULAR PURPOSE. You are solely responsible for determining the
152
+ appropriateness of using or redistributing the Work and assume any
153
+ risks associated with Your exercise of permissions under this License.
154
+
155
+ 8. Limitation of Liability. In no event and under no legal theory,
156
+ whether in tort (including negligence), contract, or otherwise,
157
+ unless required by applicable law (such as deliberate and grossly
158
+ negligent acts) or agreed to in writing, shall any Contributor be
159
+ liable to You for damages, including any direct, indirect, special,
160
+ incidental, or consequential damages of any character arising as a
161
+ result of this License or out of the use or inability to use the
162
+ Work (including but not limited to damages for loss of goodwill,
163
+ work stoppage, computer failure or malfunction, or any and all
164
+ other commercial damages or losses), even if such Contributor
165
+ has been advised of the possibility of such damages.
166
+
167
+ 9. Accepting Warranty or Additional Liability. While redistributing
168
+ the Work or Derivative Works thereof, You may choose to offer,
169
+ and charge a fee for, acceptance of support, warranty, indemnity,
170
+ or other liability obligations and/or rights consistent with this
171
+ License. However, in accepting such obligations, You may act only
172
+ on Your own behalf and on Your sole responsibility, not on behalf
173
+ of any other Contributor, and only if You agree to indemnify,
174
+ defend, and hold each Contributor harmless for any liability
175
+ incurred by, or claims asserted against, such Contributor by reason
176
+ of your accepting any such warranty or additional liability.
177
+
178
+ END OF TERMS AND CONDITIONS
179
+
180
+ APPENDIX: How to apply the Apache License to your work.
181
+
182
+ To apply the Apache License to your work, attach the following
183
+ boilerplate notice, with the fields enclosed by brackets "{}"
184
+ replaced with your own identifying information. (Don't include
185
+ the brackets!) The text should be enclosed in the appropriate
186
+ comment syntax for the file format. We also recommend that a
187
+ file or class name and description of purpose be included on the
188
+ same "printed page" as the copyright notice for easier
189
+ identification within third-party archives.
190
+
191
+ Copyright {yyyy} {name of copyright owner}
192
+
193
+ Licensed under the Apache License, Version 2.0 (the "License");
194
+ you may not use this file except in compliance with the License.
195
+ You may obtain a copy of the License at
196
+
197
+ http://www.apache.org/licenses/LICENSE-2.0
198
+
199
+ Unless required by applicable law or agreed to in writing, software
200
+ distributed under the License is distributed on an "AS IS" BASIS,
201
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
202
+ See the License for the specific language governing permissions and
203
+ limitations under the License.
204
+
205
+------------------------------------------------------------------------------------
206
+
207
+This product bundles various third-party components under other open source licenses.
208
+This section summarizes those components and their licenses. See licenses/
209
+for text of these licenses.
210
+
211
+
212
+Apache-2.0
213
+* third_party/nvbench (with LLVM exception)
214
+* third_party/flashinfer
215
+
216
+BSD-3-Clause:
217
+* third_party/cutlass
\ No newline at end of file
csrc/punica/bgmv/bgmv_all.cu
new
+21
@@ -0,0 +1,21 @@
1
+#include "bgmv_config.h"
2
+#include "bgmv_impl.cuh"
3
+
4
+FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_half, nv_half, nv_half)
5
+FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_half, nv_half, nv_bfloat16)
6
+FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_bfloat16, nv_bfloat16, nv_bfloat16)
7
+FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_bfloat16, nv_bfloat16, nv_half)
8
+FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_half, nv_bfloat16, nv_half)
9
+FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_bfloat16, nv_half, nv_half)
10
+FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_half, nv_bfloat16, nv_bfloat16)
11
+FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_bfloat16, nv_half, nv_bfloat16)
12
+FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, float, nv_half, nv_half)
13
+FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, float, nv_half, nv_bfloat16)
14
+FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, float, nv_bfloat16, nv_bfloat16)
15
+FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, float, nv_bfloat16, nv_half)
16
+FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_half, float, nv_half)
17
+FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_half, float, nv_bfloat16)
18
+FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_bfloat16, float, nv_bfloat16)
19
+FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_bfloat16, float, nv_half)
20
+FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, float, float, nv_half)
21
+FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, float, float, nv_bfloat16)
csrc/punica/bgmv/bgmv_config.h
new
+59
@@ -0,0 +1,59 @@
1
+#pragma once
2
+
3
+template <int feat_in, int feat_out, typename in_T, typename out_T,
4
+ typename W_T>
5
+void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
6
+ const W_T *__restrict__ W,
7
+ const int64_t *__restrict__ indicies, int64_t y_offset,
8
+ int64_t full_y_size, int64_t batch_size, int64_t num_layers,
9
+ int64_t layer_idx, float scale);
10
+
11
+// clang-format off
12
+
13
+#define FOR_BGMV_WIDE(f, in_T, out_T, W_T, narrow) \
14
+ f(in_T, out_T, W_T, narrow, 128) \
15
+ f(in_T, out_T, W_T, narrow, 256) \
16
+ f(in_T, out_T, W_T, narrow, 512) \
17
+ f(in_T, out_T, W_T, narrow, 1024) \
18
+ f(in_T, out_T, W_T, narrow, 1280) \
19
+ f(in_T, out_T, W_T, narrow, 1728) \
20
+ f(in_T, out_T, W_T, narrow, 1792) \
21
+ f(in_T, out_T, W_T, narrow, 2048) \
22
+ f(in_T, out_T, W_T, narrow, 2560) \
23
+ f(in_T, out_T, W_T, narrow, 2752) \
24
+ f(in_T, out_T, W_T, narrow, 3072) \
25
+ f(in_T, out_T, W_T, narrow, 3456) \
26
+ f(in_T, out_T, W_T, narrow, 3584) \
27
+ f(in_T, out_T, W_T, narrow, 4096) \
28
+ f(in_T, out_T, W_T, narrow, 5120) \
29
+ f(in_T, out_T, W_T, narrow, 5504) \
30
+ f(in_T, out_T, W_T, narrow, 5632) \
31
+ f(in_T, out_T, W_T, narrow, 6912) \
32
+ f(in_T, out_T, W_T, narrow, 7168) \
33
+ f(in_T, out_T, W_T, narrow, 8192) \
34
+ f(in_T, out_T, W_T, narrow, 9216) \
35
+ f(in_T, out_T, W_T, narrow, 10240) \
36
+ f(in_T, out_T, W_T, narrow, 11008) \
37
+ f(in_T, out_T, W_T, narrow, 12288) \
38
+ f(in_T, out_T, W_T, narrow, 13824) \
39
+ f(in_T, out_T, W_T, narrow, 14336) \
40
+ f(in_T, out_T, W_T, narrow, 16384) \
41
+ f(in_T, out_T, W_T, narrow, 20480) \
42
+ f(in_T, out_T, W_T, narrow, 28672) \
43
+ f(in_T, out_T, W_T, narrow, 32000) \
44
+ f(in_T, out_T, W_T, narrow, 32256) \
45
+ f(in_T, out_T, W_T, narrow, 32512) \
46
+ f(in_T, out_T, W_T, narrow, 32768) \
47
+ f(in_T, out_T, W_T, narrow, 33024) \
48
+ f(in_T, out_T, W_T, narrow, 36864) \
49
+ f(in_T, out_T, W_T, narrow, 49152) \
50
+// Keep above in sync with vllm/lora/layers::SamplerWithLoRA
51
+
52
+// Keep this in sync with vllm/config::LoRAConfig
53
+#define FOR_BGMV_WIDE_NARROW(f, in_T, out_T, W_T) \
54
+ FOR_BGMV_WIDE(f, in_T, out_T, W_T, 8) \
55
+ FOR_BGMV_WIDE(f, in_T, out_T, W_T, 16) \
56
+ FOR_BGMV_WIDE(f, in_T, out_T, W_T, 32) \
57
+ FOR_BGMV_WIDE(f, in_T, out_T, W_T, 64)
58
+
59
+// clang-format on
csrc/punica/bgmv/bgmv_impl.cuh
new
+294
@@ -0,0 +1,294 @@
1
+#pragma once
2
+
3
+#include <ATen/cuda/CUDAContext.h>
4
+#include <cooperative_groups.h>
5
+#include <cuda/pipeline>
6
+#include <cuda_runtime.h>
7
+#include <iostream>
8
+#include <stdio.h>
9
+
10
+#include "vec_dtypes.cuh"
11
+
12
+namespace cg = cooperative_groups;
13
+
14
+// nthrs = (32, 4)
15
+template <int feat_in, int feat_out, size_t vec_size, size_t X_copy_size,
16
+ size_t W_copy_size, int tx, int ty, int tz, typename in_T,
17
+ typename out_T, typename W_T>
18
+__global__ void
19
+bgmv_shrink_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
20
+ const W_T *__restrict__ W,
21
+ const int64_t *__restrict__ indicies, int64_t y_offset,
22
+ int64_t full_y_size, int64_t num_layers, int64_t layer_idx,
23
+ float scale) {
24
+ size_t batch_idx = blockIdx.y;
25
+ int64_t idx = indicies[batch_idx] * num_layers + layer_idx;
26
+ if (idx < 0) {
27
+ return;
28
+ }
29
+
30
+ auto block = cg::this_thread_block();
31
+ size_t j = blockIdx.x;
32
+ constexpr size_t num_pipeline_stages = 2;
33
+ constexpr size_t tile_size = tx * ty * vec_size;
34
+ __shared__ W_T W_shared[num_pipeline_stages * tile_size];
35
+ __shared__ in_T X_shared[num_pipeline_stages * tile_size];
36
+ __shared__ float y_warpwise[ty];
37
+
38
+ size_t W_shared_offset[num_pipeline_stages] = {0U, 1U * tile_size};
39
+ size_t X_shared_offset[num_pipeline_stages] = {0U, 1U * tile_size};
40
+ auto pipe = cuda::make_pipeline();
41
+
42
+ // pipeline load W/X and compute WX;
43
+ pipe.producer_acquire();
44
+ cuda::memcpy_async(W_shared + (threadIdx.y * tx + threadIdx.x) * vec_size,
45
+ W + (idx * feat_out + j) * feat_in +
46
+ (threadIdx.y * tx + threadIdx.x) * vec_size,
47
+ cuda::aligned_size_t<W_copy_size>(W_copy_size), pipe);
48
+ cuda::memcpy_async(X_shared + (threadIdx.y * tx + threadIdx.x) * vec_size,
49
+ X + (batch_idx * feat_in) +
50
+ (threadIdx.y * tx + threadIdx.x) * vec_size,
51
+ cuda::aligned_size_t<X_copy_size>(X_copy_size), pipe);
52
+ pipe.producer_commit();
53
+ size_t copy_idx, compute_idx;
54
+ float y = 0.f;
55
+ vec_t<in_T, vec_size> x_vec;
56
+ vec_t<W_T, vec_size> w_vec;
57
+ size_t tile_idx;
58
+
59
+#pragma unroll
60
+ for (tile_idx = 1; tile_idx < (feat_in + tile_size - 1) / tile_size;
61
+ ++tile_idx) {
62
+ copy_idx = tile_idx % num_pipeline_stages;
63
+ // pipeline stage: async copy W fragment
64
+ pipe.producer_acquire();
65
+ if (tile_idx * tile_size + threadIdx.y * tx * vec_size < feat_in) {
66
+ cuda::memcpy_async(W_shared + W_shared_offset[copy_idx] +
67
+ (threadIdx.y * tx + threadIdx.x) * vec_size,
68
+ W + (idx * feat_out + j) * feat_in +
69
+ tile_idx * tile_size +
70
+ (threadIdx.y * tx + threadIdx.x) * vec_size,
71
+ cuda::aligned_size_t<W_copy_size>(W_copy_size), pipe);
72
+ cuda::memcpy_async(X_shared + X_shared_offset[copy_idx] +
73
+ (threadIdx.y * tx + threadIdx.x) * vec_size,
74
+ X + (batch_idx * feat_in) + tile_idx * tile_size +
75
+ (threadIdx.y * tx + threadIdx.x) * vec_size,
76
+ cuda::aligned_size_t<X_copy_size>(X_copy_size), pipe);
77
+ }
78
+ pipe.producer_commit();
79
+
80
+ compute_idx = (tile_idx - 1) % num_pipeline_stages;
81
+ // pipeline stage: compute WX
82
+ pipe.consumer_wait();
83
+ block.sync();
84
+ x_vec.load(X_shared + X_shared_offset[compute_idx] +
85
+ (threadIdx.y * tx + threadIdx.x) * vec_size);
86
+ w_vec.load(W_shared + W_shared_offset[compute_idx] +
87
+ (threadIdx.y * tx + threadIdx.x) * vec_size);
88
+ float sum = 0.f;
89
+#pragma unroll
90
+ for (size_t i = 0; i < vec_size; ++i) {
91
+ sum += float(w_vec[i]) * float(x_vec[i]) * scale;
92
+ }
93
+#pragma unroll
94
+ for (size_t offset = tx / 2; offset > 0; offset /= 2) {
95
+ sum += __shfl_down_sync(0xffffffff, sum, offset);
96
+ }
97
+ y_warpwise[threadIdx.y] = sum;
98
+ block.sync();
99
+#pragma unroll
100
+ for (size_t i = 0; i < ty; ++i) {
101
+ y += y_warpwise[i];
102
+ }
103
+
104
+ block.sync();
105
+ pipe.consumer_release();
106
+ }
107
+
108
+ compute_idx = (tile_idx - 1) % num_pipeline_stages;
109
+ // final pipeline stage
110
+ pipe.consumer_wait();
111
+ block.sync();
112
+ x_vec.load(X_shared + X_shared_offset[compute_idx] +
113
+ (threadIdx.y * tx + threadIdx.x) * vec_size);
114
+ w_vec.load(W_shared + W_shared_offset[compute_idx] +
115
+ (threadIdx.y * tx + threadIdx.x) * vec_size);
116
+ float sum = 0.f;
117
+#pragma unroll
118
+ for (size_t i = 0; i < vec_size; ++i) {
119
+ sum += float(w_vec[i]) * float(x_vec[i]) * scale;
120
+ }
121
+#pragma unroll
122
+ for (size_t offset = tx / 2; offset > 0; offset /= 2) {
123
+ sum += __shfl_down_sync(0xffffffff, sum, offset);
124
+ }
125
+ y_warpwise[threadIdx.y] =
126
+ ((tile_idx - 1) * tile_size + threadIdx.y * tx * vec_size < feat_in)
127
+ ? sum
128
+ : 0.f;
129
+ block.sync();
130
+#pragma unroll
131
+ for (size_t i = 0; i < ty; ++i) {
132
+ y += y_warpwise[i];
133
+ }
134
+
135
+ block.sync();
136
+ pipe.consumer_release();
137
+
138
+ // write Y;
139
+ if (block.thread_rank() == 0) {
140
+ Y[batch_idx * full_y_size + y_offset + j] += static_cast<out_T>(y);
141
+ }
142
+}
143
+
144
+// nthrs = (2, 16, 4)
145
+template <int feat_in, int feat_out, size_t vec_size, int tx, int ty, int tz,
146
+ typename in_T, typename out_T, typename W_T>
147
+__global__ void
148
+bgmv_expand_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
149
+ const W_T *__restrict__ W,
150
+ const int64_t *__restrict__ indicies, int64_t y_offset,
151
+ int64_t full_y_size, int64_t num_layers, int64_t layer_idx,
152
+ float scale) {
153
+ size_t batch_idx = blockIdx.y;
154
+ int64_t idx = indicies[batch_idx] * num_layers + layer_idx;
155
+
156
+ if (idx < 0) {
157
+ return;
158
+ }
159
+
160
+ auto block = cg::this_thread_block();
161
+ size_t tile_idx = blockIdx.x;
162
+
163
+ // load X;
164
+ vec_t<in_T, vec_size> x_vec;
165
+ x_vec.load(X + batch_idx * feat_in + threadIdx.x * vec_size);
166
+
167
+ // load W;
168
+ vec_t<W_T, vec_size> w_vec;
169
+ w_vec.load(W + (idx * feat_out + tile_idx * tz * ty) * feat_in +
170
+ block.thread_rank() * vec_size);
171
+
172
+ float sum = 0.f;
173
+#pragma unroll
174
+ for (size_t i = 0; i < vec_size; ++i) {
175
+ sum += float(w_vec[i]) * float(x_vec[i]) * scale;
176
+ }
177
+
178
+ cg::thread_block_tile g = cg::tiled_partition<tx>(block);
179
+#pragma unroll
180
+ for (size_t offset = tx / 2; offset > 0; offset /= 2) {
181
+ sum += g.shfl_down(sum, offset);
182
+ }
183
+ sum = g.shfl(sum, 0);
184
+
185
+ if (threadIdx.x == 0) {
186
+ Y[batch_idx * full_y_size + y_offset + tile_idx * (tz * ty) +
187
+ threadIdx.z * ty + threadIdx.y] += static_cast<out_T>(sum);
188
+ }
189
+}
190
+
191
+template <int feat_in, int feat_out, typename in_T, typename out_T,
192
+ typename W_T>
193
+void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
194
+ const W_T *__restrict__ W,
195
+ const int64_t *__restrict__ indicies, int64_t y_offset,
196
+ int64_t full_y_size, int64_t batch_size, int64_t num_layers,
197
+ int64_t layer_idx, float scale) {
198
+ constexpr size_t vec_size = 8;
199
+ constexpr int tz = 4;
200
+ const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
201
+
202
+ if constexpr (feat_in < feat_out) {
203
+ static_assert(feat_in % vec_size == 0);
204
+ constexpr int tx = feat_in / vec_size;
205
+
206
+ static_assert((32 % tx == 0 && feat_out % (32 / tx * tz) == 0) ||
207
+ (16 % tx == 0 && feat_out % (16 / tx * tz) == 0) ||
208
+ (8 % tx == 0 && feat_out % (8 / tx * tz) == 0));
209
+
210
+ if constexpr (32 % tx == 0 && feat_out % (32 / tx * tz) == 0) {
211
+ constexpr int ty = 32 / tx;
212
+ dim3 nblks(feat_out / (ty * tz), batch_size);
213
+ dim3 nthrs(tx, ty, tz);
214
+
215
+ bgmv_expand_kernel<feat_in, feat_out, vec_size, tx, ty, tz>
216
+ <<<nblks, nthrs, 0, stream>>>(Y, X, W, indicies, y_offset,
217
+ full_y_size, num_layers, layer_idx,
218
+ scale);
219
+ } else if (16 % tx == 0 && feat_out % (16 / tx * tz) == 0) {
220
+ constexpr int ty = 16 / tx;
221
+ dim3 nblks(feat_out / (ty * tz), batch_size);
222
+ dim3 nthrs(tx, ty, tz);
223
+
224
+ bgmv_expand_kernel<feat_in, feat_out, vec_size, tx, ty, tz>
225
+ <<<nblks, nthrs, 0, stream>>>(Y, X, W, indicies, y_offset,
226
+ full_y_size, num_layers, layer_idx,
227
+ scale);
228
+ } else {
229
+ constexpr int ty = 8 / tx;
230
+ dim3 nblks(feat_out / (ty * tz), batch_size);
231
+ dim3 nthrs(tx, ty, tz);
232
+
233
+ bgmv_expand_kernel<feat_in, feat_out, vec_size, tx, ty, tz>
234
+ <<<nblks, nthrs, 0, stream>>>(Y, X, W, indicies, y_offset,
235
+ full_y_size, num_layers, layer_idx,
236
+ scale);
237
+ }
238
+ } else {
239
+ static_assert(feat_in % (vec_size * 32) == 0 ||
240
+ feat_in % (vec_size * 16) == 0 ||
241
+ feat_in % (vec_size * 8) == 0);
242
+
243
+ if constexpr (feat_in % (vec_size * 32) == 0) {
244
+ constexpr int tx = 32;
245
+ constexpr int ty = 4;
246
+
247
+ dim3 nblks(feat_out, batch_size);
248
+ dim3 nthrs(tx, ty);
249
+
250
+ bgmv_shrink_kernel<feat_in, feat_out, vec_size, vec_size * sizeof(in_T),
251
+ vec_size * sizeof(W_T), tx, ty, tz>
252
+ <<<nblks, nthrs, 0, stream>>>(Y, X, W, indicies, y_offset,
253
+ full_y_size, num_layers, layer_idx,
254
+ scale);
255
+ } else if constexpr (feat_in % (vec_size / 2 * 32) == 0) {
256
+ constexpr int tx = 32;
257
+ constexpr int ty = 4;
258
+
259
+ dim3 nblks(feat_out, batch_size);
260
+ dim3 nthrs(tx, ty);
261
+
262
+ bgmv_shrink_kernel<feat_in, feat_out, vec_size / 2,
263
+ vec_size * sizeof(in_T) / 2,
264
+ vec_size * sizeof(W_T) / 2, tx, ty, tz>
265
+ <<<nblks, nthrs, 0, stream>>>(Y, X, W, indicies, y_offset,
266
+ full_y_size, num_layers, layer_idx,
267
+ scale);
268
+ } else if constexpr (feat_in % (vec_size / 2 * 16) == 0) {
269
+ constexpr int tx = 16;
270
+ constexpr int ty = 4;
271
+
272
+ dim3 nblks(feat_out, batch_size);
273
+ dim3 nthrs(tx, ty);
274
+
275
+ bgmv_shrink_kernel<feat_in, feat_out, vec_size / 2,
276
+ vec_size * sizeof(in_T) / 2,
277
+ vec_size * sizeof(W_T) / 2, tx, ty, tz>
278
+ <<<nblks, nthrs, 0, stream>>>(Y, X, W, indicies, y_offset,
279
+ full_y_size, num_layers, layer_idx,
280
+ scale);
281
+ }
282
+ }
283
+}
284
+
285
+#define INST_BGMV(feat_in, feat_out, in_T, out_T, W_T) \
286
+ template void bgmv_kernel<feat_in, feat_out>( \
287
+ out_T * __restrict__ Y, const in_T *__restrict__ X, \
288
+ const W_T *__restrict__ W, const int64_t *__restrict__ indicies, \
289
+ int64_t y_offset, int64_t full_y_size, int64_t batch_size, \
290
+ int64_t num_layers, int64_t layer_idx, float scale);
291
+
292
+#define INST_BGMV_TWOSIDE(in_T, out_T, W_T, narrow, wide) \
293
+ INST_BGMV(narrow, wide, in_T, out_T, W_T) \
294
+ INST_BGMV(wide, narrow, in_T, out_T, W_T)
csrc/punica/bgmv/vec_dtypes.cuh
new
+1324
@@ -0,0 +1,1324 @@
1
+#ifndef VEC_DTYPES_CUH_
2
+#define VEC_DTYPES_CUH_
3
+
4
+#include <cuda_bf16.h>
5
+#include <cuda_fp16.h>
6
+#ifdef FLASHINFER_USE_FP8
7
+#include <cuda_fp8.h>
8
+#endif
9
+#include <cuda_runtime.h>
10
+
11
+#include <type_traits>
12
+
13
+#define FLASHINFER_INLINE \
14
+ inline __attribute__((always_inline)) __device__ __host__
15
+
16
+template <typename float_t, size_t vec_size>
17
+struct vec_t {
18
+ FLASHINFER_INLINE float_t &operator[](size_t i);
19
+ FLASHINFER_INLINE const float_t &operator[](size_t i) const;
20
+ FLASHINFER_INLINE void fill(float_t val);
21
+ FLASHINFER_INLINE void load(const float_t *ptr);
22
+ FLASHINFER_INLINE void store(float_t *ptr) const;
23
+ template <typename T>
24
+ FLASHINFER_INLINE void cast_from(const vec_t<T, vec_size> &src);
25
+ template <typename T>
26
+ FLASHINFER_INLINE void cast_load(const T *ptr);
27
+ template <typename T>
28
+ FLASHINFER_INLINE void cast_store(T *ptr) const;
29
+ FLASHINFER_INLINE static void memcpy(float_t *dst, const float_t *src);
30
+};
31
+
32
+template <typename src_float_t, typename tgt_float_t, size_t vec_size>
33
+FLASHINFER_INLINE void cast_from_impl(const vec_t<src_float_t, vec_size> &src,
34
+ vec_t<tgt_float_t, vec_size> &dst) {
35
+#pragma unroll
36
+ for (size_t i = 0; i < vec_size; ++i) {
37
+ dst[i] = tgt_float_t(src[i]);
38
+ }
39
+}
40
+
41
+template <typename src_float_t, typename tgt_float_t, size_t vec_size>
42
+FLASHINFER_INLINE void cast_load_impl(const src_float_t *src_ptr,
43
+ vec_t<tgt_float_t, vec_size> &dst) {
44
+ if constexpr (std::is_same<src_float_t, tgt_float_t>::value) {
45
+ dst.load(src_ptr);
46
+ } else {
47
+ vec_t<src_float_t, vec_size> tmp;
48
+ tmp.load(src_ptr);
49
+ dst.cast_from(tmp);
50
+ }
51
+}
52
+
53
+template <typename src_float_t, typename tgt_float_t, size_t vec_size>
54
+FLASHINFER_INLINE void cast_store_impl(const vec_t<src_float_t, vec_size> &src,
55
+ tgt_float_t *dst_ptr) {
56
+ if constexpr (std::is_same<src_float_t, tgt_float_t>::value) {
57
+ src.store(dst_ptr);
58
+ } else {
59
+ vec_t<tgt_float_t, vec_size> tmp;
60
+ tmp.cast_from(src);
61
+ tmp.store(dst_ptr);
62
+ }
63
+}
64
+
65
+#ifdef FLASHINFER_USE_FP8
66
+/******************* vec_t<__nv_fp8_e4m3> *******************/
67
+
68
+// __nv_fp8_e4m3 x 1
69
+template <>
70
+struct vec_t<__nv_fp8_e4m3, 1> {
71
+ __nv_fp8_e4m3 data;
72
+
73
+ FLASHINFER_INLINE __nv_fp8_e4m3 &operator[](size_t i) {
74
+ return ((__nv_fp8_e4m3 *)(&data))[i];
75
+ }
76
+ FLASHINFER_INLINE const __nv_fp8_e4m3 &operator[](size_t i) const {
77
+ return ((const __nv_fp8_e4m3 *)(&data))[i];
78
+ }
79
+ FLASHINFER_INLINE void fill(__nv_fp8_e4m3 val);
80
+ FLASHINFER_INLINE void load(const __nv_fp8_e4m3 *ptr);
81
+ FLASHINFER_INLINE void store(__nv_fp8_e4m3 *ptr) const;
82
+ template <typename T>
83
+ FLASHINFER_INLINE void cast_from(const vec_t<T, 1> &src) {
84
+ cast_from_impl(src, *this);
85
+ }
86
+ template <typename T>
87
+ FLASHINFER_INLINE void cast_load(const T *ptr) {
88
+ cast_load_impl(ptr, *this);
89
+ }
90
+ template <typename T>
91
+ FLASHINFER_INLINE void cast_store(T *ptr) const {
92
+ cast_store_impl(*this, ptr);
93
+ }
94
+
95
+ FLASHINFER_INLINE static void memcpy(__nv_fp8_e4m3 *dst,
96
+ const __nv_fp8_e4m3 *src);
97
+};
98
+
99
+FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 1>::fill(__nv_fp8_e4m3 val) {
100
+ data = val;
101
+}
102
+
103
+FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 1>::load(const __nv_fp8_e4m3 *ptr) {
104
+ data = *ptr;
105
+}
106
+
107
+FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 1>::store(
108
+ __nv_fp8_e4m3 *ptr) const {
109
+ *ptr = data;
110
+}
111
+
112
+FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 1>::memcpy(
113
+ __nv_fp8_e4m3 *dst, const __nv_fp8_e4m3 *src) {
114
+ *dst = *src;
115
+}
116
+
117
+// __nv_fp8_e4m3 x 2
118
+template <>
119
+struct vec_t<__nv_fp8_e4m3, 2> {
120
+ __nv_fp8x2_e4m3 data;
121
+
122
+ FLASHINFER_INLINE __nv_fp8_e4m3 &operator[](size_t i) {
123
+ return ((__nv_fp8_e4m3 *)(&data))[i];
124
+ }
125
+ FLASHINFER_INLINE const __nv_fp8_e4m3 &operator[](size_t i) const {
126
+ return ((const __nv_fp8_e4m3 *)(&data))[i];
127
+ }
128
+ FLASHINFER_INLINE void fill(__nv_fp8_e4m3 val);
129
+ FLASHINFER_INLINE void load(const __nv_fp8_e4m3 *ptr);
130
+ FLASHINFER_INLINE void store(__nv_fp8_e4m3 *ptr) const;
131
+ template <typename T>
132
+ FLASHINFER_INLINE void cast_from(const vec_t<T, 2> &src) {
133
+ cast_from_impl(src, *this);
134
+ }
135
+ template <typename T>
136
+ FLASHINFER_INLINE void cast_load(const T *ptr) {
137
+ cast_load_impl(ptr, *this);
138
+ }
139
+ template <typename T>
140
+ FLASHINFER_INLINE void cast_store(T *ptr) const {
141
+ cast_store_impl(*this, ptr);
142
+ }
143
+
144
+ FLASHINFER_INLINE static void memcpy(__nv_fp8_e4m3 *dst,
145
+ const __nv_fp8_e4m3 *src);
146
+};
147
+
148
+FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 2>::fill(__nv_fp8_e4m3 val) {
149
+ data.__x =
150
+ (__nv_fp8x2_storage_t(val.__x) << 8) | __nv_fp8x2_storage_t(val.__x);
151
+}
152
+
153
+FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 2>::load(const __nv_fp8_e4m3 *ptr) {
154
+ data = *((__nv_fp8x2_e4m3 *)ptr);
155
+}
156
+
157
+FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 2>::store(
158
+ __nv_fp8_e4m3 *ptr) const {
159
+ *((__nv_fp8x2_e4m3 *)ptr) = data;
160
+}
161
+
162
+FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 2>::memcpy(
163
+ __nv_fp8_e4m3 *dst, const __nv_fp8_e4m3 *src) {
164
+ *((__nv_fp8x2_e4m3 *)dst) = *((__nv_fp8x2_e4m3 *)src);
165
+}
166
+
167
+// __nv_fp8_e4m3 x 4
168
+
169
+template <>
170
+struct vec_t<__nv_fp8_e4m3, 4> {
171
+ __nv_fp8x4_e4m3 data;
172
+
173
+ FLASHINFER_INLINE __nv_fp8_e4m3 &operator[](size_t i) {
174
+ return ((__nv_fp8_e4m3 *)(&data))[i];
175
+ }
176
+ FLASHINFER_INLINE const __nv_fp8_e4m3 &operator[](size_t i) const {
177
+ return ((const __nv_fp8_e4m3 *)(&data))[i];
178
+ }
179
+ FLASHINFER_INLINE void fill(__nv_fp8_e4m3 val);
180
+ FLASHINFER_INLINE void load(const __nv_fp8_e4m3 *ptr);
181
+ FLASHINFER_INLINE void store(__nv_fp8_e4m3 *ptr) const;
182
+ template <typename T>
183
+ FLASHINFER_INLINE void cast_from(const vec_t<T, 4> &src) {
184
+ cast_from_impl(src, *this);
185
+ }
186
+ template <typename T>
187
+ FLASHINFER_INLINE void cast_load(const T *ptr) {
188
+ cast_load_impl(ptr, *this);
189
+ }
190
+ template <typename T>
191
+ FLASHINFER_INLINE void cast_store(T *ptr) const {
192
+ cast_store_impl(*this, ptr);
193
+ }
194
+
195
+ FLASHINFER_INLINE static void memcpy(__nv_fp8_e4m3 *dst,
196
+ const __nv_fp8_e4m3 *src);
197
+};
198
+
199
+FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 4>::fill(__nv_fp8_e4m3 val) {
200
+ data.__x = (__nv_fp8x4_storage_t(val.__x) << 24) |
201
+ (__nv_fp8x4_storage_t(val.__x) << 16) |
202
+ (__nv_fp8x4_storage_t(val.__x) << 8) |
203
+ __nv_fp8x4_storage_t(val.__x);
204
+}
205
+
206
+FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 4>::load(const __nv_fp8_e4m3 *ptr) {
207
+ data = *((__nv_fp8x4_e4m3 *)ptr);
208
+}
209
+
210
+FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 4>::store(
211
+ __nv_fp8_e4m3 *ptr) const {
212
+ *((__nv_fp8x4_e4m3 *)ptr) = data;
213
+}
214
+
215
+FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 4>::memcpy(
216
+ __nv_fp8_e4m3 *dst, const __nv_fp8_e4m3 *src) {
217
+ *((__nv_fp8x4_e4m3 *)dst) = *((__nv_fp8x4_e4m3 *)src);
218
+}
219
+
220
+// __nv_fp8_e4m3 x 8
221
+
222
+template <>
223
+struct vec_t<__nv_fp8_e4m3, 8> {
224
+ uint2 data;
225
+
226
+ FLASHINFER_INLINE __nv_fp8_e4m3 &operator[](size_t i) {
227
+ return ((__nv_fp8_e4m3 *)(&data))[i];
228
+ }
229
+ FLASHINFER_INLINE const __nv_fp8_e4m3 &operator[](size_t i) const {
230
+ return ((const __nv_fp8_e4m3 *)(&data))[i];
231
+ }
232
+ FLASHINFER_INLINE void fill(__nv_fp8_e4m3 val);
233
+ FLASHINFER_INLINE void load(const __nv_fp8_e4m3 *ptr);
234
+ FLASHINFER_INLINE void store(__nv_fp8_e4m3 *ptr) const;
235
+ template <typename T>
236
+ FLASHINFER_INLINE void cast_from(const vec_t<T, 8> &src) {
237
+ cast_from_impl(src, *this);
238
+ }
239
+ template <typename T>
240
+ FLASHINFER_INLINE void cast_load(const T *ptr) {
241
+ cast_load_impl(ptr, *this);
242
+ }
243
+ template <typename T>
244
+ FLASHINFER_INLINE void cast_store(T *ptr) const {
245
+ cast_store_impl(*this, ptr);
246
+ }
247
+
248
+ FLASHINFER_INLINE static void memcpy(__nv_fp8_e4m3 *dst,
249
+ const __nv_fp8_e4m3 *src);
250
+};
251
+
252
+FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 8>::fill(__nv_fp8_e4m3 val) {
253
+ ((__nv_fp8x4_e4m3 *)(&data.x))->__x = (__nv_fp8x4_storage_t(val.__x) << 24) |
254
+ (__nv_fp8x4_storage_t(val.__x) << 16) |
255
+ (__nv_fp8x4_storage_t(val.__x) << 8) |
256
+ __nv_fp8x4_storage_t(val.__x);
257
+ ((__nv_fp8x4_e4m3 *)(&data.y))->__x = (__nv_fp8x4_storage_t(val.__x) << 24) |
258
+ (__nv_fp8x4_storage_t(val.__x) << 16) |
259
+ (__nv_fp8x4_storage_t(val.__x) << 8) |
260
+ __nv_fp8x4_storage_t(val.__x);
261
+}
262
+
263
+FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 8>::load(const __nv_fp8_e4m3 *ptr) {
264
+ data = *((uint2 *)ptr);
265
+}
266
+
267
+FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 8>::store(
268
+ __nv_fp8_e4m3 *ptr) const {
269
+ *((uint2 *)ptr) = data;
270
+}
271
+
272
+FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 8>::memcpy(
273
+ __nv_fp8_e4m3 *dst, const __nv_fp8_e4m3 *src) {
274
+ *((__nv_fp8_e4m3 *)dst) = *((__nv_fp8_e4m3 *)src);
275
+}
276
+
277
+// __nv_fp8_e4m3 x 16 or more
278
+template <size_t vec_size>
279
+struct vec_t<__nv_fp8_e4m3, vec_size> {
280
+ uint4 data[vec_size / 16];
281
+
282
+ FLASHINFER_INLINE __nv_fp8_e4m3 &operator[](size_t i) {
283
+ return ((__nv_fp8_e4m3 *)data)[i];
284
+ }
285
+ FLASHINFER_INLINE const __nv_fp8_e4m3 &operator[](size_t i) const {
286
+ return ((const __nv_fp8_e4m3 *)data)[i];
287
+ }
288
+ FLASHINFER_INLINE void fill(__nv_fp8_e4m3 val) {
289
+#pragma unroll
290
+ for (size_t i = 0; i < vec_size / 16; ++i) {
291
+ ((__nv_fp8x4_e4m3 *)(&(data[i].x)))->__x =
292
+ (__nv_fp8x4_storage_t(val.__x) << 24) |
293
+ (__nv_fp8x4_storage_t(val.__x) << 16) |
294
+ (__nv_fp8x4_storage_t(val.__x) << 8) | __nv_fp8x4_storage_t(val.__x);
295
+ ((__nv_fp8x4_e4m3 *)(&(data[i].y)))->__x =
296
+ (__nv_fp8x4_storage_t(val.__x) << 24) |
297
+ (__nv_fp8x4_storage_t(val.__x) << 16) |
298
+ (__nv_fp8x4_storage_t(val.__x) << 8) | __nv_fp8x4_storage_t(val.__x);
299
+ ((__nv_fp8x4_e4m3 *)(&(data[i].z)))->__x =
300
+ (__nv_fp8x4_storage_t(val.__x) << 24) |
301
+ (__nv_fp8x4_storage_t(val.__x) << 16) |
302
+ (__nv_fp8x4_storage_t(val.__x) << 8) | __nv_fp8x4_storage_t(val.__x);
303
+ ((__nv_fp8x4_e4m3 *)(&(data[i].w)))->__x =
304
+ (__nv_fp8x4_storage_t(val.__x) << 24) |
305
+ (__nv_fp8x4_storage_t(val.__x) << 16) |
306
+ (__nv_fp8x4_storage_t(val.__x) << 8) | __nv_fp8x4_storage_t(val.__x);
307
+ }
308
+ }
309
+ FLASHINFER_INLINE void load(const __nv_fp8_e4m3 *ptr) {
310
+#pragma unroll
311
+ for (size_t i = 0; i < vec_size / 16; ++i) {
312
+ data[i] = ((uint4 *)ptr)[i];
313
+ }
314
+ }
315
+ FLASHINFER_INLINE void store(__nv_fp8_e4m3 *ptr) const {
316
+#pragma unroll
317
+ for (size_t i = 0; i < vec_size / 16; ++i) {
318
+ ((uint4 *)ptr)[i] = data[i];
319
+ }
320
+ }
321
+ template <typename T>
322
+ FLASHINFER_INLINE void cast_from(const vec_t<T, vec_size> &src) {
323
+ cast_from_impl(src, *this);
324
+ }
325
+ template <typename T>
326
+ FLASHINFER_INLINE void cast_load(const T *ptr) {
327
+ cast_load_impl(ptr, *this);
328
+ }
329
+ template <typename T>
330
+ FLASHINFER_INLINE void cast_store(T *ptr) const {
331
+ cast_store_impl(*this, ptr);
332
+ }
333
+
334
+ FLASHINFER_INLINE static void memcpy(__nv_fp8_e4m3 *dst,
335
+ const __nv_fp8_e4m3 *src) {
336
+#pragma unroll
337
+ for (size_t i = 0; i < vec_size / 16; ++i) {
338
+ ((uint4 *)dst)[i] = ((uint4 *)src)[i];
339
+ }
340
+ }
341
+};
342
+
343
+/******************* vec_t<__nv_fp8_e5m2> *******************/
344
+
345
+// __nv_fp8_e5m2 x 1
346
+template <>
347
+struct vec_t<__nv_fp8_e5m2, 1> {
348
+ __nv_fp8_e5m2 data;
349
+
350
+ FLASHINFER_INLINE __nv_fp8_e5m2 &operator[](size_t i) {
351
+ return ((__nv_fp8_e5m2 *)(&data))[i];
352
+ }
353
+ FLASHINFER_INLINE const __nv_fp8_e5m2 &operator[](size_t i) const {
354
+ return ((const __nv_fp8_e5m2 *)(&data))[i];
355
+ }
356
+ FLASHINFER_INLINE void fill(__nv_fp8_e5m2 val);
357
+ FLASHINFER_INLINE void load(const __nv_fp8_e5m2 *ptr);
358
+ FLASHINFER_INLINE void store(__nv_fp8_e5m2 *ptr) const;
359
+ template <typename T>
360
+ FLASHINFER_INLINE void cast_from(const vec_t<T, 1> &src) {
361
+ cast_from_impl(src, *this);
362
+ }
363
+ template <typename T>
364
+ FLASHINFER_INLINE void cast_load(const T *ptr) {
365
+ cast_load_impl(ptr, *this);
366
+ }
367
+ template <typename T>
368
+ FLASHINFER_INLINE void cast_store(T *ptr) const {
369
+ cast_store_impl(*this, ptr);
370
+ }
371
+
372
+ FLASHINFER_INLINE static void memcpy(__nv_fp8_e5m2 *dst,
373
+ const __nv_fp8_e5m2 *src);
374
+};
375
+
376
+FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 1>::fill(__nv_fp8_e5m2 val) {
377
+ data = val;
378
+}
379
+
380
+FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 1>::load(const __nv_fp8_e5m2 *ptr) {
381
+ data = *ptr;
382
+}
383
+
384
+FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 1>::store(
385
+ __nv_fp8_e5m2 *ptr) const {
386
+ *ptr = data;
387
+}
388
+
389
+FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 1>::memcpy(
390
+ __nv_fp8_e5m2 *dst, const __nv_fp8_e5m2 *src) {
391
+ *dst = *src;
392
+}
393
+
394
+// __nv_fp8_e5m2 x 2
395
+template <>
396
+struct vec_t<__nv_fp8_e5m2, 2> {
397
+ __nv_fp8x2_e5m2 data;
398
+
399
+ FLASHINFER_INLINE __nv_fp8_e5m2 &operator[](size_t i) {
400
+ return ((__nv_fp8_e5m2 *)(&data))[i];
401
+ }
402
+ FLASHINFER_INLINE const __nv_fp8_e5m2 &operator[](size_t i) const {
403
+ return ((const __nv_fp8_e5m2 *)(&data))[i];
404
+ }
405
+ FLASHINFER_INLINE void fill(__nv_fp8_e5m2 val);
406
+ FLASHINFER_INLINE void load(const __nv_fp8_e5m2 *ptr);
407
+ FLASHINFER_INLINE void store(__nv_fp8_e5m2 *ptr) const;
408
+ template <typename T>
409
+ FLASHINFER_INLINE void cast_from(const vec_t<T, 2> &src) {
410
+ cast_from_impl(src, *this);
411
+ }
412
+ template <typename T>
413
+ FLASHINFER_INLINE void cast_load(const T *ptr) {
414
+ cast_load_impl(ptr, *this);
415
+ }
416
+ template <typename T>
417
+ FLASHINFER_INLINE void cast_store(T *ptr) const {
418
+ cast_store_impl(*this, ptr);
419
+ }
420
+
421
+ FLASHINFER_INLINE static void memcpy(__nv_fp8_e5m2 *dst,
422
+ const __nv_fp8_e5m2 *src);
423
+};
424
+
425
+FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 2>::fill(__nv_fp8_e5m2 val) {
426
+ data.__x =
427
+ (__nv_fp8x2_storage_t(val.__x) << 8) | __nv_fp8x2_storage_t(val.__x);
428
+}
429
+
430
+FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 2>::load(const __nv_fp8_e5m2 *ptr) {
431
+ data = *((__nv_fp8x2_e5m2 *)ptr);
432
+}
433
+
434
+FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 2>::store(
435
+ __nv_fp8_e5m2 *ptr) const {
436
+ *((__nv_fp8x2_e5m2 *)ptr) = data;
437
+}
438
+
439
+FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 2>::memcpy(
440
+ __nv_fp8_e5m2 *dst, const __nv_fp8_e5m2 *src) {
441
+ *((__nv_fp8x2_e5m2 *)dst) = *((__nv_fp8x2_e5m2 *)src);
442
+}
443
+
444
+// __nv_fp8_e5m2 x 4
445
+
446
+template <>
447
+struct vec_t<__nv_fp8_e5m2, 4> {
448
+ __nv_fp8x4_e5m2 data;
449
+
450
+ FLASHINFER_INLINE __nv_fp8_e5m2 &operator[](size_t i) {
451
+ return ((__nv_fp8_e5m2 *)(&data))[i];
452
+ }
453
+ FLASHINFER_INLINE const __nv_fp8_e5m2 &operator[](size_t i) const {
454
+ return ((const __nv_fp8_e5m2 *)(&data))[i];
455
+ }
456
+ FLASHINFER_INLINE void fill(__nv_fp8_e5m2 val);
457
+ FLASHINFER_INLINE void load(const __nv_fp8_e5m2 *ptr);
458
+ FLASHINFER_INLINE void store(__nv_fp8_e5m2 *ptr) const;
459
+ template <typename T>
460
+ FLASHINFER_INLINE void cast_from(const vec_t<T, 4> &src) {
461
+ cast_from_impl(src, *this);
462
+ }
463
+ template <typename T>
464
+ FLASHINFER_INLINE void cast_load(const T *ptr) {
465
+ cast_load_impl(ptr, *this);
466
+ }
467
+ template <typename T>
468
+ FLASHINFER_INLINE void cast_store(T *ptr) const {
469
+ cast_store_impl(*this, ptr);
470
+ }
471
+
472
+ FLASHINFER_INLINE static void memcpy(__nv_fp8_e5m2 *dst,
473
+ const __nv_fp8_e5m2 *src);
474
+};
475
+
476
+FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 4>::fill(__nv_fp8_e5m2 val) {
477
+ data.__x = (__nv_fp8x4_storage_t(val.__x) << 24) |
478
+ (__nv_fp8x4_storage_t(val.__x) << 16) |
479
+ (__nv_fp8x4_storage_t(val.__x) << 8) |
480
+ __nv_fp8x4_storage_t(val.__x);
481
+}
482
+
483
+FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 4>::load(const __nv_fp8_e5m2 *ptr) {
484
+ data = *((__nv_fp8x4_e5m2 *)ptr);
485
+}
486
+
487
+FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 4>::store(
488
+ __nv_fp8_e5m2 *ptr) const {
489
+ *((__nv_fp8x4_e5m2 *)ptr) = data;
490
+}
491
+
492
+FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 4>::memcpy(
493
+ __nv_fp8_e5m2 *dst, const __nv_fp8_e5m2 *src) {
494
+ *((__nv_fp8x4_e5m2 *)dst) = *((__nv_fp8x4_e5m2 *)src);
495
+}
496
+
497
+// __nv_fp8_e5m2 x 8
498
+
499
+template <>
500
+struct vec_t<__nv_fp8_e5m2, 8> {
501
+ uint2 data;
502
+
503
+ FLASHINFER_INLINE __nv_fp8_e5m2 &operator[](size_t i) {
504
+ return ((__nv_fp8_e5m2 *)(&data))[i];
505
+ }
506
+ FLASHINFER_INLINE const __nv_fp8_e5m2 &operator[](size_t i) const {
507
+ return ((const __nv_fp8_e5m2 *)(&data))[i];
508
+ }
509
+ FLASHINFER_INLINE void fill(__nv_fp8_e5m2 val);
510
+ FLASHINFER_INLINE void load(const __nv_fp8_e5m2 *ptr);
511
+ FLASHINFER_INLINE void store(__nv_fp8_e5m2 *ptr) const;
512
+ template <typename T>
513
+ FLASHINFER_INLINE void cast_from(const vec_t<T, 8> &src) {
514
+ cast_from_impl(src, *this);
515
+ }
516
+ template <typename T>
517
+ FLASHINFER_INLINE void cast_load(const T *ptr) {
518
+ cast_load_impl(ptr, *this);
519
+ }
520
+ template <typename T>
521
+ FLASHINFER_INLINE void cast_store(T *ptr) const {
522
+ cast_store_impl(*this, ptr);
523
+ }
524
+
525
+ FLASHINFER_INLINE static void memcpy(__nv_fp8_e5m2 *dst,
526
+ const __nv_fp8_e5m2 *src);
527
+};
528
+
529
+FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 8>::fill(__nv_fp8_e5m2 val) {
530
+ ((__nv_fp8x4_e5m2 *)(&data.x))->__x = (__nv_fp8x4_storage_t(val.__x) << 24) |
531
+ (__nv_fp8x4_storage_t(val.__x) << 16) |
532
+ (__nv_fp8x4_storage_t(val.__x) << 8) |
533
+ __nv_fp8x4_storage_t(val.__x);
534
+ ((__nv_fp8x4_e5m2 *)(&data.y))->__x = (__nv_fp8x4_storage_t(val.__x) << 24) |
535
+ (__nv_fp8x4_storage_t(val.__x) << 16) |
536
+ (__nv_fp8x4_storage_t(val.__x) << 8) |
537
+ __nv_fp8x4_storage_t(val.__x);
538
+}
539
+
540
+FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 8>::load(const __nv_fp8_e5m2 *ptr) {
541
+ data = *((uint2 *)ptr);
542
+}
543
+
544
+FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 8>::store(
545
+ __nv_fp8_e5m2 *ptr) const {
546
+ *((uint2 *)ptr) = data;
547
+}
548
+
549
+FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 8>::memcpy(
550
+ __nv_fp8_e5m2 *dst, const __nv_fp8_e5m2 *src) {
551
+ *((__nv_fp8_e5m2 *)dst) = *((__nv_fp8_e5m2 *)src);
552
+}
553
+
554
+// __nv_fp8_e5m2 x 16 or more
555
+
556
+template <size_t vec_size>
557
+struct vec_t<__nv_fp8_e5m2, vec_size> {
558
+ uint4 data[vec_size / 16];
559
+
560
+ FLASHINFER_INLINE __nv_fp8_e5m2 &operator[](size_t i) {
561
+ return ((__nv_fp8_e5m2 *)data)[i];
562
+ }
563
+ FLASHINFER_INLINE const __nv_fp8_e5m2 &operator[](size_t i) const {
564
+ return ((const __nv_fp8_e5m2 *)data)[i];
565
+ }
566
+ FLASHINFER_INLINE void fill(__nv_fp8_e5m2 val) {
567
+#pragma unroll
568
+ for (size_t i = 0; i < vec_size / 16; ++i) {
569
+ ((__nv_fp8x4_e5m2 *)(&(data[i].x)))->__x =
570
+ (__nv_fp8x4_storage_t(val.__x) << 24) |
571
+ (__nv_fp8x4_storage_t(val.__x) << 16) |
572
+ (__nv_fp8x4_storage_t(val.__x) << 8) | __nv_fp8x4_storage_t(val.__x);
573
+ ((__nv_fp8x4_e5m2 *)(&(data[i].y)))->__x =
574
+ (__nv_fp8x4_storage_t(val.__x) << 24) |
575
+ (__nv_fp8x4_storage_t(val.__x) << 16) |
576
+ (__nv_fp8x4_storage_t(val.__x) << 8) | __nv_fp8x4_storage_t(val.__x);
577
+ ((__nv_fp8x4_e5m2 *)(&(data[i].z)))->__x =
578
+ (__nv_fp8x4_storage_t(val.__x) << 24) |
579
+ (__nv_fp8x4_storage_t(val.__x) << 16) |
580
+ (__nv_fp8x4_storage_t(val.__x) << 8) | __nv_fp8x4_storage_t(val.__x);
581
+ ((__nv_fp8x4_e5m2 *)(&(data[i].w)))->__x =
582
+ (__nv_fp8x4_storage_t(val.__x) << 24) |
583
+ (__nv_fp8x4_storage_t(val.__x) << 16) |
584
+ (__nv_fp8x4_storage_t(val.__x) << 8) | __nv_fp8x4_storage_t(val.__x);
585
+ }
586
+ }
587
+ FLASHINFER_INLINE void load(const __nv_fp8_e5m2 *ptr) {
588
+#pragma unroll
589
+ for (size_t i = 0; i < vec_size / 16; ++i) {
590
+ data[i] = ((uint4 *)ptr)[i];
591
+ }
592
+ }
593
+ FLASHINFER_INLINE void store(__nv_fp8_e5m2 *ptr) const {
594
+#pragma unroll
595
+ for (size_t i = 0; i < vec_size / 16; ++i) {
596
+ ((uint4 *)ptr)[i] = data[i];
597
+ }
598
+ }
599
+ template <typename T>
600
+ FLASHINFER_INLINE void cast_from(const vec_t<T, vec_size> &src) {
601
+ cast_from_impl(src, *this);
602
+ }
603
+ template <typename T>
604
+ FLASHINFER_INLINE void cast_load(const T *ptr) {
605
+ cast_load_impl(ptr, *this);
606
+ }
607
+ template <typename T>
608
+ FLASHINFER_INLINE void cast_store(T *ptr) const {
609
+ cast_store_impl(*this, ptr);
610
+ }
611
+
612
+ FLASHINFER_INLINE static void memcpy(__nv_fp8_e5m2 *dst,
613
+ const __nv_fp8_e5m2 *src) {
614
+#pragma unroll
615
+ for (size_t i = 0; i < vec_size / 16; ++i) {
616
+ ((uint4 *)dst)[i] = ((uint4 *)src)[i];
617
+ }
618
+ }
619
+};
620
+#endif
621
+
622
+/******************* vec_t<half> *******************/
623
+
624
+// half x 1
625
+template <>
626
+struct vec_t<half, 1> {
627
+ half data;
628
+
629
+ FLASHINFER_INLINE half &operator[](size_t i) { return ((half *)(&data))[i]; }
630
+ FLASHINFER_INLINE const half &operator[](size_t i) const {
631
+ return ((const half *)(&data))[i];
632
+ }
633
+ FLASHINFER_INLINE void fill(half val);
634
+ FLASHINFER_INLINE void load(const half *ptr);
635
+ FLASHINFER_INLINE void store(half *ptr) const;
636
+ template <typename T>
637
+ FLASHINFER_INLINE void cast_from(const vec_t<T, 1> &src) {
638
+ cast_from_impl(src, *this);
639
+ }
640
+ template <typename T>
641
+ FLASHINFER_INLINE void cast_load(const T *ptr) {
642
+ cast_load_impl(ptr, *this);
643
+ }
644
+ template <typename T>
645
+ FLASHINFER_INLINE void cast_store(T *ptr) const {
646
+ cast_store_impl(*this, ptr);
647
+ }
648
+
649
+ FLASHINFER_INLINE static void memcpy(half *dst, const half *src);
650
+};
651
+
652
+FLASHINFER_INLINE void vec_t<half, 1>::fill(half val) { data = val; }
653
+
654
+FLASHINFER_INLINE void vec_t<half, 1>::load(const half *ptr) { data = *ptr; }
655
+
656
+FLASHINFER_INLINE void vec_t<half, 1>::store(half *ptr) const { *ptr = data; }
657
+
658
+FLASHINFER_INLINE void vec_t<half, 1>::memcpy(half *dst, const half *src) {
659
+ *dst = *src;
660
+}
661
+
662
+// half x 2
663
+template <>
664
+struct vec_t<half, 2> {
665
+ half2 data;
666
+
667
+ FLASHINFER_INLINE half &operator[](size_t i) { return ((half *)(&data))[i]; }
668
+ FLASHINFER_INLINE const half &operator[](size_t i) const {
669
+ return ((const half *)(&data))[i];
670
+ }
671
+ FLASHINFER_INLINE void fill(half val);
672
+ FLASHINFER_INLINE void load(const half *ptr);
673
+ FLASHINFER_INLINE void store(half *ptr) const;
674
+ template <typename T>
675
+ FLASHINFER_INLINE void cast_from(const vec_t<T, 2> &src) {
676
+ cast_from_impl(src, *this);
677
+ }
678
+ template <typename T>
679
+ FLASHINFER_INLINE void cast_load(const T *ptr) {
680
+ cast_load_impl(ptr, *this);
681
+ }
682
+ template <typename T>
683
+ FLASHINFER_INLINE void cast_store(T *ptr) const {
684
+ cast_store_impl(*this, ptr);
685
+ }
686
+
687
+ FLASHINFER_INLINE static void memcpy(half *dst, const half *src);
688
+};
689
+
690
+FLASHINFER_INLINE void vec_t<half, 2>::fill(half val) {
691
+ data = make_half2(val, val);
692
+}
693
+
694
+FLASHINFER_INLINE void vec_t<half, 2>::load(const half *ptr) {
695
+ data = *((half2 *)ptr);
696
+}
697
+
698
+FLASHINFER_INLINE void vec_t<half, 2>::store(half *ptr) const {
699
+ *((half2 *)ptr) = data;
700
+}
701
+
702
+FLASHINFER_INLINE void vec_t<half, 2>::memcpy(half *dst, const half *src) {
703
+ *((half2 *)dst) = *((half2 *)src);
704
+}
705
+
706
+// half x 4
707
+
708
+template <>
709
+struct vec_t<half, 4> {
710
+ uint2 data;
711
+
712
+ FLASHINFER_INLINE half &operator[](size_t i) { return ((half *)(&data))[i]; }
713
+ FLASHINFER_INLINE const half &operator[](size_t i) const {
714
+ return ((const half *)(&data))[i];
715
+ }
716
+ FLASHINFER_INLINE void fill(half val);
717
+ FLASHINFER_INLINE void load(const half *ptr);
718
+ FLASHINFER_INLINE void store(half *ptr) const;
719
+ template <typename T>
720
+ FLASHINFER_INLINE void cast_from(const vec_t<T, 4> &src) {
721
+ cast_from_impl(src, *this);
722
+ }
723
+ template <typename T>
724
+ FLASHINFER_INLINE void cast_load(const T *ptr) {
725
+ cast_load_impl(ptr, *this);
726
+ }
727
+ template <typename T>
728
+ FLASHINFER_INLINE void cast_store(T *ptr) const {
729
+ cast_store_impl(*this, ptr);
730
+ }
731
+
732
+ FLASHINFER_INLINE static void memcpy(half *dst, const half *src);
733
+};
734
+
735
+FLASHINFER_INLINE void vec_t<half, 4>::fill(half val) {
736
+ *(half2 *)(&data.x) = make_half2(val, val);
737
+ *(half2 *)(&data.y) = make_half2(val, val);
738
+}
739
+
740
+FLASHINFER_INLINE void vec_t<half, 4>::load(const half *ptr) {
741
+ data = *((uint2 *)ptr);
742
+}
743
+
744
+FLASHINFER_INLINE void vec_t<half, 4>::store(half *ptr) const {
745
+ *((uint2 *)ptr) = data;
746
+}
747
+
748
+FLASHINFER_INLINE void vec_t<half, 4>::memcpy(half *dst, const half *src) {
749
+ *((uint2 *)dst) = *((uint2 *)src);
750
+}
751
+
752
+// half x 8 or more
753
+
754
+template <size_t vec_size>
755
+struct vec_t<half, vec_size> {
756
+ uint4 data[vec_size / 8];
757
+ FLASHINFER_INLINE half &operator[](size_t i) { return ((half *)data)[i]; }
758
+ FLASHINFER_INLINE const half &operator[](size_t i) const {
759
+ return ((const half *)data)[i];
760
+ }
761
+ FLASHINFER_INLINE void fill(half val) {
762
+#pragma unroll
763
+ for (size_t i = 0; i < vec_size; ++i) {
764
+ *(half2 *)(&(data[i].x)) = make_half2(val, val);
765
+ *(half2 *)(&(data[i].y)) = make_half2(val, val);
766
+ *(half2 *)(&(data[i].z)) = make_half2(val, val);
767
+ *(half2 *)(&(data[i].w)) = make_half2(val, val);
768
+ }
769
+ }
770
+ FLASHINFER_INLINE void load(const half *ptr) {
771
+#pragma unroll
772
+ for (size_t i = 0; i < vec_size / 8; ++i) {
773
+ data[i] = ((uint4 *)ptr)[i];
774
+ }
775
+ }
776
+ FLASHINFER_INLINE void store(half *ptr) const {
777
+#pragma unroll
778
+ for (size_t i = 0; i < vec_size / 8; ++i) {
779
+ ((uint4 *)ptr)[i] = data[i];
780
+ }
781
+ }
782
+ template <typename T>
783
+ FLASHINFER_INLINE void cast_from(const vec_t<T, vec_size> &src) {
784
+ cast_from_impl(src, *this);
785
+ }
786
+ template <typename T>
787
+ FLASHINFER_INLINE void cast_load(const T *ptr) {
788
+ cast_load_impl(ptr, *this);
789
+ }
790
+ template <typename T>
791
+ FLASHINFER_INLINE void cast_store(T *ptr) const {
792
+ cast_store_impl(*this, ptr);
793
+ }
794
+
795
+ FLASHINFER_INLINE static void memcpy(half *dst, const half *src) {
796
+#pragma unroll
797
+ for (size_t i = 0; i < vec_size / 8; ++i) {
798
+ ((uint4 *)dst)[i] = ((uint4 *)src)[i];
799
+ }
800
+ }
801
+};
802
+
803
+/******************* vec_t<nv_bfloat16> *******************/
804
+
805
+// nv_bfloat16 x 1
806
+template <>
807
+struct vec_t<nv_bfloat16, 1> {
808
+ nv_bfloat16 data;
809
+
810
+ FLASHINFER_INLINE nv_bfloat16 &operator[](size_t i) {
811
+ return ((nv_bfloat16 *)(&data))[i];
812
+ }
813
+ FLASHINFER_INLINE const nv_bfloat16 &operator[](size_t i) const {
814
+ return ((const nv_bfloat16 *)(&data))[i];
815
+ }
816
+ FLASHINFER_INLINE void fill(nv_bfloat16 val);
817
+ FLASHINFER_INLINE void load(const nv_bfloat16 *ptr);
818
+ FLASHINFER_INLINE void store(nv_bfloat16 *ptr) const;
819
+ template <typename T>
820
+ FLASHINFER_INLINE void cast_from(const vec_t<T, 1> &src) {
821
+ cast_from_impl(src, *this);
822
+ }
823
+ template <typename T>
824
+ FLASHINFER_INLINE void cast_load(const T *ptr) {
825
+ cast_load_impl(ptr, *this);
826
+ }
827
+ template <typename T>
828
+ FLASHINFER_INLINE void cast_store(T *ptr) const {
829
+ cast_store_impl(*this, ptr);
830
+ }
831
+
832
+ FLASHINFER_INLINE static void memcpy(nv_bfloat16 *dst,
833
+ const nv_bfloat16 *src);
834
+};
835
+
836
+FLASHINFER_INLINE void vec_t<nv_bfloat16, 1>::fill(nv_bfloat16 val) {
837
+ data = val;
838
+}
839
+
840
+FLASHINFER_INLINE void vec_t<nv_bfloat16, 1>::load(const nv_bfloat16 *ptr) {
841
+ data = *ptr;
842
+}
843
+
844
+FLASHINFER_INLINE void vec_t<nv_bfloat16, 1>::store(nv_bfloat16 *ptr) const {
845
+ *ptr = data;
846
+}
847
+
848
+FLASHINFER_INLINE void vec_t<nv_bfloat16, 1>::memcpy(nv_bfloat16 *dst,
849
+ const nv_bfloat16 *src) {
850
+ *dst = *src;
851
+}
852
+
853
+// nv_bfloat16 x 2
854
+template <>
855
+struct vec_t<nv_bfloat16, 2> {
856
+ nv_bfloat162 data;
857
+
858
+ FLASHINFER_INLINE nv_bfloat16 &operator[](size_t i) {
859
+ return ((nv_bfloat16 *)(&data))[i];
860
+ }
861
+ FLASHINFER_INLINE const nv_bfloat16 &operator[](size_t i) const {
862
+ return ((const nv_bfloat16 *)(&data))[i];
863
+ }
864
+ FLASHINFER_INLINE void fill(nv_bfloat16 val);
865
+ FLASHINFER_INLINE void load(const nv_bfloat16 *ptr);
866
+ FLASHINFER_INLINE void store(nv_bfloat16 *ptr) const;
867
+ template <typename T>
868
+ FLASHINFER_INLINE void cast_from(const vec_t<T, 2> &src) {
869
+ cast_from_impl(src, *this);
870
+ }
871
+ template <typename T>
872
+ FLASHINFER_INLINE void cast_load(const T *ptr) {
873
+ cast_load_impl(ptr, *this);
874
+ }
875
+ template <typename T>
876
+ FLASHINFER_INLINE void cast_store(T *ptr) const {
877
+ cast_store_impl(*this, ptr);
878
+ }
879
+
880
+ FLASHINFER_INLINE static void memcpy(nv_bfloat16 *dst,
881
+ const nv_bfloat16 *src);
882
+};
883
+
884
+FLASHINFER_INLINE void vec_t<nv_bfloat16, 2>::fill(nv_bfloat16 val) {
885
+ data = make_bfloat162(val, val);
886
+}
887
+
888
+FLASHINFER_INLINE void vec_t<nv_bfloat16, 2>::load(const nv_bfloat16 *ptr) {
889
+ data = *((nv_bfloat162 *)ptr);
890
+}
891
+
892
+FLASHINFER_INLINE void vec_t<nv_bfloat16, 2>::store(nv_bfloat16 *ptr) const {
893
+ *((nv_bfloat162 *)ptr) = data;
894
+}
895
+
896
+FLASHINFER_INLINE void vec_t<nv_bfloat16, 2>::memcpy(nv_bfloat16 *dst,
897
+ const nv_bfloat16 *src) {
898
+ *((nv_bfloat162 *)dst) = *((nv_bfloat162 *)src);
899
+}
900
+
901
+// nv_bfloat16 x 4
902
+
903
+template <>
904
+struct vec_t<nv_bfloat16, 4> {
905
+ uint2 data;
906
+
907
+ FLASHINFER_INLINE nv_bfloat16 &operator[](size_t i) {
908
+ return ((nv_bfloat16 *)(&data))[i];
909
+ }
910
+ FLASHINFER_INLINE const nv_bfloat16 &operator[](size_t i) const {
911
+ return ((const nv_bfloat16 *)(&data))[i];
912
+ }
913
+ FLASHINFER_INLINE void fill(nv_bfloat16 val);
914
+ FLASHINFER_INLINE void load(const nv_bfloat16 *ptr);
915
+ FLASHINFER_INLINE void store(nv_bfloat16 *ptr) const;
916
+ template <typename T>
917
+ FLASHINFER_INLINE void cast_from(const vec_t<T, 4> &src) {
918
+ cast_from_impl(src, *this);
919
+ }
920
+ template <typename T>
921
+ FLASHINFER_INLINE void cast_load(const T *ptr) {
922
+ cast_load_impl(ptr, *this);
923
+ }
924
+ template <typename T>
925
+ FLASHINFER_INLINE void cast_store(T *ptr) const {
926
+ cast_store_impl(*this, ptr);
927
+ }
928
+
929
+ FLASHINFER_INLINE static void memcpy(nv_bfloat16 *dst,
930
+ const nv_bfloat16 *src);
931
+};
932
+
933
+FLASHINFER_INLINE void vec_t<nv_bfloat16, 4>::fill(nv_bfloat16 val) {
934
+ *(nv_bfloat162 *)(&data.x) = make_bfloat162(val, val);
935
+ *(nv_bfloat162 *)(&data.y) = make_bfloat162(val, val);
936
+}
937
+
938
+FLASHINFER_INLINE void vec_t<nv_bfloat16, 4>::load(const nv_bfloat16 *ptr) {
939
+ data = *((uint2 *)ptr);
940
+}
941
+
942
+FLASHINFER_INLINE void vec_t<nv_bfloat16, 4>::store(nv_bfloat16 *ptr) const {
943
+ *((uint2 *)ptr) = data;
944
+}
945
+
946
+FLASHINFER_INLINE void vec_t<nv_bfloat16, 4>::memcpy(nv_bfloat16 *dst,
947
+ const nv_bfloat16 *src) {
948
+ *((uint2 *)dst) = *((uint2 *)src);
949
+}
950
+
951
+// nv_bfloat16 x 8 or more
952
+
953
+template <size_t vec_size>
954
+struct vec_t<nv_bfloat16, vec_size> {
955
+ uint4 data[vec_size / 8];
956
+
957
+ FLASHINFER_INLINE nv_bfloat16 &operator[](size_t i) {
958
+ return ((nv_bfloat16 *)data)[i];
959
+ }
960
+ FLASHINFER_INLINE const nv_bfloat16 &operator[](size_t i) const {
961
+ return ((const nv_bfloat16 *)data)[i];
962
+ }
963
+ FLASHINFER_INLINE void fill(nv_bfloat16 val) {
964
+#pragma unoll
965
+ for (size_t i = 0; i < vec_size / 8; ++i) {
966
+ *(nv_bfloat162 *)(&(data[i].x)) = make_bfloat162(val, val);
967
+ *(nv_bfloat162 *)(&(data[i].y)) = make_bfloat162(val, val);
968
+ *(nv_bfloat162 *)(&(data[i].z)) = make_bfloat162(val, val);
969
+ *(nv_bfloat162 *)(&(data[i].w)) = make_bfloat162(val, val);
970
+ }
971
+ }
972
+ FLASHINFER_INLINE void load(const nv_bfloat16 *ptr) {
973
+#pragma unoll
974
+ for (size_t i = 0; i < vec_size / 8; ++i) {
975
+ data[i] = ((uint4 *)ptr)[i];
976
+ }
977
+ }
978
+ FLASHINFER_INLINE void store(nv_bfloat16 *ptr) const {
979
+#pragma unoll
980
+ for (size_t i = 0; i < vec_size / 8; ++i) {
981
+ ((uint4 *)ptr)[i] = data[i];
982
+ }
983
+ }
984
+ template <typename T>
985
+ FLASHINFER_INLINE void cast_from(const vec_t<T, vec_size> &src) {
986
+ cast_from_impl(src, *this);
987
+ }
988
+ template <typename T>
989
+ FLASHINFER_INLINE void cast_load(const T *ptr) {
990
+ cast_load_impl(ptr, *this);
991
+ }
992
+ template <typename T>
993
+ FLASHINFER_INLINE void cast_store(T *ptr) const {
994
+ cast_store_impl(*this, ptr);
995
+ }
996
+
997
+ FLASHINFER_INLINE static void memcpy(nv_bfloat16 *dst,
998
+ const nv_bfloat16 *src) {
999
+#pragma unoll
1000
+ for (size_t i = 0; i < vec_size / 8; ++i) {
1001
+ ((uint4 *)dst)[i] = ((uint4 *)src)[i];
1002
+ }
1003
+ }
1004
+};
1005
+
1006
+/******************* vec_t<float> *******************/
1007
+
1008
+// float x 1
1009
+
1010
+template <>
1011
+struct vec_t<float, 1> {
1012
+ float data;
1013
+
1014
+ FLASHINFER_INLINE float &operator[](size_t i) {
1015
+ return ((float *)(&data))[i];
1016
+ }
1017
+ FLASHINFER_INLINE const float &operator[](size_t i) const {
1018
+ return ((const float *)(&data))[i];
1019
+ }
1020
+ FLASHINFER_INLINE void fill(float val);
1021
+ FLASHINFER_INLINE void load(const float *ptr);
1022
+ FLASHINFER_INLINE void store(float *ptr) const;
1023
+ template <typename T>
1024
+ FLASHINFER_INLINE void cast_from(const vec_t<T, 1> &src) {
1025
+ cast_from_impl(src, *this);
1026
+ }
1027
+ template <typename T>
1028
+ FLASHINFER_INLINE void cast_load(const T *ptr) {
1029
+ cast_load_impl(ptr, *this);
1030
+ }
1031
+ template <typename T>
1032
+ FLASHINFER_INLINE void cast_store(T *ptr) const {
1033
+ cast_store_impl(*this, ptr);
1034
+ }
1035
+
1036
+ FLASHINFER_INLINE static void memcpy(float *dst, const float *src);
1037
+};
1038
+
1039
+FLASHINFER_INLINE void vec_t<float, 1>::fill(float val) { data = val; }
1040
+
1041
+FLASHINFER_INLINE void vec_t<float, 1>::load(const float *ptr) { data = *ptr; }
1042
+
1043
+FLASHINFER_INLINE void vec_t<float, 1>::store(float *ptr) const { *ptr = data; }
1044
+
1045
+FLASHINFER_INLINE void vec_t<float, 1>::memcpy(float *dst, const float *src) {
1046
+ *dst = *src;
1047
+}
1048
+
1049
+// float x 2
1050
+
1051
+template <>
1052
+struct vec_t<float, 2> {
1053
+ float2 data;
1054
+
1055
+ FLASHINFER_INLINE float &operator[](size_t i) {
1056
+ return ((float *)(&data))[i];
1057
+ }
1058
+ FLASHINFER_INLINE const float &operator[](size_t i) const {
1059
+ return ((const float *)(&data))[i];
1060
+ }
1061
+ FLASHINFER_INLINE void fill(float val);
1062
+ FLASHINFER_INLINE void load(const float *ptr);
1063
+ FLASHINFER_INLINE void store(float *ptr) const;
1064
+ template <typename T>
1065
+ FLASHINFER_INLINE void cast_from(const vec_t<T, 2> &src) {
1066
+ cast_from_impl(src, *this);
1067
+ }
1068
+ template <typename T>
1069
+ FLASHINFER_INLINE void cast_load(const T *ptr) {
1070
+ cast_load_impl(ptr, *this);
1071
+ }
1072
+ template <typename T>
1073
+ FLASHINFER_INLINE void cast_store(T *ptr) const {
1074
+ cast_store_impl(*this, ptr);
1075
+ }
1076
+ FLASHINFER_INLINE static void memcpy(float *dst, const float *src);
1077
+};
1078
+
1079
+FLASHINFER_INLINE void vec_t<float, 2>::fill(float val) {
1080
+ data = make_float2(val, val);
1081
+}
1082
+
1083
+FLASHINFER_INLINE void vec_t<float, 2>::load(const float *ptr) {
1084
+ data = *((float2 *)ptr);
1085
+}
1086
+
1087
+FLASHINFER_INLINE void vec_t<float, 2>::store(float *ptr) const {
1088
+ *((float2 *)ptr) = data;
1089
+}
1090
+
1091
+FLASHINFER_INLINE void vec_t<float, 2>::memcpy(float *dst, const float *src) {
1092
+ *((float2 *)dst) = *((float2 *)src);
1093
+}
1094
+
1095
+// float x 4 or more
1096
+template <size_t vec_size>
1097
+struct vec_t<float, vec_size> {
1098
+ float4 data[vec_size / 4];
1099
+
1100
+ FLASHINFER_INLINE float &operator[](size_t i) { return ((float *)(data))[i]; }
1101
+ FLASHINFER_INLINE const float &operator[](size_t i) const {
1102
+ return ((const float *)(data))[i];
1103
+ }
1104
+ FLASHINFER_INLINE void fill(float val) {
1105
+#pragma unroll
1106
+ for (size_t i = 0; i < vec_size / 4; ++i) {
1107
+ data[i] = make_float4(val, val, val, val);
1108
+ }
1109
+ }
1110
+ FLASHINFER_INLINE void load(const float *ptr) {
1111
+#pragma unroll
1112
+ for (size_t i = 0; i < vec_size / 4; ++i) {
1113
+ data[i] = ((float4 *)ptr)[i];
1114
+ }
1115
+ }
1116
+ FLASHINFER_INLINE void store(float *ptr) const {
1117
+#pragma unroll
1118
+ for (size_t i = 0; i < vec_size / 4; ++i) {
1119
+ ((float4 *)ptr)[i] = data[i];
1120
+ }
1121
+ }
1122
+ template <typename T>
1123
+ FLASHINFER_INLINE void cast_from(const vec_t<T, vec_size> &src) {
1124
+ cast_from_impl(src, *this);
1125
+ }
1126
+ template <typename T>
1127
+ FLASHINFER_INLINE void cast_load(const T *ptr) {
1128
+ cast_load_impl(ptr, *this);
1129
+ }
1130
+ template <typename T>
1131
+ FLASHINFER_INLINE void cast_store(T *ptr) const {
1132
+ cast_store_impl(*this, ptr);
1133
+ }
1134
+ FLASHINFER_INLINE static void memcpy(float *dst, const float *src) {
1135
+#pragma unroll
1136
+ for (size_t i = 0; i < vec_size / 4; ++i) {
1137
+ ((float4 *)dst)[i] = ((float4 *)src)[i];
1138
+ }
1139
+ }
1140
+};
1141
+
1142
+/******************* vec_t type cast *******************/
1143
+
1144
+template <size_t vec_size>
1145
+FLASHINFER_INLINE void cast_from_impl(const vec_t<half, vec_size> &src,
1146
+ vec_t<float, vec_size> &dst) {
1147
+ if constexpr (vec_size == 1) {
1148
+ dst.data = float(src.data);
1149
+ } else {
1150
+#pragma unroll
1151
+ for (size_t i = 0; i < vec_size / 2; ++i) {
1152
+ ((float2 *)(&dst.data))[i] = __half22float2(((half2 *)(&src.data))[i]);
1153
+ }
1154
+ }
1155
+}
1156
+
1157
+template <size_t vec_size>
1158
+FLASHINFER_INLINE void cast_from_impl(const vec_t<float, vec_size> &src,
1159
+ vec_t<half, vec_size> &dst) {
1160
+ if constexpr (vec_size == 1) {
1161
+ dst.data = half(src.data);
1162
+ } else {
1163
+#pragma unroll
1164
+ for (size_t i = 0; i < vec_size / 2; ++i) {
1165
+ ((half2 *)(&dst.data))[i] = __float22half2_rn(((float2 *)(&src.data))[i]);
1166
+ }
1167
+ }
1168
+}
1169
+
1170
+template <size_t vec_size>
1171
+FLASHINFER_INLINE void cast_from_impl(const vec_t<nv_bfloat16, vec_size> &src,
1172
+ vec_t<float, vec_size> &dst) {
1173
+ if constexpr (vec_size == 1) {
1174
+ dst.data = float(src.data);
1175
+ } else {
1176
+#pragma unroll
1177
+ for (size_t i = 0; i < vec_size / 2; ++i) {
1178
+ ((float2 *)(&dst.data))[i] =
1179
+ __bfloat1622float2(((nv_bfloat162 *)(&src.data))[i]);
1180
+ }
1181
+ }
1182
+}
1183
+
1184
+template <size_t vec_size>
1185
+FLASHINFER_INLINE void cast_from_impl(const vec_t<float, vec_size> &src,
1186
+ vec_t<nv_bfloat16, vec_size> &dst) {
1187
+ if constexpr (vec_size == 1) {
1188
+ dst.data = nv_bfloat16(src.data);
1189
+ } else {
1190
+#pragma unroll
1191
+ for (size_t i = 0; i < vec_size / 2; ++i) {
1192
+ ((nv_bfloat162 *)(&dst.data))[i] =
1193
+ __float22bfloat162_rn(((float2 *)(&src.data))[i]);
1194
+ }
1195
+ }
1196
+}
1197
+
1198
+#ifdef FLASHINFER_USE_FP8
1199
+
1200
+template <size_t vec_size>
1201
+FLASHINFER_INLINE void cast_from_impl(const vec_t<__nv_fp8_e4m3, vec_size> &src,
1202
+ vec_t<float, vec_size> &dst) {
1203
+ if constexpr (vec_size == 1) {
1204
+ dst.data = float(src.data);
1205
+ } else if constexpr (vec_size == 2) {
1206
+ *(float2 *)(&dst.data) = float2(*(__nv_fp8x2_e4m3 *)(&src.data));
1207
+ } else {
1208
+#pragma unroll
1209
+ for (size_t i = 0; i < vec_size / 4; ++i) {
1210
+ ((float4 *)(&dst.data))[i] = float4(((__nv_fp8x4_e4m3 *)(&src.data))[i]);
1211
+ }
1212
+ }
1213
+}
1214
+
1215
+template <size_t vec_size>
1216
+FLASHINFER_INLINE void cast_from_impl(const vec_t<__nv_fp8_e4m3, vec_size> &src,
1217
+ vec_t<half, vec_size> &dst) {
1218
+ if constexpr (vec_size == 1) {
1219
+ dst.data = float(src.data);
1220
+ } else {
1221
+#pragma unroll
1222
+ for (size_t i = 0; i < vec_size / 2; ++i) {
1223
+ ((half2 *)(&dst.data))[i] = half2(((__nv_fp8x2_e4m3 *)(&src.data))[i]);
1224
+ }
1225
+ }
1226
+}
1227
+
1228
+template <size_t vec_size>
1229
+FLASHINFER_INLINE void cast_from_impl(const vec_t<float, vec_size> &src,
1230
+ vec_t<__nv_fp8_e4m3, vec_size> &dst) {
1231
+ if constexpr (vec_size == 1) {
1232
+ dst.data = __nv_fp8_e4m3(src.data);
1233
+ } else if constexpr (vec_size == 2) {
1234
+ *(__nv_fp8x2_e4m3 *)(&dst.data) = __nv_fp8x2_e4m3(*(float2 *)(&src.data));
1235
+ } else {
1236
+#pragma unroll
1237
+ for (size_t i = 0; i < vec_size / 4; ++i) {
1238
+ ((__nv_fp8x4_e4m3 *)(&dst.data))[i] =
1239
+ __nv_fp8x4_e4m3(((float4 *)(&src.data))[i]);
1240
+ }
1241
+ }
1242
+}
1243
+
1244
+template <size_t vec_size>
1245
+FLASHINFER_INLINE void cast_from_impl(const vec_t<half, vec_size> &src,
1246
+ vec_t<__nv_fp8_e4m3, vec_size> &dst) {
1247
+ if constexpr (vec_size == 1) {
1248
+ dst.data = __nv_fp8_e4m3(src.data);
1249
+ } else if constexpr (vec_size == 2) {
1250
+ *(__nv_fp8x2_e4m3 *)(&dst.data) = __nv_fp8x2_e4m3(*(half2 *)(&src.data));
1251
+ } else {
1252
+#pragma unroll
1253
+ for (size_t i = 0; i < vec_size / 4; ++i) {
1254
+ // NOTE(Zihao): need to double check if we properly handle flo and fhi
1255
+ ((__nv_fp8x4_e4m3 *)(&dst.data))[i] = __nv_fp8x4_e4m3(
1256
+ ((half2 *)(&src.data))[i * 2], ((half2 *)(&src.data))[i * 2 + 1]);
1257
+ }
1258
+ }
1259
+}
1260
+
1261
+template <size_t vec_size>
1262
+FLASHINFER_INLINE void cast_from_impl(const vec_t<__nv_fp8_e5m2, vec_size> &src,
1263
+ vec_t<float, vec_size> &dst) {
1264
+ if constexpr (vec_size == 1) {
1265
+ dst.data = float(src.data);
1266
+ } else if constexpr (vec_size == 2) {
1267
+ *(float2 *)(&dst.data) = float2(*(__nv_fp8x2_e5m2 *)(&src.data));
1268
+ } else {
1269
+#pragma unroll
1270
+ for (size_t i = 0; i < vec_size / 4; ++i) {
1271
+ ((float4 *)(&dst.data))[i] = float4(((__nv_fp8x4_e5m2 *)(&src.data))[i]);
1272
+ }
1273
+ }
1274
+}
1275
+
1276
+template <size_t vec_size>
1277
+FLASHINFER_INLINE void cast_from_impl(const vec_t<__nv_fp8_e5m2, vec_size> &src,
1278
+ vec_t<half, vec_size> &dst) {
1279
+ if constexpr (vec_size == 1) {
1280
+ dst.data = float(src.data);
1281
+ } else {
1282
+#pragma unroll
1283
+ for (size_t i = 0; i < vec_size / 2; ++i) {
1284
+ ((half2 *)(&dst.data))[i] = half2(((__nv_fp8x2_e5m2 *)(&src.data))[i]);
1285
+ }
1286
+ }
1287
+}
1288
+
1289
+template <size_t vec_size>
1290
+FLASHINFER_INLINE void cast_from_impl(const vec_t<float, vec_size> &src,
1291
+ vec_t<__nv_fp8_e5m2, vec_size> &dst) {
1292
+ if constexpr (vec_size == 1) {
1293
+ dst.data = __nv_fp8_e5m2(src.data);
1294
+ } else if constexpr (vec_size == 2) {
1295
+ *(__nv_fp8x2_e5m2 *)(&dst.data) = __nv_fp8x2_e5m2(*(float2 *)(&src.data));
1296
+ } else {
1297
+#pragma unroll
1298
+ for (size_t i = 0; i < vec_size / 4; ++i) {
1299
+ ((__nv_fp8x4_e5m2 *)(&dst.data))[i] =
1300
+ __nv_fp8x4_e5m2(((float4 *)(&src.data))[i]);
1301
+ }
1302
+ }
1303
+}
1304
+
1305
+template <size_t vec_size>
1306
+FLASHINFER_INLINE void cast_from_impl(const vec_t<half, vec_size> &src,
1307
+ vec_t<__nv_fp8_e5m2, vec_size> &dst) {
1308
+ if constexpr (vec_size == 1) {
1309
+ dst.data = __nv_fp8_e4m3(src.data);
1310
+ } else if constexpr (vec_size == 2) {
1311
+ *(__nv_fp8x2_e5m2 *)(&dst.data) = __nv_fp8x2_e5m2(*(half2 *)(&src.data));
1312
+ } else {
1313
+#pragma unroll
1314
+ for (size_t i = 0; i < vec_size / 4; ++i) {
1315
+ // NOTE(Zihao): need to double check if we properly handle flo and fhi
1316
+ ((__nv_fp8x4_e5m2 *)(&dst.data))[i] = __nv_fp8x4_e5m2(
1317
+ ((half2 *)(&src.data))[i * 2], ((half2 *)(&src.data))[i * 2 + 1]);
1318
+ }
1319
+ }
1320
+}
1321
+
1322
+#endif // FLASHINFER_USE_FP8
1323
+
1324
+#endif // VEC_DTYPES_CUH_
csrc/punica/punica_ops.cc
new
+563
@@ -0,0 +1,563 @@
1
+#include <cuda_bf16.h>
2
+#include <cuda_fp16.h>
3
+#include <torch/extension.h>
4
+
5
+#include <cstdint>
6
+
7
+#include "bgmv/bgmv_config.h"
8
+
9
+namespace {
10
+
11
+//====== utils ======
12
+
13
+inline void check_shape(const torch::Tensor &a, const torch::Tensor &b,
14
+ const char *a_name, const char *b_name) {
15
+ TORCH_CHECK(a.dim() == b.dim(), a_name, ".dim() != ", b_name, ".dim(). ",
16
+ a.dim(), " vs ", b.dim());
17
+ for (int i = 0; i < a.dim(); ++i) {
18
+ TORCH_CHECK(a.size(i) == b.size(i), a_name, ".size(", i, ") != ", b_name,
19
+ ".size(", i, ")");
20
+ }
21
+}
22
+
23
+inline constexpr uint32_t pack_u16(uint16_t a, uint16_t b) {
24
+ return (uint32_t(a) << 16) | uint32_t(b);
25
+}
26
+
27
+#define CHECK_CUDA(x) TORCH_CHECK(x.is_cuda(), #x " must be a CUDA tensor")
28
+
29
+#define CHECK_CONTIGUOUS(x) \
30
+ TORCH_CHECK(x.is_contiguous(), #x " must be contiguous")
31
+
32
+#define CHECK_INPUT(x) \
33
+ CHECK_CUDA(x); \
34
+ CHECK_CONTIGUOUS(x)
35
+
36
+#define CHECK_DIM(d, x) \
37
+ TORCH_CHECK(x.dim() == d, #x " must be a " #d "D tensor")
38
+
39
+#define CHECK_SHAPE(a, b) check_shape(a, b, #a, #b)
40
+
41
+#define CHECK_EQ(a, b) \
42
+ TORCH_CHECK(a == b, "CHECK_EQ(" #a ", " #b ") failed. ", a, " vs ", b)
43
+
44
+//====== bgmv ======
45
+
46
+template <typename in_T, typename out_T, typename W_T>
47
+inline bool launch_bgmv_kernel(out_T *Y, const in_T *X, const W_T *W,
48
+ const int64_t *lora_indices,
49
+ uint16_t in_features, uint16_t out_features,
50
+ int64_t y_offset, int64_t full_y_size,
51
+ int64_t batch_size, int64_t num_layers,
52
+ int64_t layer_idx, float scale) {
53
+ switch (pack_u16(in_features, out_features)) {
54
+#define CASE_ONESIDE(_in_T, _out_T, _W_T, feat_in, feat_out) \
55
+ case pack_u16(feat_in, feat_out): \
56
+ bgmv_kernel<feat_in, feat_out>(Y, X, W, lora_indices, y_offset, \
57
+ full_y_size, batch_size, num_layers, \
58
+ layer_idx, scale); \
59
+ break;
60
+#define CASE(_in_T, _out_T, _W_T, narrow, wide) \
61
+ CASE_ONESIDE(in_T, out_T, W_T, narrow, wide) \
62
+ CASE_ONESIDE(in_T, out_T, W_T, wide, narrow)
63
+
64
+ FOR_BGMV_WIDE_NARROW(CASE, _, _, _)
65
+#undef CASE
66
+#undef CASE_ONESIDE
67
+ default:
68
+ return false;
69
+ }
70
+
71
+ return true;
72
+}
73
+
74
+void dispatch_bgmv(torch::Tensor y, torch::Tensor x, torch::Tensor w,
75
+ torch::Tensor indicies, int64_t layer_idx, float scale) {
76
+ CHECK_INPUT(y);
77
+ CHECK_INPUT(x);
78
+ CHECK_INPUT(w);
79
+ CHECK_INPUT(indicies);
80
+
81
+ CHECK_DIM(2, y);
82
+ CHECK_DIM(2, x);
83
+ CHECK_DIM(4, w);
84
+ CHECK_DIM(1, indicies);
85
+
86
+ int64_t B = x.size(0);
87
+ int64_t h_in = x.size(1);
88
+ int64_t h_out = y.size(1);
89
+ int64_t num_layers = w.size(1);
90
+ CHECK_EQ(w.size(3), h_in);
91
+ CHECK_EQ(w.size(2), h_out);
92
+ CHECK_EQ(indicies.size(0), x.size(0));
93
+ CHECK_EQ(y.size(0), x.size(0));
94
+ bool ok = false;
95
+ if (h_in < 65536 && h_out < 65536) {
96
+ // TODO: See if we can get rid of this massive nested switch
97
+ switch (x.scalar_type()) {
98
+ case at::ScalarType::Half:
99
+ switch (y.scalar_type()) {
100
+ case at::ScalarType::Half:
101
+ switch (w.scalar_type()) {
102
+ case at::ScalarType::Half:
103
+ ok = launch_bgmv_kernel(static_cast<nv_half *>(y.data_ptr()),
104
+ static_cast<nv_half *>(x.data_ptr()),
105
+ static_cast<nv_half *>(w.data_ptr()),
106
+ indicies.data_ptr<int64_t>(), h_in, h_out, 0,
107
+ h_out, B, num_layers, layer_idx, scale);
108
+ break;
109
+ case at::ScalarType::BFloat16:
110
+ ok = launch_bgmv_kernel(static_cast<nv_half *>(y.data_ptr()),
111
+ static_cast<nv_half *>(x.data_ptr()),
112
+ static_cast<nv_bfloat16 *>(w.data_ptr()),
113
+ indicies.data_ptr<int64_t>(), h_in, h_out, 0,
114
+ h_out, B, num_layers, layer_idx, scale);
115
+ break;
116
+ default:
117
+ break;
118
+ }
119
+ break;
120
+ case at::ScalarType::BFloat16:
121
+ switch (w.scalar_type()) {
122
+ case at::ScalarType::Half:
123
+ ok = launch_bgmv_kernel(static_cast<nv_bfloat16 *>(y.data_ptr()),
124
+ static_cast<nv_half *>(x.data_ptr()),
125
+ static_cast<nv_half *>(w.data_ptr()),
126
+ indicies.data_ptr<int64_t>(), h_in, h_out, 0,
127
+ h_out, B, num_layers, layer_idx, scale);
128
+ break;
129
+ case at::ScalarType::BFloat16:
130
+ ok = launch_bgmv_kernel(static_cast<nv_bfloat16 *>(y.data_ptr()),
131
+ static_cast<nv_half *>(x.data_ptr()),
132
+ static_cast<nv_bfloat16 *>(w.data_ptr()),
133
+ indicies.data_ptr<int64_t>(), h_in, h_out, 0,
134
+ h_out, B, num_layers, layer_idx, scale);
135
+ break;
136
+ default:
137
+ break;
138
+ }
139
+ break;
140
+ case at::ScalarType::Float:
141
+ switch (w.scalar_type()) {
142
+ case at::ScalarType::Half:
143
+ ok = launch_bgmv_kernel(static_cast<float *>(y.data_ptr()),
144
+ static_cast<nv_half *>(x.data_ptr()),
145
+ static_cast<nv_half *>(w.data_ptr()),
146
+ indicies.data_ptr<int64_t>(), h_in, h_out, 0,
147
+ h_out, B, num_layers, layer_idx, scale);
148
+ break;
149
+ case at::ScalarType::BFloat16:
150
+ ok = launch_bgmv_kernel(static_cast<float *>(y.data_ptr()),
151
+ static_cast<nv_half *>(x.data_ptr()),
152
+ static_cast<nv_bfloat16 *>(w.data_ptr()),
153
+ indicies.data_ptr<int64_t>(), h_in, h_out, 0,
154
+ h_out, B, num_layers, layer_idx, scale);
155
+ break;
156
+ default:
157
+ break;
158
+ }
159
+ break;
160
+ default:
161
+ break;
162
+ }
163
+ break;
164
+ case at::ScalarType::BFloat16:
165
+ switch (y.scalar_type()) {
166
+ case at::ScalarType::Half:
167
+ switch (w.scalar_type()) {
168
+ case at::ScalarType::Half:
169
+ ok = launch_bgmv_kernel(static_cast<nv_half *>(y.data_ptr()),
170
+ static_cast<nv_bfloat16 *>(x.data_ptr()),
171
+ static_cast<nv_half *>(w.data_ptr()),
172
+ indicies.data_ptr<int64_t>(), h_in, h_out, 0,
173
+ h_out, B, num_layers, layer_idx, scale);
174
+ break;
175
+ case at::ScalarType::BFloat16:
176
+ ok = launch_bgmv_kernel(static_cast<nv_half *>(y.data_ptr()),
177
+ static_cast<nv_bfloat16 *>(x.data_ptr()),
178
+ static_cast<nv_bfloat16 *>(w.data_ptr()),
179
+ indicies.data_ptr<int64_t>(), h_in, h_out, 0,
180
+ h_out, B, num_layers, layer_idx, scale);
181
+ break;
182
+ default:
183
+ break;
184
+ }
185
+ break;
186
+ case at::ScalarType::BFloat16:
187
+ switch (w.scalar_type()) {
188
+ case at::ScalarType::Half:
189
+ ok = launch_bgmv_kernel(static_cast<nv_bfloat16 *>(y.data_ptr()),
190
+ static_cast<nv_bfloat16 *>(x.data_ptr()),
191
+ static_cast<nv_half *>(w.data_ptr()),
192
+ indicies.data_ptr<int64_t>(), h_in, h_out, 0,
193
+ h_out, B, num_layers, layer_idx, scale);
194
+ break;
195
+ case at::ScalarType::BFloat16:
196
+ ok = launch_bgmv_kernel(static_cast<nv_bfloat16 *>(y.data_ptr()),
197
+ static_cast<nv_bfloat16 *>(x.data_ptr()),
198
+ static_cast<nv_bfloat16 *>(w.data_ptr()),
199
+ indicies.data_ptr<int64_t>(), h_in, h_out, 0,
200
+ h_out, B, num_layers, layer_idx, scale);
201
+ break;
202
+ default:
203
+ break;
204
+ }
205
+ break;
206
+ case at::ScalarType::Float:
207
+ switch (w.scalar_type()) {
208
+ case at::ScalarType::Half:
209
+ ok = launch_bgmv_kernel(static_cast<float *>(y.data_ptr()),
210
+ static_cast<nv_bfloat16 *>(x.data_ptr()),
211
+ static_cast<nv_half *>(w.data_ptr()),
212
+ indicies.data_ptr<int64_t>(), h_in, h_out, 0,
213
+ h_out, B, num_layers, layer_idx, scale);
214
+ break;
215
+ case at::ScalarType::BFloat16:
216
+ ok = launch_bgmv_kernel(static_cast<float *>(y.data_ptr()),
217
+ static_cast<nv_bfloat16 *>(x.data_ptr()),
218
+ static_cast<nv_bfloat16 *>(w.data_ptr()),
219
+ indicies.data_ptr<int64_t>(), h_in, h_out, 0,
220
+ h_out, B, num_layers, layer_idx, scale);
221
+ break;
222
+ default:
223
+ break;
224
+ }
225
+ break;
226
+ default:
227
+ break;
228
+ }
229
+ break;
230
+ case at::ScalarType::Float:
231
+ switch (y.scalar_type()) {
232
+ case at::ScalarType::Half:
233
+ switch (w.scalar_type()) {
234
+ case at::ScalarType::Half:
235
+ ok = launch_bgmv_kernel(static_cast<nv_half *>(y.data_ptr()),
236
+ static_cast<float *>(x.data_ptr()),
237
+ static_cast<nv_half *>(w.data_ptr()),
238
+ indicies.data_ptr<int64_t>(), h_in, h_out, 0,
239
+ h_out, B, num_layers, layer_idx, scale);
240
+ break;
241
+ case at::ScalarType::BFloat16:
242
+ ok = launch_bgmv_kernel(static_cast<nv_half *>(y.data_ptr()),
243
+ static_cast<float *>(x.data_ptr()),
244
+ static_cast<nv_bfloat16 *>(w.data_ptr()),
245
+ indicies.data_ptr<int64_t>(), h_in, h_out, 0,
246
+ h_out, B, num_layers, layer_idx, scale);
247
+ break;
248
+ default:
249
+ break;
250
+ }
251
+ break;
252
+ case at::ScalarType::BFloat16:
253
+ switch (w.scalar_type()) {
254
+ case at::ScalarType::Half:
255
+ ok = launch_bgmv_kernel(static_cast<nv_bfloat16 *>(y.data_ptr()),
256
+ static_cast<float *>(x.data_ptr()),
257
+ static_cast<nv_half *>(w.data_ptr()),
258
+ indicies.data_ptr<int64_t>(), h_in, h_out, 0,
259
+ h_out, B, num_layers, layer_idx, scale);
260
+ break;
261
+ case at::ScalarType::BFloat16:
262
+ ok = launch_bgmv_kernel(static_cast<nv_bfloat16 *>(y.data_ptr()),
263
+ static_cast<float *>(x.data_ptr()),
264
+ static_cast<nv_bfloat16 *>(w.data_ptr()),
265
+ indicies.data_ptr<int64_t>(), h_in, h_out, 0,
266
+ h_out, B, num_layers, layer_idx, scale);
267
+ break;
268
+ default:
269
+ break;
270
+ }
271
+ break;
272
+ case at::ScalarType::Float:
273
+ switch (w.scalar_type()) {
274
+ case at::ScalarType::Half:
275
+ ok = launch_bgmv_kernel(static_cast<float *>(y.data_ptr()),
276
+ static_cast<float *>(x.data_ptr()),
277
+ static_cast<nv_half *>(w.data_ptr()),
278
+ indicies.data_ptr<int64_t>(), h_in, h_out, 0,
279
+ h_out, B, num_layers, layer_idx, scale);
280
+ break;
281
+ case at::ScalarType::BFloat16:
282
+ ok = launch_bgmv_kernel(static_cast<float *>(y.data_ptr()),
283
+ static_cast<float *>(x.data_ptr()),
284
+ static_cast<nv_bfloat16 *>(w.data_ptr()),
285
+ indicies.data_ptr<int64_t>(), h_in, h_out, 0,
286
+ h_out, B, num_layers, layer_idx, scale);
287
+ break;
288
+ default:
289
+ break;
290
+ }
291
+ break;
292
+ default:
293
+ break;
294
+ }
295
+ break;
296
+ default:
297
+ break;
298
+ }
299
+ }
300
+ TORCH_CHECK(ok, "No suitable kernel.", " h_in=", h_in, " h_out=", h_out,
301
+ " dtype=", x.scalar_type(), " out_dtype=", y.scalar_type());
302
+}
303
+
304
+void dispatch_bgmv_low_level(torch::Tensor y, torch::Tensor x, torch::Tensor w,
305
+ torch::Tensor indicies, int64_t layer_idx,
306
+ float scale, int64_t h_in, int64_t h_out,
307
+ int64_t y_offset) {
308
+ CHECK_INPUT(y);
309
+ CHECK_INPUT(x);
310
+ CHECK_INPUT(w);
311
+ CHECK_INPUT(indicies);
312
+
313
+ CHECK_DIM(2, y);
314
+ CHECK_DIM(2, x);
315
+ CHECK_DIM(4, w);
316
+ CHECK_DIM(1, indicies);
317
+
318
+ int64_t B = x.size(0);
319
+ int64_t num_layers = w.size(1);
320
+ int64_t full_y_size = y.size(1);
321
+ CHECK_EQ(w.size(3), h_in);
322
+ CHECK_EQ(w.size(2), h_out);
323
+ CHECK_EQ(indicies.size(0), x.size(0));
324
+ CHECK_EQ(y.size(0), x.size(0));
325
+ bool ok = false;
326
+ if (h_in < 65536 && h_out < 65536) {
327
+ // TODO: See if we can get rid of this massive nested switch
328
+ switch (x.scalar_type()) {
329
+ case at::ScalarType::Half:
330
+ switch (y.scalar_type()) {
331
+ case at::ScalarType::Half:
332
+ switch (w.scalar_type()) {
333
+ case at::ScalarType::Half:
334
+ ok = launch_bgmv_kernel(static_cast<nv_half *>(y.data_ptr()),
335
+ static_cast<nv_half *>(x.data_ptr()),
336
+ static_cast<nv_half *>(w.data_ptr()),
337
+ indicies.data_ptr<int64_t>(), h_in, h_out,
338
+ y_offset, full_y_size, B, num_layers,
339
+ layer_idx, scale);
340
+ break;
341
+ case at::ScalarType::BFloat16:
342
+ ok = launch_bgmv_kernel(static_cast<nv_half *>(y.data_ptr()),
343
+ static_cast<nv_half *>(x.data_ptr()),
344
+ static_cast<nv_bfloat16 *>(w.data_ptr()),
345
+ indicies.data_ptr<int64_t>(), h_in, h_out,
346
+ y_offset, full_y_size, B, num_layers,
347
+ layer_idx, scale);
348
+ break;
349
+ default:
350
+ break;
351
+ }
352
+ break;
353
+ case at::ScalarType::BFloat16:
354
+ switch (w.scalar_type()) {
355
+ case at::ScalarType::Half:
356
+ ok = launch_bgmv_kernel(static_cast<nv_bfloat16 *>(y.data_ptr()),
357
+ static_cast<nv_half *>(x.data_ptr()),
358
+ static_cast<nv_half *>(w.data_ptr()),
359
+ indicies.data_ptr<int64_t>(), h_in, h_out,
360
+ y_offset, full_y_size, B, num_layers,
361
+ layer_idx, scale);
362
+ break;
363
+ case at::ScalarType::BFloat16:
364
+ ok = launch_bgmv_kernel(static_cast<nv_bfloat16 *>(y.data_ptr()),
365
+ static_cast<nv_half *>(x.data_ptr()),
366
+ static_cast<nv_bfloat16 *>(w.data_ptr()),
367
+ indicies.data_ptr<int64_t>(), h_in, h_out,
368
+ y_offset, full_y_size, B, num_layers,
369
+ layer_idx, scale);
370
+ break;
371
+ default:
372
+ break;
373
+ }
374
+ break;
375
+ case at::ScalarType::Float:
376
+ switch (w.scalar_type()) {
377
+ case at::ScalarType::Half:
378
+ ok = launch_bgmv_kernel(static_cast<float *>(y.data_ptr()),
379
+ static_cast<nv_half *>(x.data_ptr()),
380
+ static_cast<nv_half *>(w.data_ptr()),
381
+ indicies.data_ptr<int64_t>(), h_in, h_out,
382
+ y_offset, full_y_size, B, num_layers,
383
+ layer_idx, scale);
384
+ break;
385
+ case at::ScalarType::BFloat16:
386
+ ok = launch_bgmv_kernel(static_cast<float *>(y.data_ptr()),
387
+ static_cast<nv_half *>(x.data_ptr()),
388
+ static_cast<nv_bfloat16 *>(w.data_ptr()),
389
+ indicies.data_ptr<int64_t>(), h_in, h_out,
390
+ y_offset, full_y_size, B, num_layers,
391
+ layer_idx, scale);
392
+ break;
393
+ default:
394
+ break;
395
+ }
396
+ break;
397
+ default:
398
+ break;
399
+ }
400
+ break;
401
+ case at::ScalarType::BFloat16:
402
+ switch (y.scalar_type()) {
403
+ case at::ScalarType::Half:
404
+ switch (w.scalar_type()) {
405
+ case at::ScalarType::Half:
406
+ ok = launch_bgmv_kernel(static_cast<nv_half *>(y.data_ptr()),
407
+ static_cast<nv_bfloat16 *>(x.data_ptr()),
408
+ static_cast<nv_half *>(w.data_ptr()),
409
+ indicies.data_ptr<int64_t>(), h_in, h_out,
410
+ y_offset, full_y_size, B, num_layers,
411
+ layer_idx, scale);
412
+ break;
413
+ case at::ScalarType::BFloat16:
414
+ ok = launch_bgmv_kernel(static_cast<nv_half *>(y.data_ptr()),
415
+ static_cast<nv_bfloat16 *>(x.data_ptr()),
416
+ static_cast<nv_bfloat16 *>(w.data_ptr()),
417
+ indicies.data_ptr<int64_t>(), h_in, h_out,
418
+ y_offset, full_y_size, B, num_layers,
419
+ layer_idx, scale);
420
+ break;
421
+ default:
422
+ break;
423
+ }
424
+ break;
425
+ case at::ScalarType::BFloat16:
426
+ switch (w.scalar_type()) {
427
+ case at::ScalarType::Half:
428
+ ok = launch_bgmv_kernel(static_cast<nv_bfloat16 *>(y.data_ptr()),
429
+ static_cast<nv_bfloat16 *>(x.data_ptr()),
430
+ static_cast<nv_half *>(w.data_ptr()),
431
+ indicies.data_ptr<int64_t>(), h_in, h_out,
432
+ y_offset, full_y_size, B, num_layers,
433
+ layer_idx, scale);
434
+ break;
435
+ case at::ScalarType::BFloat16:
436
+ ok = launch_bgmv_kernel(static_cast<nv_bfloat16 *>(y.data_ptr()),
437
+ static_cast<nv_bfloat16 *>(x.data_ptr()),
438
+ static_cast<nv_bfloat16 *>(w.data_ptr()),
439
+ indicies.data_ptr<int64_t>(), h_in, h_out,
440
+ y_offset, full_y_size, B, num_layers,
441
+ layer_idx, scale);
442
+ break;
443
+ default:
444
+ break;
445
+ }
446
+ break;
447
+ case at::ScalarType::Float:
448
+ switch (w.scalar_type()) {
449
+ case at::ScalarType::Half:
450
+ ok = launch_bgmv_kernel(static_cast<float *>(y.data_ptr()),
451
+ static_cast<nv_bfloat16 *>(x.data_ptr()),
452
+ static_cast<nv_half *>(w.data_ptr()),
453
+ indicies.data_ptr<int64_t>(), h_in, h_out,
454
+ y_offset, full_y_size, B, num_layers,
455
+ layer_idx, scale);
456
+ break;
457
+ case at::ScalarType::BFloat16:
458
+ ok = launch_bgmv_kernel(static_cast<float *>(y.data_ptr()),
459
+ static_cast<nv_bfloat16 *>(x.data_ptr()),
460
+ static_cast<nv_bfloat16 *>(w.data_ptr()),
461
+ indicies.data_ptr<int64_t>(), h_in, h_out,
462
+ y_offset, full_y_size, B, num_layers,
463
+ layer_idx, scale);
464
+ break;
465
+ default:
466
+ break;
467
+ }
468
+ break;
469
+ default:
470
+ break;
471
+ }
472
+ break;
473
+ case at::ScalarType::Float:
474
+ switch (y.scalar_type()) {
475
+ case at::ScalarType::Half:
476
+ switch (w.scalar_type()) {
477
+ case at::ScalarType::Half:
478
+ ok = launch_bgmv_kernel(static_cast<nv_half *>(y.data_ptr()),
479
+ static_cast<float *>(x.data_ptr()),
480
+ static_cast<nv_half *>(w.data_ptr()),
481
+ indicies.data_ptr<int64_t>(), h_in, h_out,
482
+ y_offset, full_y_size, B, num_layers,
483
+ layer_idx, scale);
484
+ break;
485
+ case at::ScalarType::BFloat16:
486
+ ok = launch_bgmv_kernel(static_cast<nv_half *>(y.data_ptr()),
487
+ static_cast<float *>(x.data_ptr()),
488
+ static_cast<nv_bfloat16 *>(w.data_ptr()),
489
+ indicies.data_ptr<int64_t>(), h_in, h_out,
490
+ y_offset, full_y_size, B, num_layers,
491
+ layer_idx, scale);
492
+ break;
493
+ default:
494
+ break;
495
+ }
496
+ break;
497
+ case at::ScalarType::BFloat16:
498
+ switch (w.scalar_type()) {
499
+ case at::ScalarType::Half:
500
+ ok = launch_bgmv_kernel(static_cast<nv_bfloat16 *>(y.data_ptr()),
501
+ static_cast<float *>(x.data_ptr()),
502
+ static_cast<nv_half *>(w.data_ptr()),
503
+ indicies.data_ptr<int64_t>(), h_in, h_out,
504
+ y_offset, full_y_size, B, num_layers,
505
+ layer_idx, scale);
506
+ break;
507
+ case at::ScalarType::BFloat16:
508
+ ok = launch_bgmv_kernel(static_cast<nv_bfloat16 *>(y.data_ptr()),
509
+ static_cast<float *>(x.data_ptr()),
510
+ static_cast<nv_bfloat16 *>(w.data_ptr()),
511
+ indicies.data_ptr<int64_t>(), h_in, h_out,
512
+ y_offset, full_y_size, B, num_layers,
513
+ layer_idx, scale);
514
+ break;
515
+ default:
516
+ break;
517
+ }
518
+ break;
519
+ case at::ScalarType::Float:
520
+ switch (w.scalar_type()) {
521
+ case at::ScalarType::Half:
522
+ ok = launch_bgmv_kernel(static_cast<float *>(y.data_ptr()),
523
+ static_cast<float *>(x.data_ptr()),
524
+ static_cast<nv_half *>(w.data_ptr()),
525
+ indicies.data_ptr<int64_t>(), h_in, h_out,
526
+ y_offset, full_y_size, B, num_layers,
527
+ layer_idx, scale);
528
+ break;
529
+ case at::ScalarType::BFloat16:
530
+ ok = launch_bgmv_kernel(static_cast<float *>(y.data_ptr()),
531
+ static_cast<float *>(x.data_ptr()),
532
+ static_cast<nv_bfloat16 *>(w.data_ptr()),
533
+ indicies.data_ptr<int64_t>(), h_in, h_out,
534
+ y_offset, full_y_size, B, num_layers,
535
+ layer_idx, scale);
536
+ break;
537
+ default:
538
+ break;
539
+ }
540
+ break;
541
+ default:
542
+ break;
543
+ }
544
+ break;
545
+ default:
546
+ break;
547
+ }
548
+ }
549
+ TORCH_CHECK(ok, "No suitable kernel.", " h_in=", h_in, " h_out=", h_out,
550
+ " dtype=", x.scalar_type(), " out_dtype=", y.scalar_type());
551
+}
552
+
553
+} // namespace
554
+
555
+//====== pybind ======
556
+
557
+#define DEFINE_pybind(name) m.def(#name, &name, #name);
558
+
559
+PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
560
+ m.def("dispatch_bgmv", &dispatch_bgmv, "dispatch_bgmv");
561
+ m.def("dispatch_bgmv_low_level", &dispatch_bgmv_low_level,
562
+ "dispatch_bgmv_low_level");
563
+}
examples/multilora_inference.py
new
+117
@@ -0,0 +1,117 @@
1
+"""
2
+This example shows how to use the multi-LoRA functionality for offline inference.
3
+
4
+Requires HuggingFace credentials for access to Llama2.
5
+"""
6
+
7
+from typing import Optional, List, Tuple
8
+
9
+from huggingface_hub import snapshot_download
10
+
11
+from vllm import EngineArgs, LLMEngine, SamplingParams, RequestOutput
12
+from vllm.lora.request import LoRARequest
13
+
14
+
15
+def create_test_prompts(lora_path: str) -> List[Tuple[str, SamplingParams]]:
16
+ """Create a list of test prompts with their sampling parameters.
17
+
18
+ 2 requests for base model, 4 requests for the LoRA. We define 2
19
+ different LoRA adapters (using the same model for demo purposes).
20
+ Since we also set `max_loras=1`, the expectation is that the requests
21
+ with the second LoRA adapter will be ran after all requests with the
22
+ first adapter have finished.
23
+ """
24
+ return [
25
+ ("A robot may not injure a human being",
26
+ SamplingParams(temperature=0.0,
27
+ logprobs=1,
28
+ prompt_logprobs=1,
29
+ max_tokens=128), None),
30
+ ("To be or not to be,",
31
+ SamplingParams(temperature=0.8,
32
+ top_k=5,
33
+ presence_penalty=0.2,
34
+ max_tokens=128), None),
35
+ ("[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_74 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]",
36
+ SamplingParams(temperature=0.0,
37
+ logprobs=1,
38
+ prompt_logprobs=1,
39
+ max_tokens=128,
40
+ stop_token_ids=[32003]),
41
+ LoRARequest("sql-lora", 1, lora_path)),
42
+ ("[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_11 (nationality VARCHAR, elector VARCHAR)\n\n question: When Anchero Pantaleone was the elector what is under nationality? [/user] [assistant]",
43
+ SamplingParams(n=3,
44
+ best_of=3,
45
+ use_beam_search=True,
46
+ temperature=0,
47
+ max_tokens=128,
48
+ stop_token_ids=[32003]),
49
+ LoRARequest("sql-lora", 1, lora_path)),
50
+ ("[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_74 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]",
51
+ SamplingParams(temperature=0.0,
52
+ logprobs=1,
53
+ prompt_logprobs=1,
54
+ max_tokens=128,
55
+ stop_token_ids=[32003]),
56
+ LoRARequest("sql-lora2", 2, lora_path)),
57
+ ("[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_11 (nationality VARCHAR, elector VARCHAR)\n\n question: When Anchero Pantaleone was the elector what is under nationality? [/user] [assistant]",
58
+ SamplingParams(n=3,
59
+ best_of=3,
60
+ use_beam_search=True,
61
+ temperature=0,
62
+ max_tokens=128,
63
+ stop_token_ids=[32003]),
64
+ LoRARequest("sql-lora", 1, lora_path)),
65
+ ]
66
+
67
+
68
+def process_requests(engine: LLMEngine,
69
+ test_prompts: List[Tuple[str, SamplingParams,
70
+ Optional[LoRARequest]]]):
71
+ """Continuously process a list of prompts and handle the outputs."""
72
+ request_id = 0
73
+
74
+ while test_prompts or engine.has_unfinished_requests():
75
+ if test_prompts:
76
+ prompt, sampling_params, lora_request = test_prompts.pop(0)
77
+ engine.add_request(str(request_id),
78
+ prompt,
79
+ sampling_params,
80
+ lora_request=lora_request)
81
+ request_id += 1
82
+
83
+ request_outputs: List[RequestOutput] = engine.step()
84
+
85
+ for request_output in request_outputs:
86
+ if request_output.finished:
87
+ print(request_output)
88
+
89
+
90
+def initialize_engine() -> LLMEngine:
91
+ """Initialize the LLMEngine."""
92
+ # max_loras: controls the number of LoRAs that can be used in the same
93
+ # batch. Larger numbers will cause higher memory usage, as each LoRA
94
+ # slot requires its own preallocated tensor.
95
+ # max_lora_rank: controls the maximum supported rank of all LoRAs. Larger
96
+ # numbers will cause higher memory usage. If you know that all LoRAs will
97
+ # use the same rank, it is recommended to set this as low as possible.
98
+ # max_cpu_loras: controls the size of the CPU LoRA cache.
99
+ engine_args = EngineArgs(model="meta-llama/Llama-2-7b-hf",
100
+ enable_lora=True,
101
+ max_loras=1,
102
+ max_lora_rank=8,
103
+ max_cpu_loras=2,
104
+ max_num_seqs=256)
105
+ return LLMEngine.from_engine_args(engine_args)
106
+
107
+
108
+def main():
109
+ """Main function that sets up and runs the prompt processing."""
110
+ engine = initialize_engine()
111
+ lora_path = snapshot_download(repo_id="yard1/llama-2-7b-sql-lora-test")
112
+ test_prompts = create_test_prompts(lora_path)
113
+ process_requests(engine, test_prompts)
114
+
115
+
116
+if __name__ == '__main__':
117
+ main()
setup.py
+54
-5
@@ -1,13 +1,16 @@
1
+import contextlib
2
import io
3
import os
4
import re
5
import subprocess
5
-from typing import List, Set
6
import warnings
7
+from pathlib import Path
8
+from typing import List, Set
9
10
from packaging.version import parse, Version
11
import setuptools
12
import torch
13
+import torch.utils.cpp_extension as torch_cpp_ext
14
from torch.utils.cpp_extension import BuildExtension, CUDAExtension, CUDA_HOME, ROCM_HOME
15
16
ROOT_DIR = os.path.dirname(__file__)
@@ -28,7 +31,7 @@ def _is_neuron() -> bool:
31
torch_neuronx_installed = True
32
try:
33
subprocess.run(["neuron-ls"], capture_output=True, check=True)
31
- except FileNotFoundError as e:
34
+ except FileNotFoundError:
35
torch_neuronx_installed = False
36
return torch_neuronx_installed
37
@@ -96,10 +99,16 @@ def get_hipcc_rocm_version():
99
return None
100
101
102
+def glob(pattern: str):
103
+ root = Path(__name__).parent
104
+ return [str(p) for p in root.glob(pattern)]
105
+
106
+
107
def get_neuronxcc_version():
108
import sysconfig
109
site_dir = sysconfig.get_paths()["purelib"]
102
- version_file = os.path.join(site_dir, "neuronxcc", "version", "__init__.py")
110
+ version_file = os.path.join(site_dir, "neuronxcc", "version",
111
+ "__init__.py")
112
113
# Check if the command was executed successfully
114
with open(version_file, "rt") as fp:
@@ -178,6 +187,8 @@ if _is_cuda() and not compute_capabilities:
187
"GPUs with compute capability below 7.0 are not supported.")
188
compute_capabilities.add(f"{major}.{minor}")
189
190
+ext_modules = []
191
+
192
if _is_cuda():
193
nvcc_cuda_version = get_nvcc_cuda_version(CUDA_HOME)
194
if not compute_capabilities:
@@ -215,6 +226,8 @@ if _is_cuda():
226
raise RuntimeError(
227
"CUDA 11.8 or higher is required for compute capability 9.0.")
228
229
+ NVCC_FLAGS_PUNICA = NVCC_FLAGS.copy()
230
+
231
# Add target compute capabilities to NVCC flags.
232
for capability in compute_capabilities:
233
num = capability[0] + capability[2]
@@ -223,6 +236,14 @@ if _is_cuda():
236
NVCC_FLAGS += [
237
"-gencode", f"arch=compute_{num},code=compute_{num}"
238
]
239
+ if int(capability[0]) >= 8:
240
+ NVCC_FLAGS_PUNICA += [
241
+ "-gencode", f"arch=compute_{num},code=sm_{num}"
242
+ ]
243
+ if capability.endswith("+PTX"):
244
+ NVCC_FLAGS_PUNICA += [
245
+ "-gencode", f"arch=compute_{num},code=compute_{num}"
246
+ ]
247
248
# Use NVCC threads to parallelize the build.
249
if nvcc_cuda_version >= Version("11.2"):
@@ -230,6 +251,36 @@ if _is_cuda():
251
num_threads = min(os.cpu_count(), nvcc_threads)
252
NVCC_FLAGS += ["--threads", str(num_threads)]
253
254
+ # changes for punica kernels
255
+ NVCC_FLAGS += torch_cpp_ext.COMMON_NVCC_FLAGS
256
+ REMOVE_NVCC_FLAGS = [
257
+ '-D__CUDA_NO_HALF_OPERATORS__',
258
+ '-D__CUDA_NO_HALF_CONVERSIONS__',
259
+ '-D__CUDA_NO_BFLOAT16_CONVERSIONS__',
260
+ '-D__CUDA_NO_HALF2_OPERATORS__',
261
+ ]
262
+ for flag in REMOVE_NVCC_FLAGS:
263
+ with contextlib.suppress(ValueError):
264
+ torch_cpp_ext.COMMON_NVCC_FLAGS.remove(flag)
265
+
266
+ install_punica = bool(int(os.getenv("VLLM_INSTALL_PUNICA_KERNELS", "1")))
267
+ device_count = torch.cuda.device_count()
268
+ for i in range(device_count):
269
+ major, minor = torch.cuda.get_device_capability(i)
270
+ if major < 8:
271
+ install_punica = False
272
+ break
273
+ if install_punica:
274
+ ext_modules.append(
275
+ CUDAExtension(
276
+ name="vllm._punica_C",
277
+ sources=["csrc/punica/punica_ops.cc"] +
278
+ glob("csrc/punica/bgmv/*.cu"),
279
+ extra_compile_args={
280
+ "cxx": CXX_FLAGS,
281
+ "nvcc": NVCC_FLAGS_PUNICA,
282
+ },
283
+ ))
284
elif _is_hip():
285
amd_arch = get_amdgpu_offload_arch()
286
if amd_arch not in ROCM_SUPPORTED_ARCHS:
@@ -240,8 +291,6 @@ elif _is_hip():
291
elif _is_neuron():
292
neuronxcc_version = get_neuronxcc_version()
293
243
-ext_modules = []
244
-
294
vllm_extension_sources = [
295
"csrc/cache_kernels.cu",
296
"csrc/attention/attention_kernels.cu",
tests/async_engine/test_async_llm_engine.py
+11
@@ -25,6 +25,13 @@ class MockEngine:
25
return [RequestOutput(
26
request_id=self.request_id)] if self.request_id else []
27
28
+ async def encode_request_async(
29
+ self,
30
+ *args,
31
+ **kwargs,
32
+ ):
33
+ return [1]
34
+
35
def generate(self, request_id):
36
self.request_id = request_id
37
@@ -35,6 +42,10 @@ class MockEngine:
42
del kwargs # Unused
43
self.add_request_calls += 1
44
45
+ async def add_request_async(self, **kwargs):
46
+ del kwargs # Unused
47
+ self.add_request_calls += 1
48
+
49
def abort_request(self, request_id):
50
del request_id # Unused
51
self.abort_request_calls += 1
tests/lora/__init__.py
tests/lora/conftest.py
new
+143
@@ -0,0 +1,143 @@
1
+import contextlib
2
+import gc
3
+import tempfile
4
+from collections import OrderedDict
5
+from unittest.mock import patch, MagicMock
6
+
7
+import pytest
8
+import ray
9
+import torch
10
+import torch.nn as nn
11
+from huggingface_hub import snapshot_download
12
+
13
+import vllm
14
+from vllm.config import LoRAConfig
15
+from vllm.model_executor.layers.sampler import Sampler
16
+from vllm.model_executor.model_loader import get_model
17
+from vllm.model_executor.layers.linear import (ColumnParallelLinear,
18
+ MergedColumnParallelLinear,
19
+ RowParallelLinear)
20
+from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
21
+from vllm.model_executor.parallel_utils.parallel_state import (
22
+ destroy_model_parallel, initialize_model_parallel)
23
+
24
+
25
+def cleanup():
26
+ destroy_model_parallel()
27
+ with contextlib.suppress(AssertionError):
28
+ torch.distributed.destroy_process_group()
29
+ gc.collect()
30
+ torch.cuda.empty_cache()
31
+ ray.shutdown()
32
+
33
+
34
+@pytest.fixture(autouse=True)
35
+def cleanup_fixture():
36
+ yield
37
+ cleanup()
38
+
39
+
40
+@pytest.fixture
41
+def dist_init():
42
+ if not torch.distributed.is_initialized():
43
+ temp_file = tempfile.mkstemp()[1]
44
+ torch.distributed.init_process_group(
45
+ backend="nccl",
46
+ world_size=1,
47
+ rank=0,
48
+ init_method=f"file://{temp_file}",
49
+ )
50
+ torch.distributed.all_reduce(torch.zeros(1).cuda())
51
+ initialize_model_parallel(1, 1)
52
+ yield
53
+ cleanup()
54
+
55
+
56
+@pytest.fixture
57
+def dist_init_torch_only():
58
+ if torch.distributed.is_initialized():
59
+ return
60
+ temp_file = tempfile.mkstemp()[1]
61
+ torch.distributed.init_process_group(
62
+ backend="nccl",
63
+ world_size=1,
64
+ rank=0,
65
+ init_method=f"file://{temp_file}",
66
+ )
67
+
68
+
69
+@pytest.fixture
70
+def dummy_model() -> nn.Module:
71
+ model = nn.Sequential(
72
+ OrderedDict([
73
+ ("dense1", ColumnParallelLinear(764, 100)),
74
+ ("dense2", RowParallelLinear(100, 50)),
75
+ (
76
+ "layer1",
77
+ nn.Sequential(
78
+ OrderedDict([
79
+ ("dense1", ColumnParallelLinear(100, 10)),
80
+ ("dense2", RowParallelLinear(10, 50)),
81
+ ])),
82
+ ),
83
+ ("act2", nn.ReLU()),
84
+ ("output", ColumnParallelLinear(50, 10)),
85
+ ("outact", nn.Sigmoid()),
86
+ # Special handling for lm_head & sampler
87
+ ("lm_head", ParallelLMHead(512, 10)),
88
+ ("sampler", Sampler(512))
89
+ ]))
90
+ model.config = MagicMock()
91
+ return model
92
+
93
+
94
+@pytest.fixture
95
+def dummy_model_gate_up() -> nn.Module:
96
+ model = nn.Sequential(
97
+ OrderedDict([
98
+ ("dense1", ColumnParallelLinear(764, 100)),
99
+ ("dense2", RowParallelLinear(100, 50)),
100
+ (
101
+ "layer1",
102
+ nn.Sequential(
103
+ OrderedDict([
104
+ ("dense1", ColumnParallelLinear(100, 10)),
105
+ ("dense2", RowParallelLinear(10, 50)),
106
+ ])),
107
+ ),
108
+ ("act2", nn.ReLU()),
109
+ ("gate_up_proj", MergedColumnParallelLinear(50, [5, 5])),
110
+ ("outact", nn.Sigmoid()),
111
+ # Special handling for lm_head & sampler
112
+ ("lm_head", ParallelLMHead(512, 10)),
113
+ ("sampler", Sampler(512))
114
+ ]))
115
+ model.config = MagicMock()
116
+ return model
117
+
118
+
119
+@pytest.fixture(scope="session")
120
+def sql_lora_files():
121
+ return snapshot_download(repo_id="yard1/llama-2-7b-sql-lora-test")
122
+
123
+
124
+@pytest.fixture
125
+def llama_2_7b_engine_extra_embeddings() -> nn.Module:
126
+ cleanup()
127
+ get_model_old = get_model
128
+
129
+ def get_model_patched(model_config, lora_config=None):
130
+ return get_model_old(model_config,
131
+ LoRAConfig(max_loras=4, max_lora_rank=8))
132
+
133
+ with patch("vllm.worker.model_runner.get_model", get_model_patched):
134
+ engine = vllm.LLM("meta-llama/Llama-2-7b-hf", enable_lora=False)
135
+ yield engine.llm_engine
136
+ del engine
137
+ cleanup()
138
+
139
+
140
+@pytest.fixture
141
+def llama_2_7b_model_extra_embeddings(
142
+ llama_2_7b_engine_extra_embeddings) -> nn.Module:
143
+ yield llama_2_7b_engine_extra_embeddings.driver_worker.model_runner.model
tests/lora/test_layers.py
new
+709
@@ -0,0 +1,709 @@
1
+import pytest
2
+import random
3
+from copy import deepcopy
4
+from dataclasses import dataclass
5
+from typing import List, Optional, Dict, Tuple
6
+
7
+import torch
8
+import torch.nn.functional as F
9
+
10
+from vllm.lora.layers import (
11
+ ColumnParallelLinearWithLoRA,
12
+ MergedColumnParallelLinearWithLoRA,
13
+ QKVParallelLinearWithLora,
14
+ VocabParallelEmbeddingWithLoRA,
15
+ RowParallelLinearWithLoRA,
16
+ SamplerWithLoRA,
17
+ LoRAMapping,
18
+ BaseLayerWithLoRA,
19
+)
20
+from vllm.lora.models import LoRALayerWeights, convert_mapping, PackedLoRALayerWeights
21
+from vllm.config import LoRAConfig
22
+from vllm.model_executor.layers.sampler import Sampler
23
+from vllm.model_executor.layers.linear import (ColumnParallelLinear,
24
+ MergedColumnParallelLinear,
25
+ RowParallelLinear,
26
+ QKVParallelLinear)
27
+from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding, ParallelLMHead
28
+from vllm.model_executor.utils import set_random_seed
29
+
30
+from .utils import DummyLoRAManager
31
+
32
+TOLERANCES = {
33
+ torch.float16: (5e-3, 5e-3),
34
+ torch.float32: (5e-3, 5e-3),
35
+ torch.bfloat16: (3e-2, 2e-2),
36
+}
37
+
38
+
39
+def get_random_id_to_index(num_loras: int,
40
+ num_slots: int,
41
+ log: bool = True) -> List[Optional[int]]:
42
+ """Creates a random lora_id_to_index mapping.
43
+
44
+ Args:
45
+ num_loras: The number of active loras in the mapping.
46
+ num_slots: The number of slots in the mapping. Must be larger
47
+ than num_loras.
48
+ log: Whether to log the output.
49
+ """
50
+
51
+ if num_loras > num_slots:
52
+ raise ValueError(
53
+ f"num_loras is higher than num_slots: {num_loras} > {num_slots}. "
54
+ "num_loras must be less than or equal to num_slots.")
55
+
56
+ slots: List[Optional[int]] = [None] * num_slots
57
+ random_slot_selections = (torch.randperm(num_slots)[:num_loras]).tolist()
58
+ for lora_id, slot_idx in enumerate(random_slot_selections, start=1):
59
+ slots[slot_idx] = lora_id
60
+
61
+ if log:
62
+ print(f"Created lora_id_to_index mapping: {slots}.")
63
+
64
+ return slots
65
+
66
+
67
+def populate_loras(
68
+ id_to_index: List[Optional[int]],
69
+ layer: BaseLayerWithLoRA,
70
+ layer_weights: torch.Tensor,
71
+ generate_embeddings_tensor: int = 0,
72
+ repeats: int = 1,
73
+) -> Tuple[Dict[int, LoRALayerWeights], Dict[int, List[LoRALayerWeights]]]:
74
+ """This method populates the lora layers with lora weights.
75
+
76
+ Args:
77
+ id_to_index: a list of lora ids. The index of the lora id
78
+ represents which memory slot the lora matrices are
79
+ stored in. A None value indicates a free slot.
80
+ layer: the LoRAlayer to populate.
81
+ layer_weights: the PyTorch tensor containing the layer's
82
+ weights.
83
+ generate_embeddings_tensor: whether to generate an
84
+ embeddings tensor for each LoRA.
85
+ repeats: must only be set for column parallel packed
86
+ layers. Indicates the number of loras to compose
87
+ together to create a single lora layer.
88
+ """
89
+
90
+ # Dictionary that maps the lora ID to the
91
+ # corresponding lora weights.
92
+ lora_dict: Dict[int, LoRALayerWeights] = dict()
93
+
94
+ # Dictionary that maps the lora ID to the
95
+ # corresponding subloras. Only useful when
96
+ # repeats > 1.
97
+ sublora_dict: Dict[int, List[LoRALayerWeights]] = dict()
98
+
99
+ for slot_idx, lora_id in enumerate(id_to_index):
100
+ if lora_id is not None:
101
+ subloras = []
102
+ sublora_len = layer_weights.shape[0] // repeats
103
+ for i in range(repeats):
104
+ sublora = DummyLoRAManager().init_random_lora(
105
+ module_name=f"fake_{i}",
106
+ weight=layer_weights,
107
+ generate_embeddings_tensor=generate_embeddings_tensor,
108
+ )
109
+ sublora.lora_b = sublora.lora_b[:, (sublora_len *
110
+ i):(sublora_len * (i + 1))]
111
+ sublora.optimize()
112
+ subloras.append(sublora)
113
+
114
+ lora = PackedLoRALayerWeights.pack(
115
+ subloras) if repeats > 1 else subloras[0]
116
+
117
+ layer.set_lora(
118
+ slot_idx,
119
+ lora_a=lora.lora_a,
120
+ lora_b=lora.lora_b,
121
+ embeddings_tensor=lora.embeddings_tensor,
122
+ )
123
+
124
+ lora_dict[lora_id] = lora
125
+ sublora_dict[lora_id] = subloras
126
+
127
+ return lora_dict, sublora_dict
128
+
129
+
130
+def create_random_inputs(
131
+ active_lora_ids: List[int],
132
+ num_inputs: int,
133
+ input_size: Tuple[int, ...],
134
+ input_range: Tuple[float, float],
135
+ input_type: torch.dtype = torch.int,
136
+) -> Tuple[List[torch.Tensor], List[int], List[int]]:
137
+ """Creates random inputs.
138
+
139
+ Args:
140
+ active_lora_ids: lora IDs of active lora weights.
141
+ num_inputs: the number of inputs to create.
142
+ input_size: the size of each individual input.
143
+ input_range: the range of values to include in the input.
144
+ input_range[0] <= possible input values < input_range[1]
145
+ input_type: the type of values in the input.
146
+ """
147
+
148
+ low, high = input_range
149
+
150
+ inputs, index_mapping, prompt_mapping = [], [], []
151
+ for _ in range(num_inputs):
152
+ if input_type == torch.int:
153
+ inputs.append(
154
+ torch.randint(low=int(low),
155
+ high=int(high),
156
+ size=input_size,
157
+ device="cuda"))
158
+ else:
159
+ inputs.append(
160
+ torch.rand(size=input_size, dtype=input_type, device="cuda") *
161
+ high + low)
162
+
163
+ lora_id = random.choice(active_lora_ids)
164
+ index_mapping += [lora_id] * input_size[0]
165
+ prompt_mapping += [lora_id]
166
+
167
+ return inputs, index_mapping, prompt_mapping
168
+
169
+
170
+@torch.inference_mode()
171
+@pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
172
+def test_embeddings(dist_init, num_loras) -> None:
173
+
174
+ max_loras = 8
175
+ lora_config = LoRAConfig(max_loras=max_loras,
176
+ max_lora_rank=8,
177
+ lora_dtype=torch.float16)
178
+
179
+ def create_random_embedding_layer():
180
+ embedding = VocabParallelEmbedding(512, 256)
181
+ embedding.weight.data = torch.rand_like(embedding.weight.data)
182
+ embedding.weight.data[512:, :] = 0
183
+ lora_embedding = VocabParallelEmbeddingWithLoRA(embedding)
184
+ lora_embedding.create_lora_weights(max_loras, lora_config)
185
+
186
+ return embedding, lora_embedding
187
+
188
+ for i in range(10):
189
+ set_random_seed(i)
190
+
191
+ id_to_index = get_random_id_to_index(num_loras, max_loras)
192
+ embedding, lora_embedding = create_random_embedding_layer()
193
+
194
+ lora_dict, _ = populate_loras(
195
+ id_to_index,
196
+ layer=lora_embedding,
197
+ layer_weights=embedding.weight.T,
198
+ )
199
+
200
+ inputs, index_mapping, prompt_mapping = create_random_inputs(
201
+ active_lora_ids=list(lora_dict.keys()),
202
+ num_inputs=num_loras * 3,
203
+ input_size=(200, ),
204
+ input_range=(1, 512),
205
+ )
206
+ lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
207
+
208
+ mapping_info = convert_mapping(lora_mapping, id_to_index, max_loras,
209
+ 512, lora_config.lora_extra_vocab_size)
210
+ lora_embedding.set_mapping(*mapping_info)
211
+
212
+ lora_result = lora_embedding(torch.cat(inputs))
213
+
214
+ expected_results = []
215
+ for input_, lora_id in zip(inputs, prompt_mapping):
216
+ lora = lora_dict[lora_id]
217
+ result = embedding(input_)
218
+ after_a = F.embedding(
219
+ input_,
220
+ lora.lora_a,
221
+ )
222
+ result += (after_a @ lora.lora_b)
223
+ expected_results.append(result)
224
+ expected_result = torch.cat(expected_results)
225
+
226
+ rtol, atol = TOLERANCES[lora_result.dtype]
227
+ assert torch.allclose(lora_result,
228
+ expected_result,
229
+ rtol=rtol,
230
+ atol=atol)
231
+
232
+ # Check that resetting the lora weights succeeds
233
+
234
+ for slot_idx in range(max_loras):
235
+ lora_embedding.reset_lora(slot_idx)
236
+
237
+ inputs, index_mapping, prompt_mapping = create_random_inputs(
238
+ active_lora_ids=[0],
239
+ num_inputs=num_loras * 3,
240
+ input_size=(200, ),
241
+ input_range=(1, 512),
242
+ )
243
+ lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
244
+
245
+ mapping_info = convert_mapping(lora_mapping, id_to_index, max_loras,
246
+ 512, lora_config.lora_extra_vocab_size)
247
+ lora_embedding.set_mapping(*mapping_info, )
248
+
249
+ lora_result = lora_embedding(torch.cat(inputs))
250
+ expected_result = embedding(torch.cat(inputs))
251
+
252
+ rtol, atol = TOLERANCES[lora_result.dtype]
253
+ assert torch.allclose(lora_result,
254
+ expected_result,
255
+ rtol=rtol,
256
+ atol=atol)
257
+
258
+
259
+@torch.inference_mode()
260
+# @pytest.mark.skip(reason="Fails when loras are in any slot other than the first.")
261
+@pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
262
+def test_embeddings_with_new_embeddings(dist_init, num_loras) -> None:
263
+
264
+ max_loras = 8
265
+ lora_config = LoRAConfig(max_loras=max_loras,
266
+ max_lora_rank=8,
267
+ lora_dtype=torch.float16)
268
+
269
+ def create_random_embedding_layer():
270
+ embedding = VocabParallelEmbedding(512, 256)
271
+ embedding_data = torch.rand_like(embedding.weight.data)
272
+ embedding.weight.data = embedding_data
273
+ embedding.weight.data[512:, :] = 0
274
+ expanded_embedding = VocabParallelEmbedding(
275
+ 512 + lora_config.lora_extra_vocab_size * max_loras,
276
+ 256,
277
+ org_num_embeddings=512)
278
+ expanded_embedding.weight.data[:512, :] = embedding_data
279
+ # We need to deepcopy the embedding as it will be modifed
280
+ # in place
281
+ lora_embedding = VocabParallelEmbeddingWithLoRA(
282
+ deepcopy(expanded_embedding))
283
+ lora_embedding.create_lora_weights(max_loras, lora_config)
284
+
285
+ return expanded_embedding, lora_embedding
286
+
287
+ for i in range(10):
288
+ set_random_seed(i)
289
+
290
+ id_to_index = get_random_id_to_index(num_loras, max_loras)
291
+ expanded_embedding, lora_embedding = create_random_embedding_layer()
292
+ lora_dict, _ = populate_loras(
293
+ id_to_index,
294
+ layer=lora_embedding,
295
+ layer_weights=torch.zeros(
296
+ (256, 512 + lora_config.lora_extra_vocab_size)),
297
+ generate_embeddings_tensor=256,
298
+ )
299
+
300
+ # All embeddings tensors have the same shape.
301
+ embeddings_tensors = [
302
+ lora_dict[id].embeddings_tensor for id in sorted(lora_dict.keys())
303
+ ]
304
+ embeddings_tensor_len = embeddings_tensors[0].shape[0]
305
+
306
+ # Add empty embeddings_tensors for unoccupied lora slots.
307
+ for _ in range(max_loras - len(embeddings_tensors)):
308
+ embeddings_tensors.append(
309
+ torch.zeros(embeddings_tensors[0].shape, device="cuda"))
310
+
311
+ inputs, index_mapping, prompt_mapping = create_random_inputs(
312
+ active_lora_ids=list(lora_dict.keys()),
313
+ num_inputs=num_loras * 3,
314
+ input_size=(200, ),
315
+ input_range=(1, 512),
316
+ )
317
+ lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
318
+
319
+ original_inputs = deepcopy(inputs)
320
+
321
+ # Force some of the inputs to be in the extended embeddings range
322
+ # to guarantee that their behavior is tested.
323
+ for input_, original_input_, lora_id in zip(inputs, original_inputs,
324
+ prompt_mapping):
325
+ embedding_id = lora_id - 1
326
+ input_[-1] = 512 + (embedding_id * embeddings_tensor_len)
327
+ original_input_[-1] = 512
328
+ input_[-2] = 512 + ((embedding_id + 1) * embeddings_tensor_len - 1)
329
+ original_input_[-2] = 512 + embeddings_tensor_len - 1
330
+
331
+ mapping_info = convert_mapping(lora_mapping, id_to_index, max_loras,
332
+ 512, lora_config.lora_extra_vocab_size)
333
+ lora_embedding.set_mapping(*mapping_info, )
334
+
335
+ expanded_embedding.weight[512:512 +
336
+ (embeddings_tensor_len *
337
+ max_loras)] = torch.cat(embeddings_tensors)
338
+
339
+ lora_result = lora_embedding(torch.cat(original_inputs))
340
+
341
+ expected_results = []
342
+ for input_, original_input_, lora_id in zip(inputs, original_inputs,
343
+ prompt_mapping):
344
+ lora = lora_dict[lora_id]
345
+ result = expanded_embedding(input_)
346
+ after_a = F.embedding(
347
+ original_input_,
348
+ lora.lora_a,
349
+ )
350
+ result += (after_a @ lora.lora_b)
351
+ expected_results.append(result)
352
+ expected_result = torch.cat(expected_results)
353
+
354
+ rtol, atol = TOLERANCES[lora_result.dtype]
355
+ assert torch.allclose(lora_result,
356
+ expected_result,
357
+ rtol=rtol,
358
+ atol=atol)
359
+
360
+ # Check that resetting the lora weights succeeds
361
+
362
+ for slot_idx in range(max_loras):
363
+ lora_embedding.reset_lora(slot_idx)
364
+
365
+ inputs, index_mapping, prompt_mapping = create_random_inputs(
366
+ active_lora_ids=[0],
367
+ num_inputs=num_loras * 3,
368
+ input_size=(200, ),
369
+ input_range=(1, 512),
370
+ )
371
+ lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
372
+
373
+ original_inputs = deepcopy(inputs)
374
+
375
+ mapping_info = convert_mapping(lora_mapping, id_to_index, max_loras,
376
+ 512, lora_config.lora_extra_vocab_size)
377
+ lora_embedding.set_mapping(*mapping_info, )
378
+
379
+ lora_result = lora_embedding(torch.cat(original_inputs))
380
+ expected_result = expanded_embedding(torch.cat(inputs))
381
+
382
+ rtol, atol = TOLERANCES[lora_result.dtype]
383
+ assert torch.allclose(lora_result,
384
+ expected_result,
385
+ rtol=rtol,
386
+ atol=atol)
387
+
388
+
389
+@torch.inference_mode()
390
+@pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
391
+def test_lm_head_sampler(dist_init, num_loras) -> None:
392
+
393
+ max_loras = 8
394
+ lora_config = LoRAConfig(max_loras=max_loras,
395
+ max_lora_rank=8,
396
+ lora_dtype=torch.float16)
397
+
398
+ def create_random_sampler_layer():
399
+ linear = ParallelLMHead(32000 + lora_config.lora_extra_vocab_size,
400
+ 1024, 32000)
401
+ linear.weight.data = torch.rand_like(linear.weight.data)
402
+ linear.weight.data[:, 32000:] = 0
403
+ sampler = Sampler(32000 + lora_config.lora_extra_vocab_size, 32000)
404
+ lora_sampler = SamplerWithLoRA(sampler, 1024, linear.weight.dtype,
405
+ linear.weight.device)
406
+ lora_sampler.create_lora_weights(max_loras, lora_config)
407
+
408
+ return linear, sampler, lora_sampler
409
+
410
+ for i in range(10):
411
+ set_random_seed(i)
412
+
413
+ id_to_index = get_random_id_to_index(num_loras, max_loras)
414
+ linear, sampler, lora_sampler = create_random_sampler_layer()
415
+
416
+ # NOTE: all the generated loras share the same embeddings tensor.
417
+ lora_dict, _ = populate_loras(
418
+ id_to_index,
419
+ layer=lora_sampler,
420
+ layer_weights=linear.weight,
421
+ generate_embeddings_tensor=1024,
422
+ )
423
+ embeddings_tensor = list(lora_dict.values())[0].embeddings_tensor
424
+ embeddings_tensor_len = embeddings_tensor.shape[0]
425
+
426
+ inputs, index_mapping, prompt_mapping = create_random_inputs(
427
+ active_lora_ids=list(lora_dict.keys()),
428
+ num_inputs=8 * num_loras, # * 3,
429
+ input_size=(1, 1024),
430
+ input_range=(0, 1),
431
+ input_type=torch.float32,
432
+ )
433
+ lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
434
+
435
+ input_ = torch.rand(20, 1024, device="cuda")
436
+ mapping_info = convert_mapping(
437
+ lora_mapping,
438
+ id_to_index,
439
+ max_loras,
440
+ 32000,
441
+ lora_config.lora_extra_vocab_size,
442
+ )
443
+ lora_sampler.set_mapping(*mapping_info, )
444
+
445
+ lora_result = lora_sampler._get_logits(hidden_states=torch.cat(inputs),
446
+ embedding=linear.weight,
447
+ embedding_bias=None)
448
+
449
+ original_weight = linear.weight.clone()
450
+
451
+ linear.weight[sampler.org_vocab_size:sampler.org_vocab_size +
452
+ embeddings_tensor_len] = embeddings_tensor
453
+
454
+ sampler.org_vocab_size = 32000 + lora_config.lora_extra_vocab_size
455
+ expected_results = []
456
+ for input_, lora_id in zip(inputs, prompt_mapping):
457
+ lora = lora_dict[lora_id]
458
+ result = sampler._get_logits(hidden_states=input_,
459
+ embedding=linear.weight,
460
+ embedding_bias=None)
461
+ result[:, 32000 + embeddings_tensor_len:] = float("-inf")
462
+ result += input_ @ lora.lora_a @ lora.lora_b * lora.scaling
463
+ expected_results.append(result)
464
+ expected_result = torch.cat(expected_results)
465
+ sampler.org_vocab_size = 32000
466
+
467
+ # Check that resetting the lora weights succeeds
468
+
469
+ for slot_idx in range(max_loras):
470
+ lora_sampler.reset_lora(slot_idx)
471
+
472
+ inputs, index_mapping, prompt_mapping = create_random_inputs(
473
+ active_lora_ids=[0],
474
+ num_inputs=8 * num_loras * 3,
475
+ input_size=(1, 1024),
476
+ input_range=(0, 1),
477
+ input_type=torch.float32,
478
+ )
479
+ lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
480
+
481
+ mapping_info = convert_mapping(lora_mapping, id_to_index, max_loras,
482
+ 32000,
483
+ lora_config.lora_extra_vocab_size)
484
+ lora_sampler.set_mapping(*mapping_info, )
485
+
486
+ lora_result = lora_sampler._get_logits(hidden_states=torch.cat(inputs),
487
+ embedding=original_weight,
488
+ embedding_bias=None)[:, :32000]
489
+ expected_result = sampler._get_logits(hidden_states=torch.cat(inputs),
490
+ embedding=original_weight,
491
+ embedding_bias=None)
492
+
493
+ rtol, atol = TOLERANCES[lora_result.dtype]
494
+ assert torch.allclose(lora_result,
495
+ expected_result,
496
+ rtol=rtol,
497
+ atol=atol)
498
+
499
+
500
+@torch.inference_mode()
501
+@pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
502
+@pytest.mark.parametrize("orientation", ["row", "column"])
503
+def test_linear_parallel(dist_init, num_loras, orientation) -> None:
504
+
505
+ max_loras = 8
506
+ lora_config = LoRAConfig(max_loras=max_loras,
507
+ max_lora_rank=8,
508
+ lora_dtype=torch.float16)
509
+
510
+ def create_random_linear_parallel_layer():
511
+ if orientation == "row":
512
+ linear = RowParallelLinear(4096, 4096, bias=False)
513
+ linear.weight.data = torch.rand_like(linear.weight.data)
514
+ lora_linear = RowParallelLinearWithLoRA(linear)
515
+ else:
516
+ linear = ColumnParallelLinear(4096, 4096, bias=False)
517
+ linear.weight.data = torch.rand_like(linear.weight.data)
518
+ lora_linear = ColumnParallelLinearWithLoRA(linear)
519
+ lora_linear.create_lora_weights(max_loras, lora_config)
520
+
521
+ return linear, lora_linear
522
+
523
+ for i in range(10):
524
+ set_random_seed(i)
525
+
526
+ id_to_index = get_random_id_to_index(num_loras, max_loras)
527
+ linear, lora_linear = create_random_linear_parallel_layer()
528
+
529
+ lora_dict, _ = populate_loras(
530
+ id_to_index,
531
+ layer=lora_linear,
532
+ layer_weights=linear.weight,
533
+ )
534
+
535
+ inputs, index_mapping, prompt_mapping = create_random_inputs(
536
+ active_lora_ids=list(lora_dict.keys()),
537
+ num_inputs=32 * num_loras,
538
+ input_size=(1, 4096),
539
+ input_range=(0, 1),
540
+ input_type=torch.float32,
541
+ )
542
+ lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
543
+
544
+ mapping_info = convert_mapping(
545
+ lora_mapping,
546
+ id_to_index,
547
+ max_loras,
548
+ 512,
549
+ lora_config.lora_extra_vocab_size,
550
+ )
551
+ lora_linear.set_mapping(*mapping_info, )
552
+
553
+ lora_result = lora_linear(torch.cat(inputs))[0]
554
+
555
+ expected_results = []
556
+ for input_, lora_id in zip(inputs, prompt_mapping):
557
+ lora = lora_dict[lora_id]
558
+ result = linear(input_)[0]
559
+ result += input_ @ lora.lora_a @ lora.lora_b * lora.scaling
560
+ expected_results.append(result)
561
+ expected_result = torch.cat(expected_results)
562
+
563
+ rtol, atol = TOLERANCES[lora_result.dtype]
564
+ assert torch.allclose(lora_result,
565
+ expected_result,
566
+ rtol=rtol,
567
+ atol=atol)
568
+
569
+ # Check that resetting the lora weights succeeds
570
+
571
+ for slot_idx in range(max_loras):
572
+ lora_linear.reset_lora(slot_idx)
573
+
574
+ inputs, index_mapping, prompt_mapping = create_random_inputs(
575
+ active_lora_ids=[0],
576
+ num_inputs=32 * num_loras,
577
+ input_size=(1, 4096),
578
+ input_range=(0, 1),
579
+ input_type=torch.float32,
580
+ )
581
+ lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
582
+
583
+ mapping_info = convert_mapping(lora_mapping, id_to_index, max_loras,
584
+ 512, lora_config.lora_extra_vocab_size)
585
+ lora_linear.set_mapping(*mapping_info, )
586
+
587
+ lora_result = lora_linear(torch.cat(inputs))[0]
588
+ expected_result = linear(torch.cat(inputs))[0]
589
+
590
+ rtol, atol = TOLERANCES[lora_result.dtype]
591
+ assert torch.allclose(lora_result,
592
+ expected_result,
593
+ rtol=rtol,
594
+ atol=atol)
595
+
596
+
597
+@torch.inference_mode()
598
+@pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
599
+@pytest.mark.parametrize("repeats", [2, 3])
600
+def test_column_parallel_packed(dist_init, num_loras, repeats) -> None:
601
+
602
+ max_loras = 8
603
+ lora_config = LoRAConfig(max_loras=max_loras,
604
+ max_lora_rank=8,
605
+ lora_dtype=torch.float16)
606
+
607
+ def create_column_parallel_packed_layer():
608
+ if repeats == 2:
609
+ linear = MergedColumnParallelLinear(4096, [4096] * repeats,
610
+ bias=False)
611
+ linear.weight.data = torch.rand_like(linear.weight.data)
612
+ lora_linear = MergedColumnParallelLinearWithLoRA(linear)
613
+ else:
614
+ linear = QKVParallelLinear(4096, 64, 32, bias=False)
615
+ linear.weight.data = torch.rand_like(linear.weight.data)
616
+ lora_linear = QKVParallelLinearWithLora(linear)
617
+
618
+ @dataclass
619
+ class FakeConfig:
620
+ hidden_size = 4096
621
+ num_key_value_heads = 32
622
+ num_attention_heads = 32
623
+
624
+ lora_linear.create_lora_weights(max_loras,
625
+ lora_config,
626
+ model_config=FakeConfig())
627
+
628
+ return linear, lora_linear
629
+
630
+ for i in range(10):
631
+ set_random_seed(i)
632
+
633
+ id_to_index = get_random_id_to_index(num_loras, max_loras)
634
+
635
+ linear, lora_linear = create_column_parallel_packed_layer()
636
+
637
+ lora_dict, sublora_dict = populate_loras(
638
+ id_to_index,
639
+ layer=lora_linear,
640
+ layer_weights=linear.weight,
641
+ repeats=repeats,
642
+ )
643
+
644
+ inputs, index_mapping, prompt_mapping = create_random_inputs(
645
+ active_lora_ids=list(lora_dict.keys()),
646
+ num_inputs=32 * num_loras,
647
+ input_size=(1, 4096),
648
+ input_range=(0, 1),
649
+ input_type=torch.float32,
650
+ )
651
+ lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
652
+
653
+ mapping_info = convert_mapping(
654
+ lora_mapping,
655
+ id_to_index,
656
+ max_loras,
657
+ 512,
658
+ lora_config.lora_extra_vocab_size,
659
+ )
660
+ lora_linear.set_mapping(*mapping_info)
661
+
662
+ lora_result = lora_linear(torch.cat(inputs))[0]
663
+
664
+ expected_results = []
665
+ for input_, lora_id in zip(inputs, prompt_mapping):
666
+ result = linear(input_)[0]
667
+ subloras = sublora_dict[lora_id]
668
+ for i, sublora in enumerate(subloras):
669
+ result[:, sublora.lora_b.shape[1] * i:sublora.lora_b.shape[1] * (
670
+ i + 1
671
+ )] += input_ @ sublora.lora_a @ sublora.lora_b * sublora.scaling
672
+ expected_results.append(result)
673
+ expected_result = torch.cat(expected_results)
674
+
675
+ rtol, atol = TOLERANCES[lora_result.dtype]
676
+ assert torch.allclose(lora_result,
677
+ expected_result,
678
+ rtol=rtol,
679
+ atol=atol)
680
+
681
+ for slot_idx in range(max_loras):
682
+ lora_linear.reset_lora(slot_idx)
683
+
684
+ inputs, index_mapping, prompt_mapping = create_random_inputs(
685
+ active_lora_ids=[0],
686
+ num_inputs=32 * num_loras,
687
+ input_size=(1, 4096),
688
+ input_range=(0, 1),
689
+ input_type=torch.float32,
690
+ )
691
+ lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
692
+
693
+ mapping_info = convert_mapping(
694
+ lora_mapping,
695
+ id_to_index,
696
+ max_loras,
697
+ 512,
698
+ lora_config.lora_extra_vocab_size,
699
+ )
700
+ lora_linear.set_mapping(*mapping_info)
701
+
702
+ lora_result = lora_linear(torch.cat(inputs))[0]
703
+ expected_result = linear(torch.cat(inputs))[0]
704
+
705
+ rtol, atol = TOLERANCES[lora_result.dtype]
706
+ assert torch.allclose(lora_result,
707
+ expected_result,
708
+ rtol=rtol,
709
+ atol=atol)
tests/lora/test_llama.py
new
+144
@@ -0,0 +1,144 @@
1
+import pytest
2
+import ray
3
+
4
+import vllm
5
+from vllm.lora.request import LoRARequest
6
+from .conftest import cleanup
7
+
8
+MODEL_PATH = "meta-llama/Llama-2-7b-hf"
9
+
10
+
11
+def do_sample(llm, lora_path: str, lora_id: int):
12
+ prompts = [
13
+ "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_74 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]",
14
+ "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_11 (nationality VARCHAR, elector VARCHAR)\n\n question: When Anchero Pantaleone was the elector what is under nationality? [/user] [assistant]",
15
+ "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_95 (one_mora VARCHAR, gloss VARCHAR, accented_mora VARCHAR)\n\n question: What is the one mora for a low tone mora with a gloss of /˩okiru/ [òkìɽɯ́]? [/user] [assistant]",
16
+ "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE candidate (people_id VARCHAR, unsure_rate INTEGER); CREATE TABLE people (sex VARCHAR, people_id VARCHAR)\n\n question: which gender got the highest average uncertain ratio. [/user] [assistant]",
17
+ "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_60 (pick INTEGER, former_wnba_team VARCHAR)\n\n question: What pick was a player that previously played for the Minnesota Lynx? [/user] [assistant]",
18
+ "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_28138035_4 (womens_doubles VARCHAR, mens_singles VARCHAR)\n\n question: Name the womens doubles for werner schlager [/user] [assistant]"
19
+ ]
20
+ sampling_params = vllm.SamplingParams(temperature=0,
21
+ max_tokens=256,
22
+ stop=["[/assistant]"])
23
+ outputs = llm.generate(
24
+ prompts,
25
+ sampling_params,
26
+ lora_request=LoRARequest(str(lora_id), lora_id, lora_path)
27
+ if lora_id else None)
28
+ # Print the outputs.
29
+ generated_texts = []
30
+ for output in outputs:
31
+ prompt = output.prompt
32
+ generated_text = output.outputs[0].text
33
+ generated_texts.append(generated_text)
34
+ print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
35
+ return generated_texts
36
+
37
+
38
+@pytest.mark.parametrize("tp_size", [1])
39
+def test_llama_lora(sql_lora_files, tp_size):
40
+ # Cannot use as it will initialize torch.cuda too early...
41
+ # if torch.cuda.device_count() < tp_size:
42
+ # pytest.skip(f"Not enough GPUs for tensor parallelism {tp_size}")
43
+
44
+ llm = vllm.LLM(MODEL_PATH,
45
+ enable_lora=True,
46
+ max_num_seqs=16,
47
+ max_loras=4,
48
+ tensor_parallel_size=tp_size)
49
+
50
+ expected_no_lora_output = [
51
+ "\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_75 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_76 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_77 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_78 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user]",
52
+ " Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_11 (nationality VARCHAR, elector VARCHAR)\n\n question: When Anchero Pantaleone was the elector what is under nationality? ",
53
+ "\n\n answer: 1\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_96 (one_mora VARCHAR, gloss VARCHAR, accented_mora VARCHAR)\n\n question: What is the one mora for a high tone mora with a gloss of /˧kot/ [kòt]? [/user] [assistant]\n\n answer: 2\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_97 (one_mora VARCHAR, gloss VARCHAR, accented_mora VARCHAR)\n\n question: What is the one mora for a high tone mora with a gloss of /˧kot/ [kòt]? [/user] [assistant]\n\n answer: 2\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_98 (one_mora VARCHAR, gloss VARCHAR, accented_mora VARCHAR)\n\n question: What is the one m",
54
+ " Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE candidate (people_id VARCHAR, unsure_rate INTEGER); CREATE TABLE people (sex VARCHAR, people_id VARCHAR)\n\n question: which gender got the highest average uncertain ratio. ",
55
+ " Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_60 (pick INTEGER, former_wnba_team VARCHAR)\n\n question: What pick was a player that previously played for the Minnesota Lynx? ",
56
+ "\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_28138035_4 (womens_doubles VARCHAR, mens_singles VARCHAR)\n\n question: Name the womens doubles for werner schlager [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_28138035_4 (womens_doubles VARCHAR, mens_singles VARCHAR)\n\n question: Name the womens doubles for werner schlager [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_28138035_4 (womens_doubles VARCHAR, mens_singles VARCHAR)\n\n question: Name the womens doubles for werner schlager [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE",
57
+ ]
58
+ expected_lora_output = [
59
+ " SELECT icao FROM table_name_74 WHERE airport = 'lilongwe international airport' ",
60
+ " SELECT nationality FROM table_name_11 WHERE elector = 'anchero pantaleone' ",
61
+ " SELECT one_mora FROM table_name_95 WHERE gloss = 'low tone mora with a gloss of /˩okiru/' [òkìɽɯ́] AND accented_mora = 'low tone mora with a gloss of /˩okiru/' [òkìɽɯ́] ",
62
+ " SELECT sex FROM people WHERE people_id IN (SELECT people_id FROM candidate GROUP BY sex ORDER BY COUNT(people_id) DESC LIMIT 1) ",
63
+ " SELECT pick FROM table_name_60 WHERE former_wnba_team = 'Minnesota Lynx' ",
64
+ " SELECT womens_doubles FROM table_28138035_4 WHERE mens_singles = 'Werner Schlager' "
65
+ ]
66
+
67
+ print("lora adapter created")
68
+ assert do_sample(llm, sql_lora_files, lora_id=0) == expected_no_lora_output
69
+
70
+ print("lora 1")
71
+ assert do_sample(llm, sql_lora_files, lora_id=1) == expected_lora_output
72
+
73
+ print("no lora")
74
+ assert do_sample(llm, sql_lora_files, lora_id=0) == expected_no_lora_output
75
+
76
+ print("lora 2")
77
+ assert do_sample(llm, sql_lora_files, lora_id=2) == expected_lora_output
78
+
79
+ print("removing lora")
80
+
81
+
82
+@pytest.mark.skip("Requires multiple GPUs")
83
+def test_llama_tensor_parallel_equality(sql_lora_files):
84
+ # Cannot use as it will initialize torch.cuda too early...
85
+ # if torch.cuda.device_count() < 4:
86
+ # pytest.skip(f"Not enough GPUs for tensor parallelism {4}")
87
+
88
+ llm_tp1 = vllm.LLM(MODEL_PATH,
89
+ enable_lora=True,
90
+ max_num_seqs=16,
91
+ max_loras=4,
92
+ tensor_parallel_size=1)
93
+ output_tp1 = do_sample(llm_tp1, sql_lora_files, lora_id=1)
94
+
95
+ del llm_tp1
96
+ cleanup()
97
+
98
+ llm_tp2 = vllm.LLM(MODEL_PATH,
99
+ enable_lora=True,
100
+ max_num_seqs=16,
101
+ max_loras=4,
102
+ tensor_parallel_size=2)
103
+ output_tp2 = do_sample(llm_tp2, sql_lora_files, lora_id=1)
104
+
105
+ del llm_tp2
106
+ cleanup()
107
+
108
+ assert output_tp1 == output_tp2
109
+
110
+ llm_tp4 = vllm.LLM(MODEL_PATH,
111
+ enable_lora=True,
112
+ max_num_seqs=16,
113
+ max_loras=4,
114
+ tensor_parallel_size=4)
115
+ output_tp4 = do_sample(llm_tp4, sql_lora_files, lora_id=1)
116
+
117
+ del llm_tp4
118
+ cleanup()
119
+
120
+ assert output_tp1 == output_tp4
121
+
122
+
123
+def test_llama_lora_warmup(sql_lora_files):
124
+ """Test that the LLM initialization works with a warmup LORA path and is more conservative"""
125
+
126
+ @ray.remote(num_gpus=1)
127
+ def get_num_gpu_blocks_lora():
128
+ llm = vllm.LLM(MODEL_PATH, enable_lora=True, max_num_seqs=16)
129
+ num_gpu_blocks_lora_warmup = llm.llm_engine.cache_config.num_gpu_blocks
130
+ return num_gpu_blocks_lora_warmup
131
+
132
+ @ray.remote(num_gpus=1)
133
+ def get_num_gpu_blocks_no_lora():
134
+ llm = vllm.LLM(MODEL_PATH, max_num_seqs=16)
135
+ num_gpu_blocks_no_lora_warmup = llm.llm_engine.cache_config.num_gpu_blocks
136
+ return num_gpu_blocks_no_lora_warmup
137
+
138
+ num_gpu_blocks_lora_warmup = ray.get(get_num_gpu_blocks_lora.remote())
139
+ num_gpu_blocks_no_lora_warmup = ray.get(
140
+ get_num_gpu_blocks_no_lora.remote())
141
+ assert num_gpu_blocks_lora_warmup < num_gpu_blocks_no_lora_warmup, (
142
+ "The warmup with lora should be more"
143
+ " conservative than without lora, therefore the number of memory blocks for the KV cache should be "
144
+ "less when using lora than when not using lora")
tests/lora/test_lora.py
new
+224
@@ -0,0 +1,224 @@
1
+import pytest
2
+import torch
3
+
4
+from vllm.lora.layers import _apply_lora, _apply_lora_packed_nslice
5
+
6
+from .utils import DummyLoRAManager
7
+
8
+TENSOR_SIZES = [128, 1024, 2048, 4096, 8192, 11008, 11008 // 2, 11008 // 4]
9
+QKV_TENSOR_SIZES = [
10
+ (8192, 1024, 1024),
11
+ (8192 // 8, 1024 // 8, 1024 // 8),
12
+ (4096, 4096, 4096),
13
+ (4096 // 2, 4096 // 2, 4096 // 2),
14
+]
15
+BATCH_SIZES = [8, 32, 256]
16
+RANKS = [8]
17
+DTYPES = [torch.float16]
18
+TOLERANCES = {
19
+ torch.float16: (5e-3, 5e-3),
20
+ torch.bfloat16: (3e-2, 2e-2),
21
+}
22
+
23
+
24
+@pytest.mark.parametrize("m", TENSOR_SIZES)
25
+@pytest.mark.parametrize("n", TENSOR_SIZES)
26
+@pytest.mark.parametrize("k", BATCH_SIZES)
27
+@pytest.mark.parametrize("rank", RANKS)
28
+@pytest.mark.parametrize("dtype", DTYPES)
29
+def test_apply_lora(m, n, k, rank, dtype) -> None:
30
+ manager = DummyLoRAManager()
31
+
32
+ module_name = "module"
33
+ weight = torch.rand([m, n], device="cuda", dtype=dtype)
34
+
35
+ manager.init_random_lora(module_name, weight, rank=rank)
36
+ lora = manager.get_module_lora(module_name)
37
+
38
+ input = torch.rand(k, n, device="cuda", dtype=dtype)
39
+ expected = input @ lora.lora_a @ lora.lora_b * lora.scaling
40
+
41
+ lora_a_stack = torch.zeros(8,
42
+ 1,
43
+ lora.lora_a.shape[1],
44
+ lora.lora_a.shape[0],
45
+ device="cuda",
46
+ dtype=dtype)
47
+ lora_b_stack = torch.zeros(8,
48
+ 1,
49
+ lora.lora_b.shape[1],
50
+ lora.lora_b.shape[0],
51
+ device="cuda",
52
+ dtype=dtype)
53
+ for i in range(lora_a_stack.shape[0]):
54
+ lora_a_stack[i][0] = lora.lora_a.T
55
+ lora_b_stack[i][0] = (lora.lora_b * lora.scaling).T
56
+
57
+ output = torch.zeros(k, m, device="cuda", dtype=dtype)
58
+ _apply_lora(
59
+ input, lora_a_stack, lora_b_stack,
60
+ torch.randint(0, lora_a_stack.shape[0], (len(input), ), device="cuda"),
61
+ output)
62
+
63
+ rtol, atol = TOLERANCES[dtype]
64
+ assert torch.allclose(expected, output, rtol=rtol, atol=atol)
65
+
66
+ output[:] = 0
67
+ _apply_lora(input, lora_a_stack, lora_b_stack,
68
+ torch.full((len(input), ), -1, device="cuda"), output)
69
+ assert torch.allclose(torch.zeros_like(output), output)
70
+
71
+ manager.reset_lora()
72
+
73
+
74
+@pytest.mark.parametrize("m", TENSOR_SIZES)
75
+@pytest.mark.parametrize("n", TENSOR_SIZES)
76
+@pytest.mark.parametrize("k", BATCH_SIZES)
77
+@pytest.mark.parametrize("rank", RANKS)
78
+@pytest.mark.parametrize("dtype", DTYPES)
79
+def test_apply_lora_packed_2slice(m, n, k, rank, dtype) -> None:
80
+ if m % 2 != 0:
81
+ pytest.skip("m must be divisible by 2")
82
+ if m // 2 not in TENSOR_SIZES:
83
+ pytest.skip("m//2 must be in TENSOR_SIZES")
84
+
85
+ manager = DummyLoRAManager()
86
+
87
+ module_name = "module"
88
+ weight = torch.rand([m // 2, n], device="cuda", dtype=dtype)
89
+
90
+ manager.init_random_lora(module_name + "1", weight, rank=rank)
91
+ lora_1 = manager.get_module_lora(module_name + "1")
92
+ manager.init_random_lora(module_name + "2", weight, rank=rank)
93
+ lora_2 = manager.get_module_lora(module_name + "2")
94
+
95
+ input = torch.rand(k, n, device="cuda", dtype=dtype)
96
+ expected = torch.cat([
97
+ input @ lora_1.lora_a @ lora_1.lora_b * lora_1.scaling,
98
+ input @ lora_2.lora_a @ lora_2.lora_b * lora_2.scaling
99
+ ],
100
+ dim=1)
101
+
102
+ lora_a_stacks = [
103
+ torch.zeros(8,
104
+ 1,
105
+ lora_1.lora_a.shape[1],
106
+ lora_1.lora_a.shape[0],
107
+ device="cuda",
108
+ dtype=dtype) for i in range(2)
109
+ ]
110
+ lora_b_stacks = [
111
+ torch.zeros(8,
112
+ 1,
113
+ lora_1.lora_b.shape[1],
114
+ lora_1.lora_b.shape[0],
115
+ device="cuda",
116
+ dtype=dtype) for i in range(2)
117
+ ]
118
+ for i in range(lora_a_stacks[0].shape[0]):
119
+ lora_a_stacks[0][i][0] = lora_1.lora_a.T
120
+ lora_b_stacks[0][i][0] = (lora_1.lora_b * lora_1.scaling).T
121
+ lora_a_stacks[1][i][0] = lora_2.lora_a.T
122
+ lora_b_stacks[1][i][0] = (lora_2.lora_b * lora_2.scaling).T
123
+
124
+ output = torch.zeros(k, m, device="cuda", dtype=dtype)
125
+ _apply_lora_packed_nslice(
126
+ input, lora_a_stacks, lora_b_stacks,
127
+ torch.randint(0,
128
+ lora_a_stacks[0].shape[0], (len(input), ),
129
+ device="cuda"), output, (m // 2, m // 2))
130
+
131
+ rtol, atol = TOLERANCES[dtype]
132
+ assert torch.allclose(expected, output, rtol=rtol, atol=atol)
133
+
134
+ output[:] = 0
135
+ _apply_lora_packed_nslice(input, lora_a_stacks, lora_b_stacks,
136
+ torch.full((len(input), ), -1, device="cuda"),
137
+ output, (m // 2, m // 2))
138
+ assert torch.allclose(torch.zeros_like(output), output)
139
+
140
+ manager.reset_lora()
141
+
142
+
143
+@pytest.mark.parametrize("qkv", QKV_TENSOR_SIZES)
144
+@pytest.mark.parametrize("n", TENSOR_SIZES)
145
+@pytest.mark.parametrize("k", BATCH_SIZES)
146
+@pytest.mark.parametrize("rank", RANKS)
147
+@pytest.mark.parametrize("dtype", DTYPES)
148
+def test_apply_lora_packed_3slice(qkv, n, k, rank, dtype) -> None:
149
+ manager = DummyLoRAManager()
150
+
151
+ module_name = "module"
152
+ weight_q = torch.empty(qkv[0], n, device="cuda", dtype=dtype)
153
+ weight_kv = torch.empty(qkv[1], n, device="cuda", dtype=dtype)
154
+
155
+ manager.init_random_lora(module_name + "q", weight_q, rank=rank)
156
+ lora_q = manager.get_module_lora(module_name + "q")
157
+ manager.init_random_lora(module_name + "k", weight_kv, rank=rank)
158
+ lora_k = manager.get_module_lora(module_name + "k")
159
+ manager.init_random_lora(module_name + "v", weight_kv, rank=rank)
160
+ lora_v = manager.get_module_lora(module_name + "v")
161
+
162
+ input = torch.rand(k, n, device="cuda", dtype=dtype)
163
+ expected = torch.cat([
164
+ input @ lora_q.lora_a @ lora_q.lora_b * lora_q.scaling,
165
+ input @ lora_k.lora_a @ lora_k.lora_b * lora_k.scaling,
166
+ input @ lora_v.lora_a @ lora_v.lora_b * lora_v.scaling
167
+ ],
168
+ dim=1)
169
+
170
+ lora_a_stacks = [
171
+ torch.zeros(8,
172
+ 1,
173
+ lora_q.lora_a.shape[1],
174
+ lora_q.lora_a.shape[0],
175
+ device="cuda",
176
+ dtype=dtype)
177
+ ] + [
178
+ torch.zeros(8,
179
+ 1,
180
+ lora_k.lora_a.shape[1],
181
+ lora_k.lora_a.shape[0],
182
+ device="cuda",
183
+ dtype=dtype) for i in range(2)
184
+ ]
185
+ lora_b_stacks = [
186
+ torch.zeros(8,
187
+ 1,
188
+ lora_q.lora_b.shape[1],
189
+ lora_q.lora_b.shape[0],
190
+ device="cuda",
191
+ dtype=dtype)
192
+ ] + [
193
+ torch.zeros(8,
194
+ 1,
195
+ lora_k.lora_b.shape[1],
196
+ lora_k.lora_b.shape[0],
197
+ device="cuda",
198
+ dtype=dtype) for i in range(2)
199
+ ]
200
+ for i in range(lora_a_stacks[0].shape[0]):
201
+ lora_a_stacks[0][i][0] = lora_q.lora_a.T
202
+ lora_b_stacks[0][i][0] = (lora_q.lora_b * lora_q.scaling).T
203
+ lora_a_stacks[1][i][0] = lora_k.lora_a.T
204
+ lora_b_stacks[1][i][0] = (lora_k.lora_b * lora_k.scaling).T
205
+ lora_a_stacks[2][i][0] = lora_v.lora_a.T
206
+ lora_b_stacks[2][i][0] = (lora_v.lora_b * lora_v.scaling).T
207
+
208
+ output = torch.zeros(k, sum(qkv), device="cuda", dtype=dtype)
209
+ _apply_lora_packed_nslice(
210
+ input, lora_a_stacks, lora_b_stacks,
211
+ torch.randint(0,
212
+ lora_a_stacks[0].shape[0], (len(input), ),
213
+ device="cuda"), output, (qkv[0], qkv[1], qkv[2]))
214
+
215
+ rtol, atol = TOLERANCES[dtype]
216
+ assert torch.allclose(expected, output, rtol=rtol, atol=atol)
217
+
218
+ output[:] = 0
219
+ _apply_lora_packed_nslice(input, lora_a_stacks, lora_b_stacks,
220
+ torch.full((len(input), ), -1, device="cuda"),
221
+ output, (qkv[0], qkv[1], qkv[2]))
222
+ assert torch.allclose(torch.zeros_like(output), output)
223
+
224
+ manager.reset_lora()
tests/lora/test_lora_manager.py
new
+475
@@ -0,0 +1,475 @@
1
+import os
2
+from typing import List
3
+
4
+import pytest
5
+import torch
6
+from safetensors.torch import load_file
7
+from torch import nn
8
+
9
+from vllm.config import LoRAConfig
10
+from vllm.lora.layers import (ColumnParallelLinearWithLoRA,
11
+ RowParallelLinearWithLoRA,
12
+ MergedColumnParallelLinearWithLoRA)
13
+from vllm.lora.lora import LoRALayerWeights, PackedLoRALayerWeights
14
+from vllm.lora.models import (EMBEDDING_MODULES, LoRAModel, LoRAModelManager,
15
+ LRUCacheLoRAModelManager, LoRAMapping)
16
+from vllm.lora.request import LoRARequest
17
+from vllm.lora.worker_manager import (LRUCacheWorkerLoRAManager,
18
+ WorkerLoRAManager)
19
+from vllm.model_executor.layers.linear import RowParallelLinear
20
+
21
+
22
+def test_from_lora_tensors(sql_lora_files):
23
+ tensors = load_file(
24
+ os.path.join(sql_lora_files, "adapter_model.safetensors"))
25
+ new_embeddings = load_file(
26
+ os.path.join(sql_lora_files, "new_embeddings.safetensors"))
27
+ lora_model = LoRAModel.from_lora_tensors(1,
28
+ 8,
29
+ 16,
30
+ tensors,
31
+ "cuda",
32
+ embeddings=new_embeddings)
33
+ for module_name, lora in lora_model.loras.items():
34
+ assert lora.module_name == module_name
35
+ assert lora.rank == 8
36
+ assert lora.lora_alpha == 16
37
+ assert lora.lora_a is not None
38
+ assert lora.lora_b is not None
39
+ assert (lora.lora_a.shape[1] == lora.lora_b.shape[0]
40
+ ), f"{lora.lora_a.shape=}, {lora.lora_b.shape=}"
41
+ assert lora.lora_a.shape[1] == 8
42
+ embeddings_module = next(
43
+ (k for k in EMBEDDING_MODULES if k in module_name), None)
44
+ if embeddings_module:
45
+ assert torch.equal(
46
+ lora.embeddings_tensor,
47
+ new_embeddings[EMBEDDING_MODULES[embeddings_module]].to(
48
+ device=lora.embeddings_tensor.device))
49
+ else:
50
+ assert lora.embeddings_tensor is None
51
+
52
+
53
+def create_lora(lora_id: int, model: nn.Module,
54
+ sub_modules: List[str]) -> LoRAModel:
55
+ loras = {}
56
+ for name in sub_modules:
57
+ w = model.get_submodule(name).weight
58
+ loras[name] = LoRALayerWeights(
59
+ name,
60
+ 8,
61
+ 16,
62
+ torch.rand([w.shape[1], 8], device="cuda"),
63
+ torch.rand([8, w.shape[0]], device="cuda"),
64
+ )
65
+ return LoRAModel(lora_id, 8, loras)
66
+
67
+
68
+def create_packed_lora(
69
+ lora_id: int,
70
+ model: nn.Module,
71
+ module_name,
72
+ replaced_module_names,
73
+ empty_replaced_module_name=None,
74
+) -> LoRAModel:
75
+ w = model.get_submodule(module_name).weight
76
+ loras = {}
77
+ for replaced_module_name in replaced_module_names:
78
+ if replaced_module_name == empty_replaced_module_name:
79
+ continue
80
+ loras[replaced_module_name] = LoRALayerWeights(
81
+ replaced_module_name,
82
+ 8,
83
+ 16,
84
+ torch.rand([w.shape[1], 8], device="cuda"),
85
+ torch.rand([8, w.shape[0] // len(replaced_module_names)],
86
+ device="cuda"),
87
+ )
88
+ return LoRAModel(lora_id, 8, loras)
89
+
90
+
91
+def test_replace_submodules(dist_init, dummy_model):
92
+ model = dummy_model
93
+ manager = LoRAModelManager(model,
94
+ 1,
95
+ 1,
96
+ 1,
97
+ LoRAConfig(max_lora_rank=8,
98
+ max_cpu_loras=8,
99
+ max_loras=8),
100
+ lora_target_modules=["dense1", "layer1.dense2"])
101
+ model = manager.model
102
+
103
+ assert isinstance(model.get_submodule("dense1"),
104
+ ColumnParallelLinearWithLoRA)
105
+ assert isinstance(model.get_submodule("layer1.dense1"),
106
+ ColumnParallelLinearWithLoRA)
107
+ assert isinstance(model.get_submodule("dense2"), RowParallelLinear)
108
+ assert isinstance(model.get_submodule("layer1.dense2"),
109
+ RowParallelLinearWithLoRA)
110
+
111
+
112
+def test_lora_model_manager(dist_init, dummy_model):
113
+ model = dummy_model
114
+ model_lora1 = create_lora(1, model, ["layer1.dense1", "dense2", "lm_head"])
115
+ model_lora2 = create_lora(2, model, ["dense1", "dense2", "lm_head"])
116
+ model_lora3 = create_lora(3, model, ["dense1", "dense2", "lm_head"])
117
+ manager = LoRAModelManager(
118
+ model,
119
+ 2,
120
+ 2,
121
+ 2,
122
+ LoRAConfig(max_lora_rank=8, max_cpu_loras=3, max_loras=2),
123
+ lora_target_modules=["dense1", "dense2", "lm_head"])
124
+ assert all(x is None for x in manager.lora_index_to_id)
125
+ assert manager.add_lora(model_lora1)
126
+ assert manager.activate_lora(1)
127
+ assert manager.lora_index_to_id[0] == 1
128
+ assert not manager.add_lora(model_lora1)
129
+ assert not manager.activate_lora(1)
130
+ assert manager.add_lora(model_lora2)
131
+ assert manager.activate_lora(2)
132
+ assert manager.lora_index_to_id[0] == 1
133
+ assert manager.lora_index_to_id[1] == 2
134
+ assert not manager.add_lora(model_lora2)
135
+ assert not manager.activate_lora(2)
136
+ assert manager.add_lora(model_lora3)
137
+ assert manager.lora_index_to_id[0] == 1
138
+ assert manager.lora_index_to_id[1] == 2
139
+ with pytest.raises(ValueError):
140
+ assert manager.activate_lora(3)
141
+ assert manager.lora_index_to_id[0] == 1
142
+ assert manager.lora_index_to_id[1] == 2
143
+ assert manager.remove_lora(model_lora2.id)
144
+ assert manager.lora_index_to_id[1] is None
145
+ assert not manager.remove_lora(model_lora2.id)
146
+ assert manager.remove_lora(model_lora1.id)
147
+ assert not manager.remove_lora(model_lora1.id)
148
+ assert manager.add_lora(model_lora1)
149
+ assert manager.lora_index_to_id[0] is None
150
+ assert manager.lora_index_to_id[1] is None
151
+ assert manager.add_lora(model_lora2)
152
+ assert manager.activate_lora(3)
153
+ assert manager.lora_index_to_id[0] == 3
154
+ assert manager.lora_index_to_id[1] is None
155
+ assert manager.activate_lora(2)
156
+ assert manager.lora_index_to_id[0] == 3
157
+ assert manager.lora_index_to_id[1] == 2
158
+
159
+
160
+def test_lora_lru_cache_model_manager(dist_init, dummy_model):
161
+ model = dummy_model
162
+ model_lora1 = create_lora(1, model, ["layer1.dense1", "dense2", "lm_head"])
163
+ model_lora2 = create_lora(2, model, ["dense1", "dense2", "lm_head"])
164
+ model_lora3 = create_lora(3, model, ["dense1", "dense2", "lm_head"])
165
+ manager = LRUCacheLoRAModelManager(
166
+ model,
167
+ 2,
168
+ 2,
169
+ 2,
170
+ LoRAConfig(max_lora_rank=8, max_cpu_loras=3, max_loras=2),
171
+ lora_target_modules=["dense1", "dense2", "lm_head"])
172
+ assert all(x is None for x in manager.lora_index_to_id)
173
+ assert manager.add_lora(model_lora1)
174
+ assert manager.activate_lora(1)
175
+ assert manager.lora_index_to_id[0] == 1
176
+ assert not manager.add_lora(model_lora1)
177
+ assert not manager.activate_lora(1)
178
+ assert manager.add_lora(model_lora2)
179
+ assert manager.activate_lora(2)
180
+ assert manager.lora_index_to_id[0] == 1
181
+ assert manager.lora_index_to_id[1] == 2
182
+ assert not manager.add_lora(model_lora2)
183
+ assert not manager.activate_lora(2)
184
+ assert manager.add_lora(model_lora3)
185
+ assert manager.lora_index_to_id[0] == 1
186
+ assert manager.lora_index_to_id[1] == 2
187
+ assert manager.activate_lora(3)
188
+ assert manager.lora_index_to_id[0] == 3
189
+ assert manager.lora_index_to_id[1] == 2
190
+ assert manager.remove_lora(model_lora2.id)
191
+ assert manager.lora_index_to_id[1] is None
192
+ assert not manager.remove_lora(model_lora2.id)
193
+ assert manager.remove_lora(model_lora1.id)
194
+ assert not manager.remove_lora(model_lora1.id)
195
+ assert manager.add_lora(model_lora1)
196
+ assert manager.activate_lora(1)
197
+ assert manager.lora_index_to_id[0] == 3
198
+ assert manager.lora_index_to_id[1] == 1
199
+ assert manager.add_lora(model_lora2)
200
+ assert manager.deactivate_lora(3)
201
+ assert manager.lora_index_to_id[0] is None
202
+ assert manager.lora_index_to_id[1] == 1
203
+ assert manager.activate_lora(2)
204
+ assert manager.lora_index_to_id[0] == 2
205
+ assert manager.lora_index_to_id[1] == 1
206
+ assert manager.activate_lora(3)
207
+ assert manager.lora_index_to_id[0] == 2
208
+ assert manager.lora_index_to_id[1] == 3
209
+
210
+
211
+def test_lru_lora_model_manager(dist_init, dummy_model):
212
+ # This tests just the LRU cache functionality, everything else is
213
+ # tested in test_lora_model_manager
214
+ model = dummy_model
215
+ model_lora1 = create_lora(1, model, ["layer1.dense1", "dense2", "lm_head"])
216
+ model_lora2 = create_lora(2, model, ["dense1", "dense2", "lm_head"])
217
+ model_lora3 = create_lora(3, model, ["dense1", "dense2", "lm_head"])
218
+ model_lora4 = create_lora(4, model, ["dense1", "dense2", "lm_head"])
219
+ manager = LRUCacheLoRAModelManager(
220
+ model, 2, 2, 2,
221
+ LoRAConfig(max_lora_rank=8, max_cpu_loras=2, max_loras=2),
222
+ ["dense1", "dense2", "lm_head"])
223
+
224
+ assert all(x is None for x in manager.lora_index_to_id)
225
+
226
+ # Add up to capacity
227
+ assert manager.add_lora(model_lora1)
228
+ assert manager.add_lora(model_lora2)
229
+ assert manager.activate_lora(1)
230
+ assert manager.activate_lora(2)
231
+
232
+ assert set(manager.list_loras()) == {1, 2}
233
+ assert manager.lora_index_to_id[0] == 1
234
+ assert manager.lora_index_to_id[1] == 2
235
+
236
+ # Add over capacity
237
+ assert manager.add_lora(model_lora3)
238
+ assert manager.add_lora(model_lora4)
239
+ assert manager.activate_lora(3)
240
+ assert manager.activate_lora(4)
241
+
242
+ assert set(manager.list_loras()) == {3, 4}
243
+ assert manager.lora_index_to_id[0] == 3
244
+ assert manager.lora_index_to_id[1] == 4
245
+
246
+ # Add 3 again to move it to the top and then add 2
247
+ # should return false since it's in already
248
+ assert not manager.add_lora(model_lora3)
249
+ assert not manager.activate_lora(3)
250
+ assert manager.add_lora(model_lora2)
251
+ assert manager.activate_lora(2)
252
+
253
+ assert set(manager.list_loras()) == {3, 2}
254
+ assert manager.lora_index_to_id[0] == 3
255
+ assert manager.lora_index_to_id[1] == 2
256
+
257
+ # Remove manually
258
+ assert manager.remove_lora(3)
259
+ assert not manager.remove_lora(3)
260
+
261
+ assert set(manager.list_loras()) == {2}
262
+ assert manager.lora_index_to_id[0] is None
263
+ assert manager.lora_index_to_id[1] == 2
264
+
265
+ assert manager.add_lora(model_lora3)
266
+ assert manager.activate_lora(3)
267
+ assert manager.add_lora(model_lora4)
268
+ assert manager.activate_lora(4)
269
+
270
+ assert set(manager.list_loras()) == {3, 4}
271
+ assert manager.lora_index_to_id[0] == 3
272
+ assert manager.lora_index_to_id[1] == 4
273
+
274
+ assert manager.remove_oldest_lora()
275
+ assert set(manager.list_loras()) == {4}
276
+ assert manager.lora_index_to_id[0] is None
277
+ assert manager.lora_index_to_id[1] == 4
278
+
279
+ assert manager.remove_oldest_lora()
280
+ assert set(manager.list_loras()) == set()
281
+ assert all(x is None for x in manager.lora_index_to_id)
282
+
283
+ assert not manager.remove_oldest_lora()
284
+ assert set(manager.list_loras()) == set()
285
+ assert all(x is None for x in manager.lora_index_to_id)
286
+
287
+
288
+def test_lru_cache_worker_lora_manager(llama_2_7b_model_extra_embeddings,
289
+ sql_lora_files):
290
+ lora_config = LoRAConfig(max_lora_rank=8, max_cpu_loras=4, max_loras=4)
291
+ worker_lora_manager = LRUCacheWorkerLoRAManager(
292
+ 4, 2, llama_2_7b_model_extra_embeddings.config.vocab_size, lora_config,
293
+ torch.device("cuda"))
294
+ worker_lora_manager.create_lora_manager(llama_2_7b_model_extra_embeddings)
295
+
296
+ mapping = LoRAMapping([], [])
297
+ worker_lora_manager.set_active_loras([
298
+ LoRARequest("1", 1, sql_lora_files),
299
+ LoRARequest("2", 2, sql_lora_files)
300
+ ], mapping)
301
+ assert worker_lora_manager.list_loras() == {1, 2}
302
+ assert worker_lora_manager._lora_manager.lora_index_to_id[0] == 1
303
+ assert worker_lora_manager._lora_manager.lora_index_to_id[1] == 2
304
+
305
+ worker_lora_manager.set_active_loras([
306
+ LoRARequest("1", 1, sql_lora_files),
307
+ LoRARequest("3", 3, sql_lora_files),
308
+ LoRARequest("4", 4, sql_lora_files)
309
+ ], mapping)
310
+ assert worker_lora_manager.list_loras() == {1, 2, 3, 4}
311
+ assert worker_lora_manager._lora_manager.lora_index_to_id[0] == 1
312
+ assert worker_lora_manager._lora_manager.lora_index_to_id[1] == 2
313
+ assert worker_lora_manager._lora_manager.lora_index_to_id[2] == 3
314
+ assert worker_lora_manager._lora_manager.lora_index_to_id[3] == 4
315
+
316
+ worker_lora_manager.set_active_loras([
317
+ LoRARequest("1", 1, sql_lora_files),
318
+ LoRARequest("2", 2, sql_lora_files),
319
+ LoRARequest("5", 5, sql_lora_files)
320
+ ], mapping)
321
+ assert worker_lora_manager.list_loras() == {1, 2, 4, 5}
322
+ assert worker_lora_manager._lora_manager.lora_index_to_id[0] == 1
323
+ assert worker_lora_manager._lora_manager.lora_index_to_id[1] == 2
324
+ assert worker_lora_manager._lora_manager.lora_index_to_id[2] == 5
325
+ assert worker_lora_manager._lora_manager.lora_index_to_id[3] == 4
326
+
327
+ worker_lora_manager.set_active_loras([
328
+ LoRARequest("1", 1, sql_lora_files),
329
+ LoRARequest("1", 1, sql_lora_files),
330
+ LoRARequest("1", 1, sql_lora_files)
331
+ ], mapping)
332
+ assert worker_lora_manager.list_loras() == {1, 2, 4, 5}
333
+ assert worker_lora_manager._lora_manager.lora_index_to_id[0] == 1
334
+ assert worker_lora_manager._lora_manager.lora_index_to_id[1] == 2
335
+ assert worker_lora_manager._lora_manager.lora_index_to_id[2] == 5
336
+ assert worker_lora_manager._lora_manager.lora_index_to_id[3] == 4
337
+
338
+ worker_lora_manager.set_active_loras([
339
+ LoRARequest("6", 6, sql_lora_files),
340
+ LoRARequest("7", 7, sql_lora_files),
341
+ LoRARequest("8", 8, sql_lora_files)
342
+ ], mapping)
343
+ assert worker_lora_manager.list_loras() == {1, 6, 7, 8}
344
+ assert worker_lora_manager._lora_manager.lora_index_to_id[0] == 1
345
+ assert worker_lora_manager._lora_manager.lora_index_to_id[1] == 7
346
+ assert worker_lora_manager._lora_manager.lora_index_to_id[2] == 8
347
+ assert worker_lora_manager._lora_manager.lora_index_to_id[3] == 6
348
+
349
+ # Over capacity
350
+ with pytest.raises(RuntimeError):
351
+ worker_lora_manager.set_active_loras([
352
+ LoRARequest("10", 10, sql_lora_files),
353
+ LoRARequest("11", 11, sql_lora_files),
354
+ LoRARequest("12", 12, sql_lora_files),
355
+ LoRARequest("13", 13, sql_lora_files),
356
+ LoRARequest("14", 14, sql_lora_files)
357
+ ], mapping)
358
+
359
+
360
+def test_worker_lora_manager(llama_2_7b_model_extra_embeddings,
361
+ sql_lora_files):
362
+ # Should remove every LoRA not specified in the request.
363
+ lora_config = LoRAConfig(max_lora_rank=8, max_cpu_loras=4, max_loras=4)
364
+ worker_lora_manager = WorkerLoRAManager(
365
+ 4, 2, llama_2_7b_model_extra_embeddings.config.vocab_size, lora_config,
366
+ torch.device("cuda"))
367
+ worker_lora_manager.create_lora_manager(llama_2_7b_model_extra_embeddings)
368
+
369
+ mapping = LoRAMapping([], [])
370
+ worker_lora_manager.set_active_loras([
371
+ LoRARequest("1", 1, sql_lora_files),
372
+ LoRARequest("2", 2, sql_lora_files)
373
+ ], mapping)
374
+ assert worker_lora_manager.list_loras() == {1, 2}
375
+ assert worker_lora_manager._lora_manager.lora_index_to_id[0] == 1
376
+ assert worker_lora_manager._lora_manager.lora_index_to_id[1] == 2
377
+
378
+ worker_lora_manager.set_active_loras([
379
+ LoRARequest("1", 1, sql_lora_files),
380
+ LoRARequest("3", 3, sql_lora_files),
381
+ LoRARequest("4", 4, sql_lora_files)
382
+ ], mapping)
383
+ assert worker_lora_manager.list_loras() == {1, 3, 4}
384
+ assert worker_lora_manager._lora_manager.lora_index_to_id[0] == 1
385
+ assert worker_lora_manager._lora_manager.lora_index_to_id[1] == 3
386
+ assert worker_lora_manager._lora_manager.lora_index_to_id[2] == 4
387
+
388
+ worker_lora_manager.set_active_loras([
389
+ LoRARequest("1", 1, sql_lora_files),
390
+ LoRARequest("2", 2, sql_lora_files),
391
+ LoRARequest("5", 5, sql_lora_files)
392
+ ], mapping)
393
+ assert worker_lora_manager.list_loras() == {1, 2, 5}
394
+ assert worker_lora_manager._lora_manager.lora_index_to_id[0] == 1
395
+ assert worker_lora_manager._lora_manager.lora_index_to_id[1] == 2
396
+ assert worker_lora_manager._lora_manager.lora_index_to_id[2] == 5
397
+
398
+ worker_lora_manager.set_active_loras([
399
+ LoRARequest("1", 1, sql_lora_files),
400
+ LoRARequest("1", 1, sql_lora_files),
401
+ LoRARequest("1", 1, sql_lora_files)
402
+ ], mapping)
403
+ assert worker_lora_manager.list_loras() == {1}
404
+ assert worker_lora_manager._lora_manager.lora_index_to_id[0] == 1
405
+ assert worker_lora_manager._lora_manager.lora_index_to_id[1] is None
406
+ assert worker_lora_manager._lora_manager.lora_index_to_id[2] is None
407
+
408
+ worker_lora_manager.set_active_loras([
409
+ LoRARequest("6", 6, sql_lora_files),
410
+ LoRARequest("7", 7, sql_lora_files),
411
+ LoRARequest("8", 8, sql_lora_files)
412
+ ], mapping)
413
+ assert worker_lora_manager.list_loras() == {6, 7, 8}
414
+ assert worker_lora_manager._lora_manager.lora_index_to_id[0] == 8
415
+ assert worker_lora_manager._lora_manager.lora_index_to_id[1] == 6
416
+ assert worker_lora_manager._lora_manager.lora_index_to_id[2] == 7
417
+
418
+ # Over capacity
419
+ with pytest.raises(RuntimeError):
420
+ worker_lora_manager.set_active_loras([
421
+ LoRARequest("10", 10, sql_lora_files),
422
+ LoRARequest("11", 11, sql_lora_files),
423
+ LoRARequest("12", 12, sql_lora_files),
424
+ LoRARequest("13", 13, sql_lora_files),
425
+ LoRARequest("14", 14, sql_lora_files)
426
+ ], mapping)
427
+
428
+
429
+def test_packed_loras(dist_init, dummy_model_gate_up):
430
+ model = dummy_model_gate_up
431
+ model_lora = create_packed_lora(
432
+ 1,
433
+ model,
434
+ module_name="gate_up_proj",
435
+ replaced_module_names=["gate_proj", "up_proj"])
436
+ model_lora1 = create_packed_lora(
437
+ 2,
438
+ model,
439
+ module_name="gate_up_proj",
440
+ replaced_module_names=["gate_proj", "up_proj"],
441
+ empty_replaced_module_name="gate_proj",
442
+ )
443
+
444
+ manager = LoRAModelManager(
445
+ model, 2, 2, 2,
446
+ LoRAConfig(max_lora_rank=8, max_cpu_loras=2, max_loras=2),
447
+ ["gate_up_proj"])
448
+ model = manager.model
449
+
450
+ assert isinstance(model.get_submodule("gate_up_proj"),
451
+ MergedColumnParallelLinearWithLoRA)
452
+ assert manager.add_lora(model_lora)
453
+ assert manager.add_lora(model_lora1)
454
+
455
+ packed_lora = model_lora.get_lora("gate_up_proj")
456
+ assert packed_lora and isinstance(packed_lora, PackedLoRALayerWeights)
457
+
458
+ assert torch.allclose(packed_lora.lora_a[0],
459
+ model_lora.get_lora("gate_proj").lora_a)
460
+ assert torch.allclose(packed_lora.lora_b[0],
461
+ model_lora.get_lora("gate_proj").lora_b)
462
+ assert torch.allclose(packed_lora.lora_a[1],
463
+ model_lora.get_lora("up_proj").lora_a)
464
+ assert torch.allclose(packed_lora.lora_b[1],
465
+ model_lora.get_lora("up_proj").lora_b)
466
+
467
+ packed_lora1 = model_lora1.get_lora("gate_up_proj")
468
+ assert packed_lora1 and isinstance(packed_lora1, PackedLoRALayerWeights)
469
+
470
+ assert packed_lora1.lora_a[0] is None
471
+ assert packed_lora1.lora_b[0] is None
472
+ assert torch.allclose(packed_lora1.lora_a[1],
473
+ model_lora1.get_lora("up_proj").lora_a)
474
+ assert torch.allclose(packed_lora1.lora_b[1],
475
+ model_lora1.get_lora("up_proj").lora_b)
tests/lora/test_punica.py
new
+175
@@ -0,0 +1,175 @@
1
+# Based on code from https://github.com/punica-ai/punica
2
+
3
+import pytest
4
+import torch
5
+
6
+import vllm.lora.punica as punica
7
+
8
+
9
+def assert_close(a, b):
10
+ rtol, atol = {
11
+ torch.float16: (5e-3, 5e-3),
12
+ torch.bfloat16: (3e-2, 2e-2),
13
+ torch.float32: (None, None),
14
+ }[a.dtype]
15
+ torch.testing.assert_close(a, b, rtol=rtol, atol=atol)
16
+
17
+
18
+def _lora_ref_impl(
19
+ y_final: torch.Tensor,
20
+ x: torch.Tensor,
21
+ wa_T_all: torch.Tensor,
22
+ wb_T_all: torch.Tensor,
23
+ indicies: torch.LongTensor,
24
+ layer_idx: int,
25
+ scale: float,
26
+):
27
+ y_stage_1 = torch.empty(
28
+ (x.size(0), wa_T_all.size(-2)),
29
+ dtype=torch.float32,
30
+ device=x.device,
31
+ )
32
+ bs = x.shape[0]
33
+ s = torch.tensor(scale, dtype=torch.float32, device=x.device)
34
+ for i, lora_idx in zip(range(bs), indicies.cpu().tolist()):
35
+ xi = x[i].unsqueeze(0).to(torch.float32)
36
+ wa = wa_T_all[lora_idx, layer_idx].transpose(-1, -2).to(torch.float32)
37
+ wb = wb_T_all[lora_idx, layer_idx].transpose(-1, -2).to(torch.float32)
38
+
39
+ tmp = xi @ wa
40
+ y_stage_1[i] = tmp.squeeze(0)
41
+ y_final[i] += (tmp @ wb).squeeze(0) * s
42
+ return y_final, y_stage_1
43
+
44
+
45
+H1 = H2 = [
46
+ 128, 256, 512, 1024, 1280, 2048, 2560, 2752, 3072, 3456, 3584, 4096, 5120,
47
+ 5504, 5632, 6912, 7168, 8192, 9216, 10240, 11008, 13824, 14336, 32000,
48
+ 32256, 32512, 32768, 33024
49
+]
50
+SEED = [0xabcdabcd987]
51
+
52
+
53
+@pytest.mark.parametrize("dtype_str", ["float16", "bfloat16"])
54
+@pytest.mark.parametrize("h1", H1)
55
+@pytest.mark.parametrize("h2", H2)
56
+@pytest.mark.parametrize("seed", SEED)
57
+@torch.inference_mode()
58
+def test_lora_correctness(dtype_str, h1, h2, seed):
59
+ torch.manual_seed(seed)
60
+ num_loras = 4
61
+ num_layers = 1
62
+ r = 8
63
+ bs = 32
64
+ scale = 0.123
65
+ dtype = getattr(torch, dtype_str)
66
+ device = torch.device("cuda")
67
+
68
+ wa_T_all = torch.randn(num_loras,
69
+ num_layers,
70
+ r,
71
+ h1,
72
+ dtype=dtype,
73
+ device=device)
74
+ wb_T_all = torch.randn(num_loras,
75
+ num_layers,
76
+ h2,
77
+ r,
78
+ dtype=dtype,
79
+ device=device)
80
+ indices = torch.randint(num_loras, (bs, ), dtype=torch.long, device=device)
81
+
82
+ for layer_idx in range(num_layers):
83
+ x = torch.randn(bs, h1, dtype=dtype, device=device)
84
+ y = torch.randn(bs, h2, dtype=dtype, device=device)
85
+
86
+ y_ref = y.clone()
87
+ _lora_ref_impl(y_ref, x, wa_T_all, wb_T_all, indices, layer_idx, scale)
88
+
89
+ y_our = y.clone()
90
+ punica.add_lora(y_our, x, wa_T_all, wb_T_all, indices, layer_idx,
91
+ scale)
92
+
93
+ assert_close(y_ref, y_our)
94
+
95
+
96
+@pytest.mark.parametrize("dtype_str", ["float16", "bfloat16"])
97
+@pytest.mark.parametrize("h1", H1)
98
+@pytest.mark.parametrize("h2", H2)
99
+@pytest.mark.parametrize("seed", SEED)
100
+@torch.inference_mode()
101
+def test_lora_correctness_slice(dtype_str, h1, h2, seed):
102
+ if h2 % 3 != 0 or h2 // 3 not in H1:
103
+ pytest.skip("h2 must be divisible by 3 and in supported shapes")
104
+ torch.manual_seed(seed)
105
+ num_loras = 4
106
+ num_layers = 1
107
+ r = 8
108
+ bs = 32
109
+ scale = 0.123
110
+ dtype = getattr(torch, dtype_str)
111
+ device = torch.device("cuda")
112
+
113
+ wa_T_all_0 = torch.randn(num_loras,
114
+ num_layers,
115
+ r,
116
+ h1,
117
+ dtype=dtype,
118
+ device=device)
119
+ wa_T_all_1 = torch.randn(num_loras,
120
+ num_layers,
121
+ r,
122
+ h1,
123
+ dtype=dtype,
124
+ device=device)
125
+ wa_T_all_2 = torch.randn(num_loras,
126
+ num_layers,
127
+ r,
128
+ h1,
129
+ dtype=dtype,
130
+ device=device)
131
+ wb_T_all_0 = torch.randn(num_loras,
132
+ num_layers,
133
+ h2 // 3,
134
+ r,
135
+ dtype=dtype,
136
+ device=device)
137
+ wb_T_all_1 = torch.randn(num_loras,
138
+ num_layers,
139
+ h2 // 3,
140
+ r,
141
+ dtype=dtype,
142
+ device=device)
143
+ wb_T_all_2 = torch.randn(num_loras,
144
+ num_layers,
145
+ h2 // 3,
146
+ r,
147
+ dtype=dtype,
148
+ device=device)
149
+
150
+ indices = torch.randint(num_loras, (bs, ), dtype=torch.long, device=device)
151
+
152
+ for layer_idx in range(num_layers):
153
+ x = torch.randn(bs, h1, dtype=dtype, device=device)
154
+ y = torch.randn(bs, h2, dtype=dtype, device=device)
155
+ s = h2 // 3
156
+
157
+ y_ref = y.clone()
158
+ _lora_ref_impl(y_ref[:, :s], x, wa_T_all_0, wb_T_all_0, indices,
159
+ layer_idx, scale)
160
+ _lora_ref_impl(y_ref[:, s:s * 2], x, wa_T_all_1, wb_T_all_1, indices,
161
+ layer_idx, scale)
162
+ _lora_ref_impl(y_ref[:, s * 2:], x, wa_T_all_2, wb_T_all_2, indices,
163
+ layer_idx, scale)
164
+
165
+ y_our = y.clone()
166
+ punica.add_lora_slice(y_our, x, wa_T_all_0, wb_T_all_0, indices,
167
+ layer_idx, scale, 0, s)
168
+ punica.add_lora_slice(y_our, x, wa_T_all_1, wb_T_all_1, indices,
169
+ layer_idx, scale, s, s)
170
+ punica.add_lora_slice(y_our, x, wa_T_all_2, wb_T_all_2, indices,
171
+ layer_idx, scale, s * 2, s)
172
+
173
+ assert_close(y_ref[:, :s], y_our[:, :s])
174
+ assert_close(y_ref[:, s:s * 2], y_our[:, s:s * 2])
175
+ assert_close(y_ref[:, s * 2:], y_our[:, s * 2:])
tests/lora/test_tokenizer.py
new
+69
@@ -0,0 +1,69 @@
1
+import pytest
2
+from transformers import AutoTokenizer, PreTrainedTokenizerBase
3
+
4
+from vllm.lora.request import LoRARequest
5
+from vllm.transformers_utils.tokenizer import TokenizerGroup, get_lora_tokenizer
6
+
7
+
8
+@pytest.mark.asyncio
9
+async def test_transformers_tokenizer():
10
+ reference_tokenizer = AutoTokenizer.from_pretrained("gpt2")
11
+ tokenizer = TokenizerGroup(
12
+ tokenizer_id="gpt2",
13
+ enable_lora=False,
14
+ max_num_seqs=1,
15
+ max_input_length=None,
16
+ )
17
+ assert reference_tokenizer.encode("prompt") == tokenizer.encode(
18
+ request_id="request_id", prompt="prompt", lora_request=None)
19
+ assert reference_tokenizer.encode(
20
+ "prompt") == await tokenizer.encode_async(request_id="request_id",
21
+ prompt="prompt",
22
+ lora_request=None)
23
+ assert isinstance(tokenizer.get_lora_tokenizer(None),
24
+ PreTrainedTokenizerBase)
25
+ assert tokenizer.get_lora_tokenizer(
26
+ None) == await tokenizer.get_lora_tokenizer_async(None)
27
+
28
+
29
+@pytest.mark.asyncio
30
+async def test_transformers_tokenizer_lora(sql_lora_files):
31
+ reference_tokenizer = AutoTokenizer.from_pretrained(sql_lora_files)
32
+ tokenizer = TokenizerGroup(
33
+ tokenizer_id="gpt2",
34
+ enable_lora=True,
35
+ max_num_seqs=1,
36
+ max_input_length=None,
37
+ )
38
+ lora_request = LoRARequest("1", 1, sql_lora_files)
39
+ assert reference_tokenizer.encode("prompt") == tokenizer.encode(
40
+ request_id="request_id", prompt="prompt", lora_request=lora_request)
41
+ assert reference_tokenizer.encode(
42
+ "prompt") == await tokenizer.encode_async(request_id="request_id",
43
+ prompt="prompt",
44
+ lora_request=lora_request)
45
+ assert isinstance(tokenizer.get_lora_tokenizer(None),
46
+ PreTrainedTokenizerBase)
47
+ assert tokenizer.get_lora_tokenizer(
48
+ None) == await tokenizer.get_lora_tokenizer_async(None)
49
+
50
+ assert isinstance(tokenizer.get_lora_tokenizer(lora_request),
51
+ PreTrainedTokenizerBase)
52
+ assert tokenizer.get_lora_tokenizer(
53
+ lora_request) != tokenizer.get_lora_tokenizer(None)
54
+ assert tokenizer.get_lora_tokenizer(
55
+ lora_request) == await tokenizer.get_lora_tokenizer_async(lora_request)
56
+
57
+
58
+def test_get_lora_tokenizer(sql_lora_files, tmpdir):
59
+ lora_request = None
60
+ tokenizer = get_lora_tokenizer(lora_request)
61
+ assert not tokenizer
62
+
63
+ lora_request = LoRARequest("1", 1, sql_lora_files)
64
+ tokenizer = get_lora_tokenizer(lora_request)
65
+ assert tokenizer.get_added_vocab()
66
+
67
+ lora_request = LoRARequest("1", 1, str(tmpdir))
68
+ tokenizer = get_lora_tokenizer(lora_request)
69
+ assert not tokenizer
tests/lora/test_utils.py
new
+172
@@ -0,0 +1,172 @@
1
+from collections import OrderedDict
2
+
3
+from torch import nn
4
+
5
+from vllm.utils import LRUCache
6
+from vllm.lora.utils import (parse_fine_tuned_lora_name, replace_submodule)
7
+
8
+
9
+def test_parse_fine_tuned_lora_name():
10
+ fixture = {
11
+ ("base_model.model.lm_head.lora_A.weight", "lm_head", True),
12
+ ("base_model.model.lm_head.lora_B.weight", "lm_head", False),
13
+ (
14
+ "base_model.model.model.embed_tokens.lora_embedding_A",
15
+ "model.embed_tokens",
16
+ True,
17
+ ),
18
+ (
19
+ "base_model.model.model.embed_tokens.lora_embedding_B",
20
+ "model.embed_tokens",
21
+ False,
22
+ ),
23
+ (
24
+ "base_model.model.model.layers.9.mlp.down_proj.lora_A.weight",
25
+ "model.layers.9.mlp.down_proj",
26
+ True,
27
+ ),
28
+ (
29
+ "base_model.model.model.layers.9.mlp.down_proj.lora_B.weight",
30
+ "model.layers.9.mlp.down_proj",
31
+ False,
32
+ ),
33
+ }
34
+ for name, module_name, is_lora_a in fixture:
35
+ assert (module_name, is_lora_a) == parse_fine_tuned_lora_name(name)
36
+
37
+
38
+def test_replace_submodule():
39
+ model = nn.Sequential(
40
+ OrderedDict([
41
+ ("dense1", nn.Linear(764, 100)),
42
+ ("act1", nn.ReLU()),
43
+ ("dense2", nn.Linear(100, 50)),
44
+ (
45
+ "seq1",
46
+ nn.Sequential(
47
+ OrderedDict([
48
+ ("dense1", nn.Linear(100, 10)),
49
+ ("dense2", nn.Linear(10, 50)),
50
+ ])),
51
+ ),
52
+ ("act2", nn.ReLU()),
53
+ ("output", nn.Linear(50, 10)),
54
+ ("outact", nn.Sigmoid()),
55
+ ]))
56
+
57
+ sigmoid = nn.Sigmoid()
58
+
59
+ replace_submodule(model, "act1", sigmoid)
60
+ assert dict(model.named_modules())["act1"] == sigmoid
61
+
62
+ dense2 = nn.Linear(1, 5)
63
+ replace_submodule(model, "seq1.dense2", dense2)
64
+ assert dict(model.named_modules())["seq1.dense2"] == dense2
65
+
66
+
67
+class TestLRUCache(LRUCache):
68
+
69
+ def _on_remove(self, key, value):
70
+ if not hasattr(self, "_remove_counter"):
71
+ self._remove_counter = 0
72
+ self._remove_counter += 1
73
+
74
+
75
+def test_lru_cache():
76
+ cache = TestLRUCache(3)
77
+
78
+ cache.put(1, 1)
79
+ assert len(cache) == 1
80
+
81
+ cache.put(1, 1)
82
+ assert len(cache) == 1
83
+
84
+ cache.put(2, 2)
85
+ assert len(cache) == 2
86
+
87
+ cache.put(3, 3)
88
+ assert len(cache) == 3
89
+ assert set(cache.cache) == {1, 2, 3}
90
+
91
+ cache.put(4, 4)
92
+ assert len(cache) == 3
93
+ assert set(cache.cache) == {2, 3, 4}
94
+ assert cache._remove_counter == 1
95
+ assert cache.get(2) == 2
96
+
97
+ cache.put(5, 5)
98
+ assert set(cache.cache) == {2, 4, 5}
99
+ assert cache._remove_counter == 2
100
+
101
+ assert cache.pop(5) == 5
102
+ assert len(cache) == 2
103
+ assert set(cache.cache) == {2, 4}
104
+ assert cache._remove_counter == 3
105
+
106
+ cache.pop(10)
107
+ assert len(cache) == 2
108
+ assert set(cache.cache) == {2, 4}
109
+ assert cache._remove_counter == 3
110
+
111
+ cache.get(10)
112
+ assert len(cache) == 2
113
+ assert set(cache.cache) == {2, 4}
114
+ assert cache._remove_counter == 3
115
+
116
+ cache.put(6, 6)
117
+ assert len(cache) == 3
118
+ assert set(cache.cache) == {2, 4, 6}
119
+ assert 2 in cache
120
+ assert 4 in cache
121
+ assert 6 in cache
122
+
123
+ cache.remove_oldest()
124
+ assert len(cache) == 2
125
+ assert set(cache.cache) == {2, 6}
126
+ assert cache._remove_counter == 4
127
+
128
+ cache.clear()
129
+ assert len(cache) == 0
130
+ assert cache._remove_counter == 6
131
+
132
+ cache._remove_counter = 0
133
+
134
+ cache[1] = 1
135
+ assert len(cache) == 1
136
+
137
+ cache[1] = 1
138
+ assert len(cache) == 1
139
+
140
+ cache[2] = 2
141
+ assert len(cache) == 2
142
+
143
+ cache[3] = 3
144
+ assert len(cache) == 3
145
+ assert set(cache.cache) == {1, 2, 3}
146
+
147
+ cache[4] = 4
148
+ assert len(cache) == 3
149
+ assert set(cache.cache) == {2, 3, 4}
150
+ assert cache._remove_counter == 1
151
+ assert cache[2] == 2
152
+
153
+ cache[5] = 5
154
+ assert set(cache.cache) == {2, 4, 5}
155
+ assert cache._remove_counter == 2
156
+
157
+ del cache[5]
158
+ assert len(cache) == 2
159
+ assert set(cache.cache) == {2, 4}
160
+ assert cache._remove_counter == 3
161
+
162
+ cache.pop(10)
163
+ assert len(cache) == 2
164
+ assert set(cache.cache) == {2, 4}
165
+ assert cache._remove_counter == 3
166
+
167
+ cache[6] = 6
168
+ assert len(cache) == 3
169
+ assert set(cache.cache) == {2, 4, 6}
170
+ assert 2 in cache
171
+ assert 4 in cache
172
+ assert 6 in cache
tests/lora/test_worker.py
new
+61
@@ -0,0 +1,61 @@
1
+import os
2
+import random
3
+import tempfile
4
+from unittest.mock import patch
5
+
6
+from vllm.lora.models import LoRAMapping
7
+from vllm.lora.request import LoRARequest
8
+from vllm.config import ModelConfig, ParallelConfig, SchedulerConfig, LoRAConfig
9
+from vllm.worker.worker import Worker
10
+
11
+
12
+@patch.dict(os.environ, {"RANK": "0"})
13
+def test_worker_apply_lora(sql_lora_files):
14
+ worker = Worker(
15
+ model_config=ModelConfig(
16
+ "meta-llama/Llama-2-7b-hf",
17
+ "meta-llama/Llama-2-7b-hf",
18
+ tokenizer_mode="auto",
19
+ trust_remote_code=False,
20
+ download_dir=None,
21
+ load_format="dummy",
22
+ seed=0,
23
+ dtype="float16",
24
+ revision=None,
25
+ ),
26
+ parallel_config=ParallelConfig(1, 1, False),
27
+ scheduler_config=SchedulerConfig(32, 32, 32, 256),
28
+ local_rank=0,
29
+ rank=0,
30
+ lora_config=LoRAConfig(max_lora_rank=8, max_cpu_loras=32,
31
+ max_loras=32),
32
+ distributed_init_method=f"file://{tempfile.mkstemp()[1]}",
33
+ )
34
+ worker.init_model()
35
+ worker.load_model()
36
+
37
+ worker.model_runner.set_active_loras([], LoRAMapping([], []))
38
+ assert worker.list_loras() == set()
39
+
40
+ n_loras = 32
41
+ lora_requests = [
42
+ LoRARequest(str(i + 1), i + 1, sql_lora_files) for i in range(n_loras)
43
+ ]
44
+
45
+ worker.model_runner.set_active_loras(lora_requests, LoRAMapping([], []))
46
+ assert worker.list_loras() == {
47
+ lora_request.lora_int_id
48
+ for lora_request in lora_requests
49
+ }
50
+
51
+ for i in range(32):
52
+ random.seed(i)
53
+ iter_lora_requests = random.choices(lora_requests,
54
+ k=random.randint(1, n_loras))
55
+ random.shuffle(iter_lora_requests)
56
+ iter_lora_requests = iter_lora_requests[:-random.randint(0, n_loras)]
57
+ worker.model_runner.set_active_loras(iter_lora_requests,
58
+ LoRAMapping([], []))
59
+ assert worker.list_loras().issuperset(
60
+ {lora_request.lora_int_id
61
+ for lora_request in iter_lora_requests})
tests/lora/utils.py
new
+88
@@ -0,0 +1,88 @@
1
+from typing import List, Optional
2
+
3
+import torch
4
+
5
+from vllm.lora.lora import LoRALayerWeights, PackedLoRALayerWeights
6
+
7
+
8
+class DummyLoRAManager:
9
+
10
+ def __init__(self):
11
+ super().__init__()
12
+ self._loras = {}
13
+
14
+ def set_module_lora(self, module_name: str, lora: LoRALayerWeights):
15
+ self._loras[module_name] = lora
16
+
17
+ def get_module_lora(self, module_name: str) -> Optional[LoRALayerWeights]:
18
+ return self._loras.get(module_name, None)
19
+
20
+ def init_random_lora(self,
21
+ module_name: str,
22
+ weight: torch.Tensor,
23
+ rank: int = 8,
24
+ generate_embeddings_tensor: int = 0):
25
+ lora = LoRALayerWeights(
26
+ module_name,
27
+ rank=rank,
28
+ lora_alpha=1,
29
+ lora_a=torch.rand([weight.shape[1], rank],
30
+ dtype=weight.dtype,
31
+ device="cuda"),
32
+ lora_b=torch.rand([rank, weight.shape[0]],
33
+ dtype=weight.dtype,
34
+ device="cuda"),
35
+ )
36
+ if generate_embeddings_tensor:
37
+ lora.embeddings_tensor = torch.rand(5,
38
+ generate_embeddings_tensor,
39
+ dtype=weight.dtype,
40
+ device="cuda")
41
+ self.set_module_lora(module_name, lora)
42
+
43
+ return lora
44
+
45
+ def init_lora(self,
46
+ module_name: str,
47
+ input_dim: int,
48
+ output_dim: int,
49
+ rank=8,
50
+ noop=False,
51
+ embeddings_tensor=None):
52
+ lora = LoRALayerWeights(
53
+ module_name,
54
+ rank=rank,
55
+ lora_alpha=1,
56
+ lora_a=torch.rand([input_dim, rank], device="cuda"),
57
+ lora_b=torch.rand([rank, output_dim], device="cuda"),
58
+ embeddings_tensor=embeddings_tensor,
59
+ )
60
+ self.set_module_lora(module_name, lora)
61
+ return lora
62
+
63
+ def reset_lora(self):
64
+ self._loras = {}
65
+
66
+ def init_packed_lora(
67
+ self,
68
+ module_name: str,
69
+ input_dim: int,
70
+ output_dims: List[int],
71
+ noop_lora_index: List[int] = None,
72
+ rank=8,
73
+ ):
74
+ base_loras = []
75
+ noop_lora_index = set(noop_lora_index or [])
76
+
77
+ for i, out_dim in enumerate(output_dims):
78
+ base_lora = self.init_lora(
79
+ module_name + "_000_" + str(i),
80
+ input_dim,
81
+ out_dim,
82
+ rank=rank,
83
+ noop=i in noop_lora_index,
84
+ )
85
+ base_loras.append(base_lora)
86
+ packed_lora = PackedLoRALayerWeights.pack(base_loras)
87
+ self.set_module_lora(module_name, packed_lora)
88
+ return packed_lora
tests/samplers/test_sampler.py
+7
-6
@@ -19,10 +19,11 @@ class MockLogitsSampler(Sampler):
19
self.fake_logits = fake_logits
20
21
def forward(self, *args, **kwargs):
22
- with patch("vllm.model_executor.layers.sampler._prune_hidden_states",
23
- lambda x, y: x), patch(
24
- "vllm.model_executor.layers.sampler._get_logits",
25
- lambda *args, **kwargs: self.fake_logits):
22
+ with patch(
23
+ "vllm.model_executor.layers.sampler._prune_hidden_states",
24
+ lambda x, y: x), patch(
25
+ "vllm.model_executor.layers.sampler.Sampler._get_logits",
26
+ lambda *args, **kwargs: self.fake_logits):
27
return super().forward(*args, **kwargs)
28
29
@@ -38,7 +39,7 @@ def _prepare_test(
39
device=input_tensor.device,
40
dtype=input_tensor.dtype)
41
sampler = MockLogitsSampler(32000, fake_logits)
41
- model_runner = ModelRunner(None, None, None)
42
+ model_runner = ModelRunner(None, None, None, None)
43
return input_tensor, fake_logits, sampler, model_runner
44
45
@@ -266,7 +267,7 @@ def test_sampler_top_k_top_p(seed: int):
267
device=input_tensor.device,
268
dtype=input_tensor.dtype)
269
sampler = MockLogitsSampler(32000, fake_logits)
269
- model_runner = ModelRunner(None, None, None)
270
+ model_runner = ModelRunner(None, None, None, None)
271
272
generation_model = GenerationMixin()
273
generation_config = GenerationConfig(top_k=top_k,
tests/worker/spec_decode/utils.py
+2
-2
@@ -83,8 +83,8 @@ def create_worker(cls: type,
83
enforce_eager=enforce_eager,
84
)
85
86
- (model_config, cache_config, parallel_config,
87
- scheduler_config) = engine_args.create_engine_configs()
86
+ (model_config, cache_config, parallel_config, scheduler_config,
87
+ _) = engine_args.create_engine_configs()
88
89
distributed_init_method = get_distributed_init_method(
90
get_ip(), get_open_port())
tests/worker/test_model_runner.py
+2
-2
@@ -6,7 +6,7 @@ from vllm.worker.model_runner import ModelRunner
6
7
8
def test_prepare_prompt():
9
- model_runner = ModelRunner(None, None, None)
9
+ model_runner = ModelRunner(None, None, None, None)
10
model_runner.set_block_size(16)
11
12
batch_size = random.randint(1, 256)
@@ -33,7 +33,7 @@ def test_prepare_prompt():
33
expected_selected_token_indices.append(selected_token_start_idx +
34
prompt_len - 1)
35
selected_token_start_idx += max_seq_len
36
- input_tokens, input_positions, _, return_prompt_lens, _ = (
36
+ input_tokens, input_positions, _, return_prompt_lens, _, _, _, _ = (
37
model_runner._prepare_prompt(seq_group_metadata_list))
38
assert return_prompt_lens == prompt_lens
39
sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
vllm/config.py
+50
-1
@@ -1,4 +1,5 @@
1
-from typing import Optional, Union
1
+from typing import Optional, Union, ClassVar
2
+from dataclasses import dataclass
3
import os
4
5
import torch
@@ -397,6 +398,54 @@ class SchedulerConfig:
398
f"({self.max_num_seqs}).")
399
400
401
+@dataclass
402
+class LoRAConfig:
403
+ max_lora_rank: int
404
+ max_loras: int
405
+ max_cpu_loras: Optional[int] = None
406
+ lora_dtype: Optional[torch.dtype] = None
407
+ lora_extra_vocab_size: int = 256
408
+ # This is a constant.
409
+ lora_vocab_padding_size: ClassVar[int] = 256
410
+
411
+ def __post_init__(self):
412
+ # Keep this in sync with csrc/punica/bgmv/bgmv_config.h
413
+ possible_max_ranks = (8, 16, 32, 64)
414
+ possible_lora_extra_vocab_size = (0, 256, 512)
415
+ if self.max_lora_rank not in possible_max_ranks:
416
+ raise ValueError(
417
+ f"max_lora_rank ({self.max_lora_rank}) must be one of "
418
+ f"{possible_max_ranks}.")
419
+ if self.lora_extra_vocab_size not in possible_lora_extra_vocab_size:
420
+ raise ValueError(
421
+ f"lora_extra_vocab_size ({self.lora_extra_vocab_size}) "
422
+ f"must be one of {possible_lora_extra_vocab_size}.")
423
+ if self.max_loras < 1:
424
+ raise ValueError(f"max_loras ({self.max_loras}) must be >= 1.")
425
+ if self.max_cpu_loras is None:
426
+ self.max_cpu_loras = self.max_loras
427
+ elif self.max_cpu_loras < self.max_loras:
428
+ raise ValueError(
429
+ f"max_cpu_loras ({self.max_cpu_loras}) must be >= "
430
+ f"max_num_seqs ({self.max_loras})")
431
+
432
+ def verify_with_model_config(self, model_config: ModelConfig):
433
+ if self.lora_dtype in (None, "auto"):
434
+ self.lora_dtype = model_config.dtype
435
+ elif isinstance(self.lora_dtype, str):
436
+ self.lora_dtype = getattr(torch, self.lora_dtype)
437
+ if model_config.quantization is not None:
438
+ raise ValueError(
439
+ "LoRA is not supported with quantized models yet.")
440
+
441
+ def verify_with_scheduler_config(self, scheduler_config: SchedulerConfig):
442
+ if scheduler_config.max_num_batched_tokens > 65528:
443
+ raise ValueError(
444
+ "Due to limitations of the custom LoRA CUDA kernel, "
445
+ "max_num_batched_tokens must be <= 65528 when "
446
+ "LoRA is enabled.")
447
+
448
+
449
_STR_DTYPE_TO_TORCH_DTYPE = {
450
"half": torch.float16,
451
"float16": torch.float16,
vllm/core/scheduler.py
+68
-5
@@ -1,11 +1,12 @@
1
from collections import deque
2
import enum
3
import time
4
-from typing import Deque, Dict, Iterable, List, Optional, Tuple, Union
4
+from typing import Deque, Dict, Iterable, List, Optional, Tuple, Union, Set
5
6
-from vllm.config import CacheConfig, SchedulerConfig
6
+from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig
7
from vllm.core.block_manager import AllocStatus, BlockSpaceManager
8
from vllm.core.policy import PolicyFactory
9
+from vllm.lora.request import LoRARequest
10
from vllm.logger import init_logger
11
from vllm.sequence import (Sequence, SequenceData, SequenceGroup,
12
SequenceGroupMetadata, SequenceStatus)
@@ -49,11 +50,25 @@ class SchedulerOutputs:
50
assert not (blocks_to_swap_in and blocks_to_swap_out)
51
self.ignored_seq_groups = ignored_seq_groups
52
53
+ self.num_loras = len(self.lora_requests)
54
+ if self.num_loras > 0:
55
+ self._sort_by_lora_ids()
56
+
57
def is_empty(self) -> bool:
58
# NOTE: We do not consider the ignored sequence groups.
59
return (not self.scheduled_seq_groups and not self.blocks_to_swap_in
60
and not self.blocks_to_swap_out and not self.blocks_to_copy)
61
62
+ def _sort_by_lora_ids(self) -> bool:
63
+ self.scheduled_seq_groups = sorted(
64
+ self.scheduled_seq_groups,
65
+ key=lambda g: (g.lora_request.lora_int_id
66
+ if g.lora_request else 0, g.request_id))
67
+
68
+ @property
69
+ def lora_requests(self) -> Set[LoRARequest]:
70
+ return {g.lora_request for g in self.scheduled_seq_groups}
71
+
72
73
class Scheduler:
74
@@ -61,9 +76,14 @@ class Scheduler:
76
self,
77
scheduler_config: SchedulerConfig,
78
cache_config: CacheConfig,
79
+ lora_config: Optional[LoRAConfig],
80
) -> None:
81
self.scheduler_config = scheduler_config
82
self.cache_config = cache_config
83
+ # Note for LoRA scheduling: the current policy is extremely
84
+ # simple and NOT fair. It can lead to starvation of some
85
+ # LoRAs. This should be improved in the future.
86
+ self.lora_config = lora_config
87
88
self.prompt_limit = min(self.scheduler_config.max_model_len,
89
self.scheduler_config.max_num_batched_tokens)
@@ -87,6 +107,10 @@ class Scheduler:
107
# Sequence groups in the SWAPPED state.
108
self.swapped: Deque[SequenceGroup] = deque()
109
110
+ @property
111
+ def lora_enabled(self) -> bool:
112
+ return bool(self.lora_config)
113
+
114
def add_seq_group(self, seq_group: SequenceGroup) -> None:
115
# Add sequence groups to the waiting queue.
116
self.waiting.append(seq_group)
@@ -150,14 +174,17 @@ class Scheduler:
174
# requests in the generation phase.
175
num_curr_seqs = sum(seq_group.get_max_num_running_seqs()
176
for seq_group in self.running)
177
+ curr_loras = set(
178
+ seq_group.lora_int_id
179
+ for seq_group in self.running) if self.lora_enabled else None
180
seq_lens: List[int] = []
181
182
# Optimization: We do not sort the waiting queue since the preempted
183
# sequence groups are added to the front and the new sequence groups
184
# are added to the back.
185
+ leftover_waiting_sequences = deque()
186
while self.waiting:
187
seq_group = self.waiting[0]
160
-
188
waiting_seqs = seq_group.get_seqs(
189
status=SequenceStatus.WAITING)
190
assert len(waiting_seqs) == 1, (
@@ -188,6 +215,17 @@ class Scheduler:
215
self.waiting.popleft()
216
continue
217
218
+ lora_int_id = 0
219
+ if self.lora_enabled:
220
+ lora_int_id = seq_group.lora_int_id
221
+ if lora_int_id > 0 and lora_int_id not in curr_loras and len(
222
+ curr_loras) >= self.lora_config.max_loras:
223
+ # We don't have a space for another LoRA, so
224
+ # we ignore this request for now.
225
+ leftover_waiting_sequences.appendleft(seq_group)
226
+ self.waiting.popleft()
227
+ continue
228
+
229
# If the number of batched tokens exceeds the limit, stop.
230
new_seq_lens = seq_lens + [num_prompt_tokens]
231
num_batched_tokens = len(new_seq_lens) * max(new_seq_lens)
@@ -207,12 +245,16 @@ class Scheduler:
245
break
246
seq_lens = new_seq_lens
247
210
- seq_group = self.waiting.popleft()
248
+ if lora_int_id > 0:
249
+ curr_loras.add(lora_int_id)
250
+ self.waiting.popleft()
251
self._allocate(seq_group)
252
self.running.append(seq_group)
253
num_curr_seqs += num_new_seqs
254
scheduled.append(seq_group)
255
256
+ self.waiting.extendleft(leftover_waiting_sequences)
257
+
258
if scheduled or ignored_seq_groups:
259
scheduler_outputs = SchedulerOutputs(
260
scheduled_seq_groups=scheduled,
@@ -260,9 +302,25 @@ class Scheduler:
302
if not preempted:
303
num_curr_seqs = sum(seq_group.get_max_num_running_seqs()
304
for seq_group in self.running)
305
+ curr_loras = set(
306
+ seq_group.lora_int_id
307
+ for seq_group in self.running) if self.lora_enabled else None
308
+
309
+ leftover_swapped = deque()
310
311
while self.swapped:
312
seq_group = self.swapped[0]
313
+ lora_int_id = 0
314
+ if self.lora_enabled:
315
+ lora_int_id = seq_group.lora_int_id
316
+ if lora_int_id > 0 and lora_int_id not in curr_loras and len(
317
+ curr_loras) >= self.lora_config.max_loras:
318
+ # We don't have a space for another LoRA, so
319
+ # we ignore this request for now.
320
+ leftover_swapped.appendleft(seq_group)
321
+ self.swapped.popleft()
322
+ continue
323
+
324
# If the sequence group cannot be swapped in, stop.
325
if not self.block_manager.can_swap_in(seq_group):
326
break
@@ -274,12 +332,16 @@ class Scheduler:
332
self.scheduler_config.max_num_seqs):
333
break
334
277
- seq_group = self.swapped.popleft()
335
+ if lora_int_id > 0:
336
+ curr_loras.add(lora_int_id)
337
+ self.swapped.popleft()
338
self._swap_in(seq_group, blocks_to_swap_in)
339
self._append_slot(seq_group, blocks_to_copy)
340
num_curr_seqs += num_new_seqs
341
self.running.append(seq_group)
342
343
+ self.swapped.extendleft(leftover_swapped)
344
+
345
# Each sequence in the generation phase only takes one token slot.
346
# Therefore, the number of batched tokens is equal to the number of
347
# sequences in the RUNNING state.
@@ -320,6 +382,7 @@ class Scheduler:
382
seq_data=seq_data,
383
sampling_params=seq_group.sampling_params,
384
block_tables=block_tables,
385
+ lora_request=seq_group.lora_request,
386
prefix=seq_group.prefix,
387
)
388
seq_group_metadata_list.append(seq_group_metadata)
vllm/engine/arg_utils.py
+50
-3
@@ -4,7 +4,7 @@ from dataclasses import dataclass
4
from typing import Optional, Tuple
5
6
from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
7
- SchedulerConfig)
7
+ SchedulerConfig, LoRAConfig)
8
9
10
@dataclass
@@ -35,6 +35,12 @@ class EngineArgs:
35
quantization: Optional[str] = None
36
enforce_eager: bool = False
37
max_context_len_to_capture: int = 8192
38
+ enable_lora: bool = False
39
+ max_loras: int = 1
40
+ max_lora_rank: int = 16
41
+ lora_extra_vocab_size: int = 256
42
+ lora_dtype = 'auto'
43
+ max_cpu_loras: Optional[int] = None
44
45
def __post_init__(self):
46
if self.tokenizer is None:
@@ -202,6 +208,39 @@ class EngineArgs:
208
help='maximum context length covered by CUDA '
209
'graphs. When a sequence has context length '
210
'larger than this, we fall back to eager mode.')
211
+ # LoRA related configs
212
+ parser.add_argument('--enable-lora',
213
+ action='store_true',
214
+ help='If True, enable handling of LoRA adapters.')
215
+ parser.add_argument('--max-loras',
216
+ type=int,
217
+ default=EngineArgs.max_loras,
218
+ help='Max number of LoRAs in a single batch.')
219
+ parser.add_argument('--max-lora-rank',
220
+ type=int,
221
+ default=EngineArgs.max_lora_rank,
222
+ help='Max LoRA rank.')
223
+ parser.add_argument(
224
+ '--lora-extra-vocab-size',
225
+ type=int,
226
+ default=EngineArgs.lora_extra_vocab_size,
227
+ help=('Maximum size of extra vocabulary that can be '
228
+ 'present in a LoRA adapter (added to the base '
229
+ 'model vocabulary).'))
230
+ parser.add_argument(
231
+ '--lora-dtype',
232
+ type=str,
233
+ default=EngineArgs.lora_dtype,
234
+ choices=['auto', 'float16', 'bfloat16', 'float32'],
235
+ help=('Data type for LoRA. If auto, will default to '
236
+ 'base model dtype.'))
237
+ parser.add_argument(
238
+ '--max-cpu-loras',
239
+ type=int,
240
+ default=EngineArgs.max_cpu_loras,
241
+ help=('Maximum number of LoRAs to store in CPU memory. '
242
+ 'Must be >= than max_num_seqs. '
243
+ 'Defaults to max_num_seqs.'))
244
return parser
245
246
@classmethod
@@ -214,7 +253,8 @@ class EngineArgs:
253
254
def create_engine_configs(
255
self,
217
- ) -> Tuple[ModelConfig, CacheConfig, ParallelConfig, SchedulerConfig]:
256
+ ) -> Tuple[ModelConfig, CacheConfig, ParallelConfig, SchedulerConfig,
257
+ Optional[LoRAConfig]]:
258
model_config = ModelConfig(self.model, self.tokenizer,
259
self.tokenizer_mode, self.trust_remote_code,
260
self.download_dir, self.load_format,
@@ -234,7 +274,14 @@ class EngineArgs:
274
self.max_num_seqs,
275
model_config.max_model_len,
276
self.max_paddings)
237
- return model_config, cache_config, parallel_config, scheduler_config
277
+ lora_config = LoRAConfig(
278
+ max_lora_rank=self.max_lora_rank,
279
+ max_loras=self.max_loras,
280
+ lora_extra_vocab_size=self.lora_extra_vocab_size,
281
+ lora_dtype=self.lora_dtype,
282
+ max_cpu_loras=self.max_cpu_loras if self.max_cpu_loras
283
+ and self.max_cpu_loras > 0 else None) if self.enable_lora else None
284
+ return model_config, cache_config, parallel_config, scheduler_config, lora_config
285
286
287
@dataclass
vllm/engine/async_llm_engine.py
+71
-8
@@ -4,6 +4,7 @@ from functools import partial
4
from typing import (Any, Dict, Iterable, List, Optional, Set, Tuple, Type,
5
Union, AsyncIterator)
6
7
+from vllm.lora.request import LoRARequest
8
from vllm.config import ModelConfig
9
from vllm.engine.arg_utils import AsyncEngineArgs
10
from vllm.engine.llm_engine import LLMEngine
@@ -203,6 +204,52 @@ class _AsyncLLMEngine(LLMEngine):
204
205
return self._process_model_outputs(output, scheduler_outputs)
206
207
+ async def encode_request_async(
208
+ self,
209
+ request_id: str, # pylint: disable=unused-argument
210
+ prompt: Optional[str],
211
+ prompt_token_ids: Optional[List[int]] = None,
212
+ lora_request: Optional[LoRARequest] = None,
213
+ ):
214
+ if prompt_token_ids is None:
215
+ assert prompt is not None
216
+ prompt_token_ids = await self.tokenizer.encode_async(
217
+ request_id=request_id,
218
+ prompt=prompt,
219
+ lora_request=lora_request)
220
+ return prompt_token_ids
221
+
222
+ async def add_request_async(
223
+ self,
224
+ request_id: str,
225
+ prompt: Optional[str],
226
+ sampling_params: SamplingParams,
227
+ prompt_token_ids: Optional[List[int]] = None,
228
+ arrival_time: Optional[float] = None,
229
+ lora_request: Optional[LoRARequest] = None,
230
+ prefix_pos: Optional[int] = None,
231
+ ) -> None:
232
+ if lora_request is not None and not self.lora_config:
233
+ raise ValueError(f"Got lora_request {lora_request} but LoRA is "
234
+ "not enabled!")
235
+ if arrival_time is None:
236
+ arrival_time = time.time()
237
+ prompt_token_ids = await self.encode_request_async(
238
+ request_id=request_id,
239
+ prompt=prompt,
240
+ prompt_token_ids=prompt_token_ids,
241
+ lora_request=lora_request)
242
+
243
+ return self.add_request(
244
+ request_id,
245
+ prompt=prompt,
246
+ prompt_token_ids=prompt_token_ids,
247
+ sampling_params=sampling_params,
248
+ arrival_time=arrival_time,
249
+ lora_request=lora_request,
250
+ prefix_pos=prefix_pos,
251
+ )
252
+
253
async def _run_workers_async(
254
self,
255
method: str,
@@ -332,7 +379,7 @@ class AsyncLLMEngine:
379
if self.engine_use_ray:
380
await self.engine.add_request.remote(**new_request)
381
else:
335
- self.engine.add_request(**new_request)
382
+ await self.engine.add_request_async(**new_request)
383
384
if finished_requests:
385
await self._engine_abort(finished_requests)
@@ -371,6 +418,7 @@ class AsyncLLMEngine:
418
sampling_params: SamplingParams,
419
prompt_token_ids: Optional[List[int]] = None,
420
arrival_time: Optional[float] = None,
421
+ lora_request: Optional[LoRARequest] = None,
422
prefix_pos: Optional[int] = None,
423
) -> AsyncStream:
424
if self.log_requests:
@@ -386,7 +434,8 @@ class AsyncLLMEngine:
434
f"prompt: {shortened_prompt!r}, "
435
f"prefix_pos: {prefix_pos},"
436
f"sampling params: {sampling_params}, "
389
- f"prompt token ids: {shortened_token_ids}.")
437
+ f"prompt token ids: {shortened_token_ids}, "
438
+ f"lora_request: {lora_request}.")
439
440
if not self.is_running:
441
if self.start_engine_loop:
@@ -398,12 +447,21 @@ class AsyncLLMEngine:
447
"error that caused the background loop to stop "
448
"(AsyncEngineDeadError).")
449
450
+ if arrival_time is None:
451
+ arrival_time = time.time()
452
+ prompt_token_ids = await self.engine.encode_request_async(
453
+ request_id=request_id,
454
+ prompt=prompt,
455
+ prompt_token_ids=prompt_token_ids,
456
+ lora_request=lora_request)
457
+
458
stream = self._request_tracker.add_request(
459
request_id,
460
prompt=prompt,
461
sampling_params=sampling_params,
462
prompt_token_ids=prompt_token_ids,
463
arrival_time=arrival_time,
464
+ lora_request=lora_request,
465
prefix_pos=prefix_pos)
466
467
return stream
@@ -414,6 +472,7 @@ class AsyncLLMEngine:
472
sampling_params: SamplingParams,
473
request_id: str,
474
prompt_token_ids: Optional[List[int]] = None,
475
+ lora_request: Optional[LoRARequest] = None,
476
prefix_pos: Optional[int] = None,
477
) -> AsyncIterator[RequestOutput]:
478
"""Generate outputs for a request.
@@ -429,6 +488,7 @@ class AsyncLLMEngine:
488
request_id: The unique id of the request.
489
prompt_token_ids: The token IDs of the prompt. If None, we
490
use the tokenizer to convert the prompts to token IDs.
491
+ lora_request: LoRA request to use for generation, if any.
492
prefix_pos: If not None, we use the given position as the prefix
493
position for each prompt. We will cache the prefix's KV
494
cache and reuse it for the next request with the same prefix.
@@ -487,12 +547,15 @@ class AsyncLLMEngine:
547
arrival_time = time.monotonic()
548
549
try:
490
- stream = await self.add_request(request_id,
491
- prompt,
492
- sampling_params,
493
- prompt_token_ids=prompt_token_ids,
494
- arrival_time=arrival_time,
495
- prefix_pos=prefix_pos)
550
+ stream = await self.add_request(
551
+ request_id,
552
+ prompt,
553
+ sampling_params,
554
+ prompt_token_ids=prompt_token_ids,
555
+ arrival_time=arrival_time,
556
+ lora_request=lora_request,
557
+ prefix_pos=prefix_pos,
558
+ )
559
560
async for request_output in stream:
561
yield request_output
vllm/engine/llm_engine.py
+87
-24
@@ -5,8 +5,9 @@ import time
5
from typing import (TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Tuple,
6
Union)
7
8
+from vllm.lora.request import LoRARequest
9
from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
9
- SchedulerConfig)
10
+ SchedulerConfig, LoRAConfig)
11
from vllm.core.scheduler import Scheduler, SchedulerOutputs
12
from vllm.engine.arg_utils import EngineArgs
13
from vllm.engine.metrics import record_metrics
@@ -17,7 +18,7 @@ from vllm.sampling_params import SamplingParams
18
from vllm.sequence import (SamplerOutput, Sequence, SequenceGroup,
19
SequenceGroupOutput, SequenceOutput, SequenceStatus)
20
from vllm.transformers_utils.tokenizer import (detokenize_incrementally,
20
- get_tokenizer)
21
+ TokenizerGroup)
22
from vllm.utils import Counter, set_cuda_visible_devices, get_ip, get_open_port, get_distributed_init_method
23
24
if ray:
@@ -64,6 +65,7 @@ class LLMEngine:
65
cache_config: CacheConfig,
66
parallel_config: ParallelConfig,
67
scheduler_config: SchedulerConfig,
68
+ lora_config: Optional[LoRAConfig],
69
placement_group: Optional["PlacementGroup"],
70
log_stats: bool,
71
) -> None:
@@ -87,17 +89,13 @@ class LLMEngine:
89
90
self.model_config = model_config
91
self.cache_config = cache_config
92
+ self.lora_config = lora_config
93
self.parallel_config = parallel_config
94
self.scheduler_config = scheduler_config
95
self.log_stats = log_stats
96
self._verify_args()
97
95
- self.tokenizer = get_tokenizer(
96
- model_config.tokenizer,
97
- tokenizer_mode=model_config.tokenizer_mode,
98
- trust_remote_code=model_config.trust_remote_code,
99
- tokenizer_revision=model_config.tokenizer_revision,
100
- revision=model_config.revision)
98
+ self._init_tokenizer()
99
self.seq_counter = Counter()
100
101
# Create the parallel GPU workers.
@@ -114,7 +112,7 @@ class LLMEngine:
112
self._init_cache()
113
114
# Create the scheduler.
117
- self.scheduler = Scheduler(scheduler_config, cache_config)
115
+ self.scheduler = Scheduler(scheduler_config, cache_config, lora_config)
116
117
# Logging.
118
self.last_logging_time = 0.0
@@ -123,6 +121,9 @@ class LLMEngine:
121
# List of (timestamp, num_tokens)
122
self.num_generation_tokens: List[Tuple[float, int]] = []
123
124
+ def get_tokenizer_for_seq(self, sequence: Sequence):
125
+ return self.tokenizer.get_lora_tokenizer(sequence.lora_request)
126
+
127
def _init_workers(self):
128
# Lazy import the Worker to avoid importing torch.cuda/xformers
129
# before CUDA_VISIBLE_DEVICES is set in the Worker
@@ -141,11 +142,24 @@ class LLMEngine:
142
local_rank=0,
143
rank=0,
144
distributed_init_method=distributed_init_method,
145
+ lora_config=self.lora_config,
146
is_driver_worker=True,
147
)
148
self._run_workers("init_model")
149
self._run_workers("load_model")
150
151
+ def _init_tokenizer(self, **tokenizer_init_kwargs):
152
+ init_kwargs = dict(
153
+ enable_lora=bool(self.lora_config),
154
+ max_num_seqs=self.scheduler_config.max_num_seqs,
155
+ max_input_length=None,
156
+ tokenizer_mode=self.model_config.tokenizer_mode,
157
+ trust_remote_code=self.model_config.trust_remote_code,
158
+ revision=self.model_config.tokenizer_revision)
159
+ init_kwargs.update(tokenizer_init_kwargs)
160
+ self.tokenizer: TokenizerGroup = TokenizerGroup(
161
+ self.model_config.tokenizer, **init_kwargs)
162
+
163
def _init_workers_ray(self, placement_group: "PlacementGroup",
164
**ray_remote_kwargs):
165
if self.parallel_config.tensor_parallel_size == 1:
@@ -233,6 +247,7 @@ class LLMEngine:
247
local_rank,
248
rank,
249
distributed_init_method,
250
+ lora_config=self.lora_config,
251
))
252
253
driver_rank = 0
@@ -244,6 +259,7 @@ class LLMEngine:
259
driver_local_rank,
260
driver_rank,
261
distributed_init_method,
262
+ lora_config=self.lora_config,
263
is_driver_worker=True,
264
)
265
@@ -257,6 +273,10 @@ class LLMEngine:
273
def _verify_args(self) -> None:
274
self.model_config.verify_with_parallel_config(self.parallel_config)
275
self.cache_config.verify_with_parallel_config(self.parallel_config)
276
+ if self.lora_config:
277
+ self.lora_config.verify_with_model_config(self.model_config)
278
+ self.lora_config.verify_with_scheduler_config(
279
+ self.scheduler_config)
280
281
def _init_cache(self) -> None:
282
"""Profiles the memory usage and initializes the KV cache.
@@ -332,6 +352,20 @@ class LLMEngine:
352
log_stats=not engine_args.disable_log_stats)
353
return engine
354
355
+ def encode_request(
356
+ self,
357
+ request_id: str, # pylint: disable=unused-argument
358
+ prompt: Optional[str],
359
+ prompt_token_ids: Optional[List[int]] = None,
360
+ lora_request: Optional[LoRARequest] = None,
361
+ ):
362
+ if prompt_token_ids is None:
363
+ assert prompt is not None
364
+ prompt_token_ids = self.tokenizer.encode(request_id=request_id,
365
+ prompt=prompt,
366
+ lora_request=lora_request)
367
+ return prompt_token_ids
368
+
369
def add_request(
370
self,
371
request_id: str,
@@ -339,6 +373,7 @@ class LLMEngine:
373
sampling_params: SamplingParams,
374
prompt_token_ids: Optional[List[int]] = None,
375
arrival_time: Optional[float] = None,
376
+ lora_request: Optional[LoRARequest] = None,
377
prefix_pos: Optional[int] = None,
378
) -> None:
379
"""Add a request to the engine's request pool.
@@ -386,24 +421,31 @@ class LLMEngine:
421
>>> # continue the request processing
422
>>> ...
423
"""
424
+ if lora_request is not None and not self.lora_config:
425
+ raise ValueError(f"Got lora_request {lora_request} but LoRA is "
426
+ "not enabled!")
427
if arrival_time is None:
428
arrival_time = time.monotonic()
391
- if prompt_token_ids is None:
392
- assert prompt is not None
393
- prompt_token_ids = self.tokenizer.encode(prompt)
429
+ prompt_token_ids = self.encode_request(
430
+ request_id=request_id,
431
+ prompt=prompt,
432
+ prompt_token_ids=prompt_token_ids,
433
+ lora_request=lora_request)
434
435
# Create the sequences.
436
block_size = self.cache_config.block_size
437
seq_id = next(self.seq_counter)
398
- seq = Sequence(seq_id, prompt, prompt_token_ids, block_size)
438
+ seq = Sequence(seq_id, prompt, prompt_token_ids, block_size,
439
+ lora_request)
440
441
# Check whether the input specifies prefix
442
prefix = self.scheduler.prefix_pool.add_or_get_prefix(
402
- prompt_token_ids[:prefix_pos]) if prefix_pos is not None else None
443
+ prompt_token_ids[:prefix_pos], lora_request.lora_int_id
444
+ if lora_request else 0) if prefix_pos is not None else None
445
446
# Create the sequence group.
447
seq_group = SequenceGroup(request_id, [seq], sampling_params,
406
- arrival_time, prefix)
448
+ arrival_time, lora_request, prefix)
449
450
# Add the sequence group to the scheduler.
451
self.scheduler.add_seq_group(seq_group)
@@ -453,11 +495,13 @@ class LLMEngine:
495
496
current_worst_score = (current_worst_seq.get_beam_search_score(
497
length_penalty=length_penalty,
456
- eos_token_id=self.tokenizer.eos_token_id))
498
+ eos_token_id=self.get_tokenizer_for_seq(
499
+ current_worst_seq).eos_token_id))
500
if early_stopping is False:
501
highest_attainable_score = (best_running_seq.get_beam_search_score(
502
length_penalty=length_penalty,
460
- eos_token_id=self.tokenizer.eos_token_id))
503
+ eos_token_id=self.get_tokenizer_for_seq(
504
+ best_running_seq).eos_token_id))
505
else:
506
assert early_stopping == "never"
507
if length_penalty > 0.0:
@@ -471,7 +515,8 @@ class LLMEngine:
515
highest_attainable_score = (
516
best_running_seq.get_beam_search_score(
517
length_penalty=length_penalty,
474
- eos_token_id=self.tokenizer.eos_token_id,
518
+ eos_token_id=self.get_tokenizer_for_seq(
519
+ best_running_seq).eos_token_id,
520
seq_len=max_possible_length))
521
else:
522
# Otherwise, beam search will prefer shorter sequences. The
@@ -480,7 +525,8 @@ class LLMEngine:
525
highest_attainable_score = (
526
best_running_seq.get_beam_search_score(
527
length_penalty=length_penalty,
483
- eos_token_id=self.tokenizer.eos_token_id))
528
+ eos_token_id=self.get_tokenizer_for_seq(
529
+ best_running_seq).eos_token_id))
530
return current_worst_score >= highest_attainable_score
531
532
def _process_sequence_group_outputs(self, seq_group: SequenceGroup,
@@ -571,7 +617,7 @@ class LLMEngine:
617
# Sort the finished sequences by their scores.
618
all_finished_seqs.sort(key=lambda x: x[0].get_beam_search_score(
619
length_penalty=length_penalty,
574
- eos_token_id=self.tokenizer.eos_token_id),
620
+ eos_token_id=self.get_tokenizer_for_seq(x[0]).eos_token_id),
621
reverse=True)
622
for seq, parent, is_new in all_finished_seqs[:beam_width]:
623
if is_new:
@@ -599,7 +645,7 @@ class LLMEngine:
645
# Sort the running sequences by their scores.
646
running_child_seqs.sort(key=lambda x: x[0].get_beam_search_score(
647
length_penalty=length_penalty,
602
- eos_token_id=self.tokenizer.eos_token_id),
648
+ eos_token_id=self.get_tokenizer_for_seq(x[0]).eos_token_id),
649
reverse=True)
650
651
# Check if we can stop the beam search.
@@ -837,7 +883,7 @@ class LLMEngine:
883
"""Decodes the new token for a sequence."""
884
(new_tokens, new_output_text, prefix_offset,
885
read_offset) = detokenize_incrementally(
840
- self.tokenizer,
886
+ self.get_tokenizer_for_seq(seq),
887
all_input_ids=seq.get_token_ids(),
888
prev_tokens=seq.tokens,
889
prefix_offset=seq.prefix_offset,
@@ -879,11 +925,28 @@ class LLMEngine:
925
return
926
927
# Check if the sequence has generated the EOS token.
882
- if ((not sampling_params.ignore_eos)
883
- and seq.get_last_token_id() == self.tokenizer.eos_token_id):
928
+ if ((not sampling_params.ignore_eos) and seq.get_last_token_id()
929
+ == self.get_tokenizer_for_seq(seq).eos_token_id):
930
seq.status = SequenceStatus.FINISHED_STOPPED
931
return
932
933
+ def add_lora(self, lora_request: LoRARequest) -> bool:
934
+ assert lora_request.lora_int_id > 0, "lora_id must be greater than 0."
935
+ return self._run_workers(
936
+ "add_lora",
937
+ lora_request=lora_request,
938
+ )
939
+
940
+ def remove_lora(self, lora_id: int) -> bool:
941
+ assert lora_id > 0, "lora_id must be greater than 0."
942
+ return self._run_workers(
943
+ "remove_lora",
944
+ lora_id=lora_id,
945
+ )
946
+
947
+ def list_loras(self) -> List[int]:
948
+ return self._run_workers("list_loras")
949
+
950
def _run_workers(
951
self,
952
method: str,
vllm/entrypoints/llm.py
+10
-1
@@ -3,6 +3,7 @@ from typing import List, Optional, Union
3
from tqdm import tqdm
4
from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
5
6
+from vllm.lora.request import LoRARequest
7
from vllm.engine.arg_utils import EngineArgs
8
from vllm.engine.llm_engine import LLMEngine
9
from vllm.outputs import RequestOutput
@@ -122,6 +123,7 @@ class LLM:
123
prompt_token_ids: Optional[List[List[int]]] = None,
124
prefix_pos: Optional[Union[int, List[int]]] = None,
125
use_tqdm: bool = True,
126
+ lora_request: Optional[LoRARequest] = None,
127
) -> List[RequestOutput]:
128
"""Generates the completions for the input prompts.
129
@@ -141,6 +143,7 @@ class LLM:
143
This is an experimental feature, and may be replaced with
144
automatic prefix caching in the future.
145
use_tqdm: Whether to use tqdm to display the progress bar.
146
+ lora_request: LoRA request to use for generation, if any.
147
148
Returns:
149
A list of `RequestOutput` objects containing the generated
@@ -168,7 +171,11 @@ class LLM:
171
prefix_pos_i = prefix_pos[i] if prefix_pos is not None else None
172
token_ids = None if prompt_token_ids is None else prompt_token_ids[
173
i]
171
- self._add_request(prompt, sampling_params, token_ids, prefix_pos_i)
174
+ self._add_request(prompt,
175
+ sampling_params,
176
+ token_ids,
177
+ lora_request=lora_request,
178
+ prefix_pos=prefix_pos_i)
179
return self._run_engine(use_tqdm)
180
181
def _add_request(
@@ -176,6 +183,7 @@ class LLM:
183
prompt: Optional[str],
184
sampling_params: SamplingParams,
185
prompt_token_ids: Optional[List[int]],
186
+ lora_request: Optional[LoRARequest] = None,
187
prefix_pos: Optional[int] = None,
188
) -> None:
189
request_id = str(next(self.request_counter))
@@ -183,6 +191,7 @@ class LLM:
191
prompt,
192
sampling_params,
193
prompt_token_ids,
194
+ lora_request=lora_request,
195
prefix_pos=prefix_pos)
196
197
def _run_engine(self, use_tqdm: bool) -> List[RequestOutput]:
vllm/lora/__init__.py
vllm/lora/layers.py
new
+975
@@ -0,0 +1,975 @@
1
+# pylint: disable=unused-argument
2
+import math
3
+from dataclasses import dataclass
4
+from typing import TYPE_CHECKING, List, Optional, Tuple
5
+
6
+import torch
7
+import torch.nn as nn
8
+import torch.nn.functional as F
9
+from transformers import PretrainedConfig
10
+
11
+from vllm.config import LoRAConfig
12
+from vllm.lora.punica import add_lora, add_lora_slice, bgmv
13
+from vllm.model_executor.layers.sampler import Sampler
14
+from vllm.model_executor.parallel_utils.communication_op import (
15
+ tensor_model_parallel_all_gather,
16
+ tensor_model_parallel_all_reduce,
17
+ tensor_model_parallel_gather,
18
+)
19
+from vllm.model_executor.layers.linear import (ColumnParallelLinear,
20
+ RowParallelLinear,
21
+ QKVParallelLinear,
22
+ MergedColumnParallelLinear)
23
+from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding, ParallelLMHead
24
+from vllm.model_executor.parallel_utils.parallel_state import (
25
+ get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
26
+from vllm.model_executor.parallel_utils.utils import split_tensor_along_last_dim
27
+
28
+if TYPE_CHECKING:
29
+ pass
30
+
31
+
32
+def _apply_lora(
33
+ x: torch.Tensor,
34
+ lora_a_stacked: torch.Tensor,
35
+ lora_b_stacked: torch.Tensor,
36
+ indices: torch.Tensor,
37
+ output: torch.Tensor,
38
+):
39
+ """Applies lora to each input.
40
+
41
+ This method applies all loras to each input. It uses the
42
+ indices vector to determine which lora yields the
43
+ correct output. An index of -1 means no lora should be
44
+ applied. This method adds the final lora results to the
45
+ output.
46
+
47
+ Input shapes:
48
+ x: (batch_size, hidden_dim)
49
+ lora_a_stacked: (num_loras, lora_rank, hidden_dim)
50
+ lora_b_stacked: (num_loras, output_dim, lora_rank)
51
+ indices: (batch_size)
52
+ output: (batch_size, output_dim)
53
+ """
54
+ org_output = output
55
+ x = x.view(-1, x.shape[-1])
56
+ output = output.view(-1, output.shape[-1])
57
+ indices = indices.view(-1)
58
+ add_lora(output, x, lora_a_stacked, lora_b_stacked, indices, 0, 1.0)
59
+ return output.view_as(org_output)
60
+
61
+
62
+def _apply_lora_packed_nslice(
63
+ x: torch.Tensor,
64
+ lora_a_stacked: Tuple[torch.Tensor, torch.Tensor, torch.Tensor],
65
+ lora_b_stacked: Tuple[torch.Tensor, torch.Tensor, torch.Tensor],
66
+ indices: torch.Tensor,
67
+ output: torch.Tensor,
68
+ output_slices: Tuple[int, ...],
69
+):
70
+ """Applies lora to each input.
71
+
72
+ This method applies all loras to each input. It uses the
73
+ indices vector to determine which lora yields the
74
+ correct output. An index of -1 means no lora should be
75
+ applied. This method adds the final lora results to the
76
+ output.
77
+
78
+ This method is used for layers that are composed of multiple sublayers
79
+ (slices) packed together.
80
+
81
+ Input shapes:
82
+ x: (batch_size, hidden_dim)
83
+ lora_a_stacked: 3 element tuple of (num_loras, lora_rank, hidden_dim)
84
+ lora_b_stacked: 3 element tuple of (num_loras, output_dim, lora_rank)
85
+ indices: (batch_size)
86
+ output: (batch_size, q_slice_size + 2*kv_slice_size)
87
+ output_slices: n-1 element tuple of (slice_size...), where n is number of slices
88
+ """
89
+ org_output = output
90
+ x = x.view(-1, x.shape[-1])
91
+ output = output.view(-1, output.shape[-1])
92
+ indices = indices.view(-1)
93
+ offset_left = 0
94
+ for slice_idx in range(len(output_slices)):
95
+ add_lora_slice(output, x, lora_a_stacked[slice_idx],
96
+ lora_b_stacked[slice_idx], indices, 0, 1.0, offset_left,
97
+ output_slices[slice_idx])
98
+ offset_left += output_slices[slice_idx]
99
+ return output.view_as(org_output)
100
+
101
+
102
+@dataclass
103
+class LoRAMapping:
104
+ # Per every token in input_ids:
105
+ index_mapping: Tuple[int, ...]
106
+ # Per sampled token:
107
+ prompt_mapping: Tuple[int, ...]
108
+
109
+ def __post_init__(self):
110
+ self.index_mapping = tuple(self.index_mapping)
111
+ self.prompt_mapping = tuple(self.prompt_mapping)
112
+
113
+
114
+class BaseLayerWithLoRA(nn.Module):
115
+
116
+ def create_lora_weights(self, max_loras: int, lora_config: LoRAConfig,
117
+ model_config: PretrainedConfig) -> None:
118
+ """Initializes lora matrices."""
119
+ ...
120
+
121
+ def reset_lora(self, index: int):
122
+ """Resets the lora weights at index back to 0."""
123
+ ...
124
+
125
+ def set_lora(
126
+ self,
127
+ index: int,
128
+ lora_a: torch.Tensor,
129
+ lora_b: torch.Tensor,
130
+ embeddings_tensor: Optional[torch.Tensor],
131
+ ):
132
+ """Overwrites lora tensors at index."""
133
+ ...
134
+
135
+ def set_mapping(
136
+ self,
137
+ base_indices: torch.Tensor,
138
+ sampler_indices: torch.Tensor,
139
+ sampler_indices_padded: torch.Tensor,
140
+ embeddings_indices: torch.Tensor,
141
+ indices_len: List[int],
142
+ ):
143
+ """Sets the mapping indices."""
144
+ ...
145
+
146
+
147
+class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA):
148
+
149
+ def __init__(self, base_layer: VocabParallelEmbedding) -> None:
150
+ super().__init__()
151
+ self.base_layer = base_layer
152
+
153
+ def create_lora_weights(
154
+ self,
155
+ max_loras: int,
156
+ lora_config: LoRAConfig,
157
+ model_config: Optional[PretrainedConfig] = None) -> None:
158
+
159
+ lora_vocab_start_idx = self.base_layer.org_vocab_size
160
+ weights_idx = None
161
+ if self.base_layer.vocab_end_index > lora_vocab_start_idx:
162
+ # We can start adding lora weights
163
+ weights_idx = max(
164
+ lora_vocab_start_idx - self.base_layer.vocab_start_index, 0)
165
+ self.embeddings_slice = (self.base_layer.vocab_start_index -
166
+ self.base_layer.org_vocab_size +
167
+ weights_idx,
168
+ self.base_layer.vocab_end_index -
169
+ self.base_layer.org_vocab_size)
170
+ self.embeddings_weights = self.base_layer.weight.data[weights_idx:]
171
+ self.embeddings_weights.fill_(0)
172
+ else:
173
+ self.embeddings_slice = None
174
+ self.embeddings_weights = None
175
+
176
+ self.embeddings_tensors = torch.zeros(
177
+ (
178
+ max_loras,
179
+ lora_config.lora_extra_vocab_size,
180
+ self.base_layer.embedding_dim,
181
+ ),
182
+ dtype=self.base_layer.weight.dtype,
183
+ device=self.base_layer.weight.device,
184
+ )
185
+ self.lora_a_stacked = torch.zeros(
186
+ (
187
+ max_loras,
188
+ self.base_layer.org_vocab_size +
189
+ lora_config.lora_extra_vocab_size,
190
+ lora_config.max_lora_rank,
191
+ ),
192
+ dtype=lora_config.lora_dtype,
193
+ device=self.base_layer.weight.device,
194
+ )
195
+ self.lora_b_stacked = torch.zeros(
196
+ (
197
+ max_loras,
198
+ 1,
199
+ self.base_layer.embedding_dim,
200
+ lora_config.max_lora_rank,
201
+ ),
202
+ dtype=lora_config.lora_dtype,
203
+ device=self.base_layer.weight.device,
204
+ )
205
+ self.lora_a_stacked_2d = self.lora_a_stacked.view(
206
+ self.lora_a_stacked.shape[0] * self.lora_a_stacked.shape[1],
207
+ self.lora_a_stacked.shape[2],
208
+ )
209
+ self.indices: Optional[torch.Tensor] = None
210
+ self.indices_len: Optional[List[int]] = None
211
+ self.embeddings_indices = None
212
+
213
+ def reset_lora(self, index: int):
214
+ self.lora_a_stacked[index] = 0
215
+ self.lora_b_stacked[index] = 0
216
+ self.embeddings_tensors[index] = 0
217
+
218
+ def set_lora(
219
+ self,
220
+ index: int,
221
+ lora_a: torch.Tensor,
222
+ lora_b: torch.Tensor,
223
+ embeddings_tensor: Optional[torch.Tensor],
224
+ ):
225
+ self.reset_lora(index)
226
+ self.lora_a_stacked[index, :lora_a.shape[0], :lora_a.shape[1]].copy_(
227
+ lora_a, non_blocking=True)
228
+ self.lora_b_stacked[index,
229
+ 0, :lora_b.shape[1], :lora_b.shape[0]].copy_(
230
+ lora_b.T, non_blocking=True)
231
+ if embeddings_tensor is not None:
232
+ self.embeddings_tensors[
233
+ index, :embeddings_tensor.shape[0], :embeddings_tensor.
234
+ shape[1]].copy_(embeddings_tensor, non_blocking=True)
235
+ if self.embeddings_slice is not None:
236
+ # TODO(yard1): Optimize this copy, we don't need to copy
237
+ # everything, just the modified part
238
+ embeddings = self.embeddings_tensors.view(
239
+ self.embeddings_tensors.shape[0] *
240
+ self.embeddings_tensors.shape[1],
241
+ self.embeddings_tensors.shape[2]
242
+ )[self.embeddings_slice[0]:self.embeddings_slice[1]]
243
+ self.embeddings_weights[:embeddings.shape[0]].copy_(embeddings)
244
+
245
+ def set_mapping(
246
+ self,
247
+ base_indices: torch.Tensor,
248
+ sampler_indices: torch.Tensor,
249
+ sampler_indices_padded: torch.Tensor,
250
+ embeddings_indices: torch.Tensor,
251
+ indices_len: List[int],
252
+ ):
253
+ self.indices = base_indices
254
+ self.embeddings_indices = embeddings_indices
255
+ self.indices_len = indices_len
256
+
257
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
258
+ added_tokens_mask = x > self.base_layer.org_vocab_size - 1
259
+ indices = self.embeddings_indices[1][:self.indices_len[3]].view_as(x)
260
+ full_lora_a_embeddings = F.embedding(
261
+ x + indices,
262
+ self.lora_a_stacked_2d,
263
+ )
264
+ indices = self.embeddings_indices[0][:self.indices_len[3]].view_as(x)
265
+ full_output = self.base_layer.forward(
266
+ x.add_(indices * added_tokens_mask))
267
+
268
+ full_output_org = full_output
269
+ if full_output.ndim == 3:
270
+ full_output = full_output.view(
271
+ full_output.shape[0] * full_output.shape[1], -1)
272
+ if full_lora_a_embeddings.ndim == 3:
273
+ full_lora_a_embeddings = full_lora_a_embeddings.view(
274
+ full_lora_a_embeddings.shape[0] *
275
+ full_lora_a_embeddings.shape[1], -1)
276
+ bgmv(full_output, full_lora_a_embeddings, self.lora_b_stacked,
277
+ self.indices[:self.indices_len[0]], 0, 1.0)
278
+ return full_output.view_as(full_output_org)
279
+
280
+
281
+class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
282
+
283
+ def __init__(self, base_layer: ColumnParallelLinear) -> None:
284
+ super().__init__()
285
+ self.base_layer = base_layer
286
+
287
+ def create_lora_weights(
288
+ self,
289
+ max_loras: int,
290
+ lora_config: LoRAConfig,
291
+ model_config: Optional[PretrainedConfig] = None) -> None:
292
+ self.lora_a_stacked = torch.zeros(
293
+ max_loras,
294
+ 1,
295
+ lora_config.max_lora_rank,
296
+ self.base_layer.weight.shape[1],
297
+ dtype=lora_config.lora_dtype,
298
+ device=self.base_layer.weight.device,
299
+ )
300
+ self.lora_b_stacked = torch.zeros(
301
+ max_loras,
302
+ 1,
303
+ self.base_layer.weight.shape[0],
304
+ lora_config.max_lora_rank,
305
+ dtype=lora_config.lora_dtype,
306
+ device=self.base_layer.weight.device,
307
+ )
308
+
309
+ self.indices: Optional[torch.Tensor] = None
310
+ self.indices_len: Optional[List[int]] = None
311
+ self.output_dim = self.lora_b_stacked.shape[1]
312
+
313
+ def reset_lora(self, index: int):
314
+ self.lora_a_stacked[index] = 0
315
+ self.lora_b_stacked[index] = 0
316
+
317
+ def set_lora(
318
+ self,
319
+ index: int,
320
+ lora_a: torch.Tensor,
321
+ lora_b: torch.Tensor,
322
+ embeddings_tensor: Optional[torch.Tensor],
323
+ ):
324
+ self.reset_lora(index)
325
+
326
+ self.lora_a_stacked[index,
327
+ 0, :lora_a.shape[1], :lora_a.shape[0]].copy_(
328
+ lora_a.T, non_blocking=True)
329
+ self.lora_b_stacked[index,
330
+ 0, :lora_b.shape[1], :lora_b.shape[0]].copy_(
331
+ lora_b.T, non_blocking=True)
332
+
333
+ def set_mapping(
334
+ self,
335
+ base_indices: torch.Tensor,
336
+ sampler_indices: torch.Tensor,
337
+ sampler_indices_padded: torch.Tensor,
338
+ embeddings_indices: torch.Tensor,
339
+ indices_len: List[int],
340
+ ):
341
+ self.indices = base_indices
342
+ self.indices_len = indices_len
343
+
344
+ def apply_weights(self, x: torch.Tensor,
345
+ bias: Optional[torch.Tensor]) -> torch.Tensor:
346
+ output = self.base_layer.linear_method.apply_weights(
347
+ self.base_layer.linear_weights, x, bias)
348
+ _apply_lora(
349
+ x,
350
+ self.lora_a_stacked,
351
+ self.lora_b_stacked,
352
+ self.indices[:self.indices_len[0]],
353
+ output,
354
+ )
355
+ return output
356
+
357
+ def forward(self, input_):
358
+ """Forward of ColumnParallelLinear
359
+
360
+ Args:
361
+ input_: Tensor whose last dimension is `input_size`.
362
+
363
+ Returns:
364
+ - output
365
+ - bias
366
+ """
367
+ bias = (self.base_layer.bias
368
+ if not self.base_layer.skip_bias_add else None)
369
+
370
+ # Matrix multiply.
371
+ output_parallel = self.apply_weights(input_, bias)
372
+ if self.base_layer.gather_output:
373
+ # All-gather across the partitions.
374
+ output = tensor_model_parallel_all_gather(output_parallel)
375
+ else:
376
+ output = output_parallel
377
+ output_bias = (self.base_layer.bias
378
+ if self.base_layer.skip_bias_add else None)
379
+ return output, output_bias
380
+
381
+ @property
382
+ def linear_weights(self):
383
+ return self.base_layer.linear_weights
384
+
385
+
386
+class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
387
+ """ColumnParallelLinear layer that is composed of 2 sublayers (slices)
388
+ packed together (eg. gate_proj + up_proj -> gate_up_proj).
389
+
390
+ This means we have 2 LoRAs, each applied to one half of the layer.
391
+
392
+ Both slices must have the same size.
393
+ """
394
+
395
+ def __init__(self, base_layer: MergedColumnParallelLinear) -> None:
396
+ super().__init__(base_layer)
397
+
398
+ def create_lora_weights(
399
+ self,
400
+ max_loras: int,
401
+ lora_config: LoRAConfig,
402
+ model_config: Optional[PretrainedConfig] = None) -> None:
403
+ n_slices = 2
404
+ if not (len(self.base_layer.output_sizes) == n_slices
405
+ and self.base_layer.output_sizes[0]
406
+ == self.base_layer.output_sizes[1]):
407
+ raise ValueError(
408
+ "LoRAColumnParallelLinear2Slice requires 2 slices with "
409
+ "the same size.")
410
+ self.tp_size = get_tensor_model_parallel_world_size()
411
+
412
+ self.lora_a_stacked = tuple(
413
+ torch.zeros(
414
+ max_loras,
415
+ 1,
416
+ lora_config.max_lora_rank,
417
+ self.base_layer.weight.shape[1],
418
+ dtype=lora_config.lora_dtype,
419
+ device=self.base_layer.weight.device,
420
+ ) for _ in range(n_slices))
421
+ self.lora_b_stacked = tuple(
422
+ torch.zeros(
423
+ max_loras,
424
+ 1,
425
+ self.base_layer.weight.shape[0] // 2,
426
+ lora_config.max_lora_rank,
427
+ dtype=lora_config.lora_dtype,
428
+ device=self.base_layer.weight.device,
429
+ ) for _ in range(n_slices))
430
+
431
+ self.indices: Optional[torch.Tensor] = None
432
+ self.output_dim = self.lora_b_stacked[0].shape[2]
433
+
434
+ def reset_lora(self, index: int):
435
+ self.lora_a_stacked[0][index] = 0
436
+ self.lora_a_stacked[1][index] = 0
437
+ self.lora_b_stacked[0][index] = 0
438
+ self.lora_b_stacked[1][index] = 0
439
+
440
+ def set_lora(
441
+ self,
442
+ index: int,
443
+ lora_a: torch.Tensor,
444
+ lora_b: torch.Tensor,
445
+ embeddings_tensor: Optional[torch.Tensor],
446
+ ):
447
+ self.reset_lora(index)
448
+
449
+ if self.tp_size > 1:
450
+ tensor_model_parallel_rank = get_tensor_model_parallel_rank()
451
+ shard_size = self.output_dim
452
+ start_idx = tensor_model_parallel_rank * shard_size
453
+ end_idx = (tensor_model_parallel_rank + 1) * shard_size
454
+ lora_b = lora_b[0][:,
455
+ start_idx:end_idx], lora_b[1][:,
456
+ start_idx:end_idx]
457
+
458
+ if lora_a[0] is not None:
459
+ self.lora_a_stacked[0][
460
+ index, 0, :lora_a[0].shape[1], :lora_a[0].shape[0]].copy_(
461
+ lora_a[0].T, non_blocking=True)
462
+ self.lora_b_stacked[0][
463
+ index, 0, :lora_b[0].shape[1], :lora_b[0].shape[0]].copy_(
464
+ lora_b[0].T, non_blocking=True)
465
+ if lora_a[1] is not None:
466
+ self.lora_a_stacked[1][
467
+ index, 0, :lora_a[1].shape[1], :lora_a[1].shape[0]].copy_(
468
+ lora_a[1].T, non_blocking=True)
469
+ self.lora_b_stacked[1][
470
+ index, 0, :lora_b[1].shape[1], :lora_b[1].shape[0]].copy_(
471
+ lora_b[1].T, non_blocking=True)
472
+
473
+ def apply_weights(self, x: torch.Tensor,
474
+ bias: Optional[torch.Tensor]) -> torch.Tensor:
475
+ output = self.base_layer.linear_method.apply_weights(
476
+ self.base_layer.linear_weights, x, bias)
477
+ _apply_lora_packed_nslice(
478
+ x,
479
+ self.lora_a_stacked,
480
+ self.lora_b_stacked,
481
+ self.indices[:self.indices_len[0]],
482
+ output,
483
+ (self.output_dim, self.output_dim),
484
+ )
485
+ return output
486
+
487
+
488
+class QKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
489
+ """ColumnParallelLinear layer that is composed of 3 sublayers (slices)
490
+ packed together in qkv proj fashion
491
+ (q_proj + k_proj + v_proj -> qkv_proj).
492
+
493
+ This means we have 3 LoRAs, each applied to one slice of the layer.
494
+
495
+ Q slice may have different shape than K and V slices (which both have
496
+ the same shape).
497
+ """
498
+
499
+ def __init__(self, base_layer: QKVParallelLinear) -> None:
500
+ super().__init__(base_layer)
501
+
502
+ def create_lora_weights(
503
+ self,
504
+ max_loras: int,
505
+ lora_config: LoRAConfig,
506
+ model_config: Optional[PretrainedConfig] = None) -> None:
507
+ self.tp_size = get_tensor_model_parallel_world_size()
508
+ tp_rank = get_tensor_model_parallel_rank()
509
+ self.q_proj_shard_size = (self.base_layer.num_heads *
510
+ self.base_layer.head_size)
511
+ self.kv_proj_shard_size = (self.base_layer.num_kv_heads *
512
+ self.base_layer.head_size)
513
+ self.q_shard_id = tp_rank
514
+ self.kv_shard_id = tp_rank // self.base_layer.num_kv_head_replicas
515
+
516
+ # q, k, v
517
+ self.lora_a_stacked = (
518
+ torch.zeros(
519
+ max_loras,
520
+ 1,
521
+ lora_config.max_lora_rank,
522
+ self.base_layer.weight.shape[1],
523
+ dtype=lora_config.lora_dtype,
524
+ device=self.base_layer.weight.device,
525
+ ),
526
+ torch.zeros(
527
+ max_loras,
528
+ 1,
529
+ lora_config.max_lora_rank,
530
+ self.base_layer.weight.shape[1],
531
+ dtype=lora_config.lora_dtype,
532
+ device=self.base_layer.weight.device,
533
+ ),
534
+ torch.zeros(
535
+ max_loras,
536
+ 1,
537
+ lora_config.max_lora_rank,
538
+ self.base_layer.weight.shape[1],
539
+ dtype=lora_config.lora_dtype,
540
+ device=self.base_layer.weight.device,
541
+ ),
542
+ )
543
+ self.lora_b_stacked = (
544
+ torch.zeros(
545
+ max_loras,
546
+ 1,
547
+ self.q_proj_shard_size,
548
+ lora_config.max_lora_rank,
549
+ dtype=lora_config.lora_dtype,
550
+ device=self.base_layer.weight.device,
551
+ ),
552
+ torch.zeros(
553
+ max_loras,
554
+ 1,
555
+ self.kv_proj_shard_size,
556
+ lora_config.max_lora_rank,
557
+ dtype=lora_config.lora_dtype,
558
+ device=self.base_layer.weight.device,
559
+ ),
560
+ torch.zeros(
561
+ max_loras,
562
+ 1,
563
+ self.kv_proj_shard_size,
564
+ lora_config.max_lora_rank,
565
+ dtype=lora_config.lora_dtype,
566
+ device=self.base_layer.weight.device,
567
+ ),
568
+ )
569
+
570
+ self.output_slices = (self.q_proj_shard_size, self.kv_proj_shard_size,
571
+ self.kv_proj_shard_size)
572
+ self.packed_indices: Optional[torch.Tensor] = None
573
+ self.standard_indices: Optional[torch.Tensor] = None
574
+ self.indices_len: Optional[List[int]] = None
575
+
576
+ def reset_lora(self, index: int):
577
+ self.lora_a_stacked[0][index] = 0
578
+ self.lora_b_stacked[0][index] = 0
579
+ self.lora_a_stacked[1][index] = 0
580
+ self.lora_b_stacked[1][index] = 0
581
+ self.lora_a_stacked[2][index] = 0
582
+ self.lora_b_stacked[2][index] = 0
583
+
584
+ def set_lora(
585
+ self,
586
+ index: int,
587
+ lora_a: torch.Tensor,
588
+ lora_b: torch.Tensor,
589
+ embeddings_tensor: Optional[torch.Tensor],
590
+ ):
591
+ self.reset_lora(index)
592
+
593
+ if self.tp_size > 1:
594
+ if lora_b[0] is not None:
595
+ lora_b_q = lora_b[0][:, self.q_proj_shard_size *
596
+ self.q_shard_id:self.q_proj_shard_size *
597
+ (self.q_shard_id + 1)]
598
+ self.lora_b_stacked[0][
599
+ index, 0, :lora_b_q.shape[1], :lora_b_q.shape[0]].copy_(
600
+ lora_b_q.T, non_blocking=True)
601
+ if lora_b[1] is not None:
602
+ lora_b_k = lora_b[1][:, self.kv_proj_shard_size *
603
+ self.kv_shard_id:self.kv_proj_shard_size *
604
+ (self.kv_shard_id + 1)]
605
+ self.lora_b_stacked[1][
606
+ index, 0, :lora_b_k.shape[1], :lora_b_k.shape[0]].copy_(
607
+ lora_b_k.T, non_blocking=True)
608
+ if lora_b[2] is not None:
609
+ lora_b_v = lora_b[2][:, self.kv_proj_shard_size *
610
+ self.kv_shard_id:self.kv_proj_shard_size *
611
+ (self.kv_shard_id + 1)]
612
+ self.lora_b_stacked[2][
613
+ index, 0, :lora_b_v.shape[1], :lora_b_v.shape[0]].copy_(
614
+ lora_b_v.T, non_blocking=True)
615
+ else:
616
+ if lora_b[0] is not None:
617
+ self.lora_b_stacked[0][
618
+ index, 0, :lora_b[0].shape[1], :lora_b[0].shape[0]].copy_(
619
+ lora_b[0].T, non_blocking=True)
620
+ if lora_b[1] is not None:
621
+ self.lora_b_stacked[1][
622
+ index, 0, :lora_b[1].shape[1], :lora_b[1].shape[0]].copy_(
623
+ lora_b[1].T, non_blocking=True)
624
+ if lora_b[2] is not None:
625
+ self.lora_b_stacked[2][
626
+ index, 0, :lora_b[2].shape[1], :lora_b[2].shape[0]].copy_(
627
+ lora_b[2].T, non_blocking=True)
628
+
629
+ if lora_a[0] is not None:
630
+ self.lora_a_stacked[0][
631
+ index, 0, :lora_a[0].shape[1], :lora_a[0].shape[0]].copy_(
632
+ lora_a[0].T, non_blocking=True)
633
+ if lora_a[1] is not None:
634
+ self.lora_a_stacked[1][
635
+ index, 0, :lora_a[1].shape[1], :lora_a[1].shape[0]].copy_(
636
+ lora_a[1].T, non_blocking=True)
637
+ if lora_a[2] is not None:
638
+ self.lora_a_stacked[2][
639
+ index, 0, :lora_a[2].shape[1], :lora_a[2].shape[0]].copy_(
640
+ lora_a[2].T, non_blocking=True)
641
+
642
+ def apply_weights(self, x: torch.Tensor,
643
+ bias: Optional[torch.Tensor]) -> torch.Tensor:
644
+ output = self.base_layer.linear_method.apply_weights(
645
+ self.base_layer.linear_weights, x, bias)
646
+ _apply_lora_packed_nslice(
647
+ x,
648
+ self.lora_a_stacked,
649
+ self.lora_b_stacked,
650
+ self.indices[:self.indices_len[0]],
651
+ output,
652
+ self.output_slices,
653
+ )
654
+ return output
655
+
656
+
657
+class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
658
+
659
+ def __init__(self, base_layer: RowParallelLinear) -> None:
660
+ super().__init__()
661
+ self.base_layer = base_layer
662
+
663
+ def create_lora_weights(
664
+ self,
665
+ max_loras: int,
666
+ lora_config: LoRAConfig,
667
+ model_config: Optional[PretrainedConfig] = None) -> None:
668
+ self.lora_a_stacked = torch.zeros(
669
+ (
670
+ max_loras,
671
+ 1,
672
+ lora_config.max_lora_rank,
673
+ self.base_layer.weight.shape[1],
674
+ ),
675
+ dtype=lora_config.lora_dtype,
676
+ device=self.base_layer.weight.device,
677
+ )
678
+ self.lora_b_stacked = torch.zeros(
679
+ (
680
+ max_loras,
681
+ 1,
682
+ self.base_layer.weight.shape[0],
683
+ lora_config.max_lora_rank,
684
+ ),
685
+ dtype=lora_config.lora_dtype,
686
+ device=self.base_layer.weight.device,
687
+ )
688
+ self.indices: Optional[torch.Tensor] = None
689
+ self.indices_len: Optional[List[int]] = None
690
+
691
+ def reset_lora(self, index: int):
692
+ self.lora_a_stacked[index] = 0
693
+ self.lora_b_stacked[index] = 0
694
+
695
+ def set_lora(
696
+ self,
697
+ index: int,
698
+ lora_a: torch.Tensor,
699
+ lora_b: torch.Tensor,
700
+ embeddings_tensor: Optional[torch.Tensor],
701
+ ):
702
+ self.reset_lora(index)
703
+ if self.base_layer.tp_size > 1:
704
+ tensor_model_parallel_rank = get_tensor_model_parallel_rank()
705
+ shard_size = self.base_layer.weight.shape[1]
706
+ start_idx = tensor_model_parallel_rank * shard_size
707
+ end_idx = (tensor_model_parallel_rank + 1) * shard_size
708
+ lora_a = lora_a[start_idx:end_idx, :]
709
+
710
+ self.lora_a_stacked[index,
711
+ 0, :lora_a.shape[1], :lora_a.shape[0]].copy_(
712
+ lora_a.T, non_blocking=True)
713
+ self.lora_b_stacked[index,
714
+ 0, :lora_b.shape[1], :lora_b.shape[0]].copy_(
715
+ lora_b.T, non_blocking=True)
716
+
717
+ def set_mapping(
718
+ self,
719
+ base_indices: torch.Tensor,
720
+ sampler_indices: torch.Tensor,
721
+ sampler_indices_padded: torch.Tensor,
722
+ embeddings_indices: torch.Tensor,
723
+ indices_len: List[int],
724
+ ):
725
+ self.indices = base_indices
726
+ self.indices_len = indices_len
727
+
728
+ def apply_weights(self, x: torch.Tensor) -> torch.Tensor:
729
+ output = self.base_layer.linear_method.apply_weights(
730
+ self.base_layer.linear_weights, x)
731
+ _apply_lora(
732
+ x,
733
+ self.lora_a_stacked,
734
+ self.lora_b_stacked,
735
+ self.indices[:self.indices_len[0]],
736
+ output,
737
+ )
738
+ return output
739
+
740
+ def forward(self, input_):
741
+ """Forward of RowParallelLinear
742
+
743
+ Args:
744
+ input_: tensor whose last dimension is `input_size`. If
745
+ `input_is_parallel` is set, then the last dimension
746
+ is `input_size // tp_size`.
747
+
748
+ Returns:
749
+ - output
750
+ - bias
751
+ """
752
+ # Set up backprop all-reduce.
753
+ if self.base_layer.input_is_parallel:
754
+ input_parallel = input_
755
+ else:
756
+ # TODO: simplify code below
757
+ tp_rank = get_tensor_model_parallel_rank()
758
+ splitted_input = split_tensor_along_last_dim(
759
+ input_, num_partitions=self.base_layer.tp_size)
760
+ input_parallel = splitted_input[tp_rank].contiguous()
761
+
762
+ # Matrix multiply.
763
+ output_parallel = self.apply_weights(input_parallel)
764
+ if self.base_layer.reduce_results and self.base_layer.tp_size > 1:
765
+ output_ = tensor_model_parallel_all_reduce(output_parallel)
766
+ else:
767
+ output_ = output_parallel
768
+
769
+ if not self.base_layer.skip_bias_add:
770
+ output = (output_ + self.base_layer.bias
771
+ if self.base_layer.bias is not None else output_)
772
+ output_bias = None
773
+ else:
774
+ output = output_
775
+ output_bias = self.base_layer.bias
776
+ return output, output_bias
777
+
778
+ @property
779
+ def weight(self):
780
+ return self.base_layer.weight
781
+
782
+
783
+class SamplerWithLoRA(BaseLayerWithLoRA):
784
+
785
+ def __init__(
786
+ self,
787
+ base_layer: Sampler,
788
+ hidden_size: int,
789
+ dtype: torch.dtype,
790
+ device: torch.device,
791
+ ) -> None:
792
+ super().__init__()
793
+ self.base_layer = base_layer
794
+ self.hidden_size = hidden_size
795
+ self.dtype = dtype
796
+ self.device = device
797
+
798
+ @property
799
+ def vocab_size(self):
800
+ return self.base_layer.vocab_size
801
+
802
+ @property
803
+ def org_vocab_size(self):
804
+ return self.base_layer.org_vocab_size
805
+
806
+ @property
807
+ def include_gpu_probs_tensor(self):
808
+ return self.base_layer.include_gpu_probs_tensor
809
+
810
+ def create_lora_weights(
811
+ self,
812
+ max_loras: int,
813
+ lora_config: LoRAConfig,
814
+ model_config: Optional[PretrainedConfig] = None,
815
+ ) -> None:
816
+ # Keep this in sync with csrc/punica/bgmv/bgmv_config.h
817
+ if 32000 < self.base_layer.vocab_size > 33024:
818
+ raise ValueError(
819
+ "When using LoRA, vocab size must be 32000 >= vocab_size <= 33024"
820
+ )
821
+ self.lora_a_stacked = torch.zeros(
822
+ (
823
+ max_loras,
824
+ 1,
825
+ lora_config.max_lora_rank,
826
+ self.hidden_size,
827
+ ),
828
+ dtype=lora_config.lora_dtype,
829
+ device=self.device,
830
+ )
831
+ self.lora_b_stacked = torch.zeros(
832
+ (
833
+ max_loras,
834
+ 1,
835
+ # Pad for kernel compatibility
836
+ math.ceil(self.base_layer.vocab_size /
837
+ lora_config.lora_vocab_padding_size) *
838
+ lora_config.lora_vocab_padding_size,
839
+ lora_config.max_lora_rank,
840
+ ),
841
+ dtype=lora_config.lora_dtype,
842
+ device=self.device,
843
+ )
844
+ self.embeddings_tensors = torch.full(
845
+ (max_loras, lora_config.lora_extra_vocab_size, self.hidden_size),
846
+ fill_value=float("-inf"),
847
+ dtype=self.dtype,
848
+ device=self.device,
849
+ )
850
+ self.indices = None
851
+ self.indices_padded = None
852
+ self.indices_len = None
853
+
854
+ def reset_lora(self, index: int):
855
+ self.lora_a_stacked[index] = 0
856
+ self.lora_b_stacked[index] = 0
857
+ self.embeddings_tensors[index] = float("-inf")
858
+
859
+ def set_lora(
860
+ self,
861
+ index: int,
862
+ lora_a: torch.Tensor,
863
+ lora_b: torch.Tensor,
864
+ embeddings_tensor: Optional[torch.Tensor],
865
+ ):
866
+ self.reset_lora(index)
867
+ self.lora_a_stacked[index,
868
+ 0, :lora_a.shape[1], :lora_a.shape[0]].copy_(
869
+ lora_a.T, non_blocking=True)
870
+ self.lora_b_stacked[index,
871
+ 0, :lora_b.shape[1], :lora_b.shape[0]].copy_(
872
+ lora_b.T, non_blocking=True)
873
+ if embeddings_tensor is not None:
874
+ self.embeddings_tensors[
875
+ index, :embeddings_tensor.shape[0], :embeddings_tensor.
876
+ shape[1], ] = embeddings_tensor
877
+
878
+ def set_mapping(
879
+ self,
880
+ base_indices: torch.Tensor,
881
+ sampler_indices: torch.Tensor,
882
+ sampler_indices_padded: torch.Tensor,
883
+ embeddings_indices: torch.Tensor,
884
+ indices_len: List[int],
885
+ ):
886
+ self.indices = sampler_indices
887
+ self.indices_padded = sampler_indices_padded
888
+ self.indices_len = indices_len
889
+
890
+ def _get_logits(
891
+ self,
892
+ hidden_states: torch.Tensor,
893
+ embedding: torch.Tensor,
894
+ embedding_bias: Optional[torch.Tensor] = None,
895
+ ) -> torch.Tensor:
896
+ # Get the logits for the next tokens.
897
+ logits = torch.matmul(hidden_states, embedding.t())
898
+ if embedding_bias is not None:
899
+ logits += embedding_bias
900
+ logits = tensor_model_parallel_gather(logits)
901
+ if logits is None:
902
+ return None
903
+
904
+ lora_logits = torch.empty(
905
+ self.embeddings_tensors.shape[0] + 1,
906
+ self.embeddings_tensors.shape[1],
907
+ hidden_states.shape[0],
908
+ dtype=self.embeddings_tensors.dtype,
909
+ device=self.embeddings_tensors.device,
910
+ )
911
+ torch.matmul(self.embeddings_tensors,
912
+ hidden_states.T,
913
+ out=lora_logits[:-1])
914
+ lora_logits[-1] = float("-inf")
915
+ lora_logits = lora_logits.mT
916
+ lora_logits = (lora_logits.reshape(
917
+ lora_logits.shape[0] * lora_logits.shape[1],
918
+ lora_logits.shape[2],
919
+ ).index_select(0,
920
+ self.indices_padded[:self.indices_len[2]]).nan_to_num_(
921
+ nan=float("-inf"),
922
+ posinf=float("inf"),
923
+ neginf=float("-inf")))
924
+ logits[:,
925
+ self.base_layer.org_vocab_size:self.base_layer.org_vocab_size +
926
+ lora_logits.shape[1]] = lora_logits
927
+
928
+ _apply_lora(
929
+ hidden_states,
930
+ self.lora_a_stacked,
931
+ self.lora_b_stacked,
932
+ self.indices[:self.indices_len[1]],
933
+ logits,
934
+ )
935
+
936
+ # Remove paddings in vocab (if any).
937
+ logits = logits[:, :self.base_layer.vocab_size]
938
+
939
+ return logits
940
+
941
+ def forward(self, *args, **kwargs):
942
+ return type(self.base_layer).forward(self, *args, **kwargs)
943
+
944
+
945
+def from_layer(
946
+ layer: nn.Module,
947
+ max_loras: int,
948
+ lora_config: LoRAConfig,
949
+ model_config: Optional[PretrainedConfig] = None) -> BaseLayerWithLoRA:
950
+ supported_layer_types = {
951
+ VocabParallelEmbedding: VocabParallelEmbeddingWithLoRA,
952
+ ColumnParallelLinear: ColumnParallelLinearWithLoRA,
953
+ QKVParallelLinear: QKVParallelLinearWithLora,
954
+ MergedColumnParallelLinear: MergedColumnParallelLinearWithLoRA,
955
+ RowParallelLinear: RowParallelLinearWithLoRA,
956
+ }
957
+ for src_layer_type, lora_layer_type in supported_layer_types.items():
958
+ if type(layer) is src_layer_type: # pylint: disable=unidiomatic-typecheck
959
+ ret = lora_layer_type(layer)
960
+ ret.create_lora_weights(max_loras, lora_config, model_config)
961
+ return ret
962
+ return layer
963
+
964
+
965
+def from_layer_sampler(
966
+ layer: Sampler,
967
+ lm_head: ParallelLMHead,
968
+ max_loras: int,
969
+ lora_config: LoRAConfig,
970
+ model_config: Optional[PretrainedConfig] = None,
971
+) -> SamplerWithLoRA:
972
+ ret = SamplerWithLoRA(layer, lm_head.embedding_dim, lm_head.weight.dtype,
973
+ lm_head.weight.device)
974
+ ret.create_lora_weights(max_loras, lora_config, model_config)
975
+ return ret
vllm/lora/lora.py
new
+160
@@ -0,0 +1,160 @@
1
+from typing import List, Optional
2
+
3
+import torch
4
+from vllm.utils import in_wsl
5
+
6
+
7
+class LoRALayerWeights:
8
+ """LoRA weights for a layer composed of two low rank matrixes."""
9
+
10
+ def __init__(
11
+ self,
12
+ module_name: str,
13
+ rank: int,
14
+ lora_alpha: int,
15
+ lora_a: torch.Tensor,
16
+ lora_b: torch.Tensor,
17
+ embeddings_tensor: Optional[torch.Tensor] = None,
18
+ scaling: Optional[float] = None,
19
+ ) -> None:
20
+ self.module_name = module_name
21
+ self.rank = rank
22
+ self.lora_alpha = lora_alpha
23
+ self.lora_a = lora_a
24
+ self.lora_b = lora_b
25
+ self.embeddings_tensor = embeddings_tensor
26
+
27
+ if scaling is None:
28
+ self.scaling = self.lora_alpha / self.rank
29
+ else:
30
+ self.scaling = scaling
31
+
32
+ def optimize(self) -> "LoRALayerWeights":
33
+ """Optimize the LoRA by merging the scaling into lora_b."""
34
+ if self.scaling == 1:
35
+ return
36
+ self.lora_b *= self.scaling
37
+ self.scaling = 1
38
+ return self
39
+
40
+ @property
41
+ def input_dim(self) -> int:
42
+ return self.lora_a.shape[0]
43
+
44
+ @property
45
+ def output_dim(self) -> int:
46
+ return self.lora_b.shape[1]
47
+
48
+ @property
49
+ def is_packed(self) -> bool:
50
+ return False
51
+
52
+ @property
53
+ def extra_vocab_size(self) -> int:
54
+ return self.embeddings_tensor.shape[
55
+ 0] if self.embeddings_tensor is not None else 0
56
+
57
+ @classmethod
58
+ def create_dummy_lora_weights(
59
+ cls,
60
+ module_name: str,
61
+ input_dim: int,
62
+ output_dim: int,
63
+ rank: int,
64
+ dtype: torch.dtype,
65
+ device: torch.device,
66
+ embeddings_tensor_dim: Optional[int] = None) -> "LoRALayerWeights":
67
+ pin_memory = str(device) == "cpu" and not in_wsl()
68
+ lora_a = torch.zeros([input_dim, rank],
69
+ dtype=dtype,
70
+ device=device,
71
+ pin_memory=pin_memory)
72
+ lora_b = torch.zeros([rank, output_dim],
73
+ dtype=dtype,
74
+ device=device,
75
+ pin_memory=pin_memory)
76
+ embeddings_tensor = torch.rand(
77
+ 10,
78
+ embeddings_tensor_dim,
79
+ dtype=dtype,
80
+ device=device,
81
+ pin_memory=pin_memory) if embeddings_tensor_dim else None
82
+ return cls(
83
+ module_name,
84
+ rank=rank,
85
+ lora_alpha=1,
86
+ lora_a=lora_a,
87
+ lora_b=lora_b,
88
+ embeddings_tensor=embeddings_tensor,
89
+ )
90
+
91
+
92
+class PackedLoRALayerWeights(LoRALayerWeights):
93
+ """LoRA used for packed layers (eg. qkv_proj)."""
94
+
95
+ def __init__(
96
+ self,
97
+ module_name: str,
98
+ rank: int,
99
+ lora_alphas: List[int],
100
+ lora_a: List[torch.Tensor],
101
+ lora_b: List[torch.Tensor],
102
+ scaling: Optional[List[float]] = None,
103
+ ) -> None:
104
+ super().__init__(
105
+ module_name=module_name,
106
+ rank=rank,
107
+ lora_alpha=0,
108
+ lora_a=lora_a,
109
+ lora_b=lora_b,
110
+ scaling=scaling,
111
+ embeddings_tensor=None,
112
+ )
113
+ self.lora_alphas = lora_alphas
114
+ if scaling is None:
115
+ self.scaling = [
116
+ lora_alpha / self.rank for lora_alpha in self.lora_alphas
117
+ ]
118
+
119
+ @classmethod
120
+ def pack(cls, loras: List["LoRALayerWeights"]) -> "PackedLoRALayerWeights":
121
+ """Pack a list of LoRAs into a single LoRA.
122
+
123
+ If LoRA is None, it signifies that the submodule does not have a LoRA.
124
+ """
125
+ first_lora = next(lora for lora in loras if lora is not None)
126
+ for lora in loras:
127
+ if lora is None:
128
+ continue
129
+ lora.optimize()
130
+ rank = first_lora.rank
131
+ module_name = first_lora.module_name
132
+ obj = cls(
133
+ module_name,
134
+ rank,
135
+ [lora.lora_alpha if lora is not None else None for lora in loras],
136
+ [lora.lora_a if lora is not None else None for lora in loras],
137
+ [lora.lora_b if lora is not None else None for lora in loras],
138
+ scaling=[1 if lora is not None else None for lora in loras])
139
+ return obj
140
+
141
+ def optimize(self) -> "PackedLoRALayerWeights":
142
+ """Optimize the LoRA by merging the scaling into lora_b."""
143
+ for i in range(len(self.lora_b)):
144
+ if self.scaling[i] == 1 or self.lora_b[i] is None:
145
+ continue
146
+ self.lora_b[i] *= self.scaling[i]
147
+ self.scaling[i] = 1
148
+ return self
149
+
150
+ @property
151
+ def input_dim(self) -> int:
152
+ raise NotImplementedError()
153
+
154
+ @property
155
+ def output_dim(self) -> int:
156
+ raise NotImplementedError()
157
+
158
+ @property
159
+ def is_packed(self) -> bool:
160
+ return True
vllm/lora/models.py
new
+654
@@ -0,0 +1,654 @@
1
+import copy
2
+import json
3
+import logging
4
+import math
5
+import os
6
+import re
7
+from typing import (Any, Callable, Dict, Hashable, List, Optional, Tuple, Type,
8
+ Union)
9
+
10
+import safetensors.torch
11
+import torch
12
+from torch import nn
13
+
14
+from vllm.config import LoRAConfig
15
+from vllm.utils import LRUCache, in_wsl
16
+
17
+from vllm.lora.layers import BaseLayerWithLoRA, LoRAMapping, from_layer, from_layer_sampler
18
+from vllm.lora.lora import LoRALayerWeights, PackedLoRALayerWeights
19
+from vllm.lora.utils import parse_fine_tuned_lora_name, replace_submodule
20
+
21
+logger = logging.getLogger(__name__)
22
+
23
+# TODO: The mappings below should be moved to individual model classes.
24
+
25
+PACKED_MODULES_CFG = {
26
+ "qkv_proj": [
27
+ "q_proj",
28
+ "k_proj",
29
+ "v_proj",
30
+ ],
31
+ "gate_up_proj": [
32
+ "gate_proj",
33
+ "up_proj",
34
+ ],
35
+}
36
+
37
+TARGET_MODULES_QKV = [
38
+ "qkv_proj",
39
+ "o_proj",
40
+ "gate_up_proj",
41
+ "down_proj",
42
+ "embed_tokens",
43
+ "lm_head",
44
+]
45
+
46
+EMBEDDING_MODULES = {
47
+ "embed_tokens": "input_embeddings",
48
+ "lm_head": "output_embeddings",
49
+}
50
+
51
+EMBEDDING_PADDING_MODULES = ["lm_head"]
52
+
53
+_GLOBAL_LORA_ID = 0
54
+
55
+
56
+def convert_mapping(
57
+ mapping: LoRAMapping, lora_index_to_id: List[Optional[int]],
58
+ max_loras: int, vocab_size: int, extra_vocab_size: int
59
+) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, List[int]]:
60
+ """Converts LoRAMapping to index tensors.
61
+
62
+ Args:
63
+ mapping: LoRAMapping mapping rows in a batch to LoRA ids.
64
+ lora_index_to_id: List mapping LoRA ids to LoRA indices.
65
+ max_loras: Maximum number of LoRAs.
66
+ vocab_size: Model vocab size.
67
+ extra_vocab_size: Extra vocab size each LoRA can have.
68
+
69
+ Returns:
70
+ A tuple of tensors:
71
+ base_indices: Tensor of shape [batch_size] mapping batch rows to
72
+ LoRA indices.
73
+ sampler_indices: Tensor of shape [batch_size] mapping requests to
74
+ LoRA indices for sampler. For generation, this will be the
75
+ same as base_indicies. For prefill, this will map requests
76
+ to LoRA indices.
77
+ sampler_indices_padded: Tensor of shape [batch_size] mapping
78
+ requests to LoRA indices for sampler with padding.
79
+ Same as sampler_indicies, but -1 is replaced with
80
+ max_loras.
81
+ embeddings_indices: Tensor of shape [2, batch_size] mapping
82
+ requests to embedding indices. First row is for embeddings
83
+ added by the LoRAs, second row is for the LoRA.lora_a
84
+ embeddings.
85
+ indices_len: List of lengths of the above tensors.
86
+ """
87
+ indices = list(mapping.index_mapping).copy()
88
+ embedding_indices = indices.copy()
89
+ lora_indices = indices.copy()
90
+ prompt_mapping = [
91
+ lora_index_to_id.index(x) if x > 0 else -1
92
+ for x in mapping.prompt_mapping
93
+ ]
94
+ lora_idx = None
95
+ for i in range(len(indices)):
96
+ # TODO index can be slow. optimize
97
+ lora_idx = (lora_index_to_id.index(indices[i])
98
+ if indices[i] > 0 else -1)
99
+ embedding_indices[i] = lora_idx if indices[i] > 0 else 0
100
+ indices[i] = i
101
+ lora_indices[i] = lora_idx
102
+
103
+ indices = torch.tensor([indices, lora_indices, embedding_indices],
104
+ dtype=torch.long,
105
+ device="cuda")
106
+ prompt_mapping = torch.tensor(prompt_mapping,
107
+ device="cuda",
108
+ dtype=torch.long)
109
+ embeddings_indices = torch.stack([
110
+ indices[2] * extra_vocab_size,
111
+ indices[2] * (vocab_size + extra_vocab_size)
112
+ ])
113
+ embeddings_indices[embeddings_indices == -1] = max_loras - 1
114
+ base_indices = indices[1]
115
+ sampler_indices = prompt_mapping
116
+ sampler_indices_padded = sampler_indices.clone()
117
+ sampler_indices_padded[sampler_indices_padded == -1] = max_loras - 1
118
+ sampler_indices_padded = (
119
+ torch.arange(
120
+ 0, len(sampler_indices_padded), device="cuda", dtype=torch.long) +
121
+ (sampler_indices_padded * len(sampler_indices_padded)))
122
+ indices_len = (base_indices.shape[-1], sampler_indices.shape[-1],
123
+ sampler_indices_padded.shape[-1],
124
+ embeddings_indices.shape[-1])
125
+
126
+ return (base_indices, sampler_indices, sampler_indices_padded,
127
+ embeddings_indices, indices_len)
128
+
129
+
130
+def get_lora_id():
131
+ global _GLOBAL_LORA_ID
132
+ _GLOBAL_LORA_ID += 1
133
+ return _GLOBAL_LORA_ID
134
+
135
+
136
+class LoRAModel:
137
+ """A LoRA fine-tuned model."""
138
+
139
+ def __init__(
140
+ self,
141
+ lora_model_id: int,
142
+ rank: int,
143
+ loras: Dict[str, LoRALayerWeights],
144
+ ) -> None:
145
+ self.id = lora_model_id
146
+ assert (lora_model_id >
147
+ 0), f"a valid lora id should be greater than 0, got {self.id}"
148
+ self.rank = rank
149
+ self.loras: Dict[str, LoRALayerWeights] = loras
150
+
151
+ @property
152
+ def extra_vocab_size(self) -> int:
153
+ return max(lora.extra_vocab_size
154
+ for lora in self.loras.values()) if self.loras else 0
155
+
156
+ def get_lora(self, module_name: str) -> Optional[LoRALayerWeights]:
157
+ """Get LoRA for a given module by name"""
158
+ return self.loras.get(module_name, None)
159
+
160
+ # (yard1): TODO see if we can derive target_embedding_padding automatically
161
+ @classmethod
162
+ def from_lora_tensors(
163
+ cls,
164
+ lora_model_id: int,
165
+ rank: int,
166
+ lora_alpha: int,
167
+ tensors: Dict[str, torch.Tensor],
168
+ device: str = "cuda",
169
+ dtype: Optional[torch.dtype] = None,
170
+ embeddings: Optional[Dict[str, torch.Tensor]] = None,
171
+ target_embedding_padding: Optional[int] = None,
172
+ ) -> "LoRAModel":
173
+ """Create a LoRAModel from a dictionary of tensors."""
174
+ pin_memory = str(device) == "cpu" and not in_wsl()
175
+ loras: Dict[str, LoRALayerWeights] = {}
176
+ for tensor_name, tensor in tensors.items():
177
+ module_name, is_lora_a = parse_fine_tuned_lora_name(tensor_name)
178
+ if module_name not in loras:
179
+ lora_embeddings_tensor = None
180
+ if embeddings:
181
+ embeddings_module = next(
182
+ (k for k in EMBEDDING_MODULES if k in module_name),
183
+ None)
184
+ if embeddings_module:
185
+ lora_embeddings_tensor = embeddings[
186
+ EMBEDDING_MODULES[embeddings_module]].to(
187
+ device=device, dtype=dtype)
188
+ if pin_memory:
189
+ lora_embeddings_tensor = (
190
+ lora_embeddings_tensor.pin_memory())
191
+ loras[module_name] = LoRALayerWeights(module_name, rank,
192
+ lora_alpha, None, None,
193
+ lora_embeddings_tensor)
194
+ if is_lora_a:
195
+ loras[module_name].lora_a = tensor.to(device=device,
196
+ dtype=dtype).t()
197
+ if pin_memory:
198
+ loras[module_name].lora_a = loras[
199
+ module_name].lora_a.pin_memory()
200
+ else:
201
+ loras[module_name].lora_b = tensor.to(device=device,
202
+ dtype=dtype).t()
203
+ if any(name in module_name
204
+ for name in EMBEDDING_PADDING_MODULES
205
+ ) and target_embedding_padding is not None:
206
+ lora_b = loras[module_name].lora_b
207
+ assert target_embedding_padding >= lora_b.shape[1]
208
+ addition = target_embedding_padding - lora_b.shape[1]
209
+ loras[module_name].lora_b = torch.nn.functional.pad(
210
+ lora_b, (0, addition))
211
+ if pin_memory:
212
+ loras[module_name].lora_b = loras[
213
+ module_name].lora_b.pin_memory()
214
+
215
+ for lora in loras.values():
216
+ lora.optimize()
217
+ return cls(lora_model_id, rank, loras)
218
+
219
+ @classmethod
220
+ def from_local_checkpoint(
221
+ cls,
222
+ lora_dir: str,
223
+ lora_model_id: Optional[int] = None,
224
+ device: str = "cuda",
225
+ dtype: Optional[torch.dtype] = None,
226
+ target_embedding_padding: Optional[int] = None) -> "LoRAModel":
227
+ """Create a LoRAModel from a local checkpoint."""
228
+ lora_config_path = os.path.join(lora_dir, "adapter_config.json")
229
+ lora_tensor_path = os.path.join(lora_dir, "adapter_model.safetensors")
230
+ lora_bin_file_path = os.path.join(lora_dir, "adapter_model.bin")
231
+ new_embeddings_tensor_path = os.path.join(
232
+ lora_dir, "new_embeddings.safetensors")
233
+ new_embeddings_bin_file_path = os.path.join(lora_dir,
234
+ "new_embeddings.bin")
235
+ if os.path.isfile(lora_tensor_path):
236
+ tensors = safetensors.torch.load_file(lora_tensor_path)
237
+ elif os.path.isfile(lora_bin_file_path):
238
+ tensors = torch.load(lora_bin_file_path)
239
+ else:
240
+ raise ValueError(f"{lora_dir} doesn't contain tensors")
241
+
242
+ embeddings = None
243
+ if os.path.isfile(new_embeddings_tensor_path):
244
+ embeddings = safetensors.torch.load_file(
245
+ new_embeddings_tensor_path)
246
+ elif os.path.isfile(new_embeddings_bin_file_path):
247
+ embeddings = torch.load(new_embeddings_bin_file_path)
248
+
249
+ with open(lora_config_path) as f:
250
+ config = json.load(f)
251
+ rank = config["r"]
252
+ lora_alpha = config["lora_alpha"]
253
+ return cls.from_lora_tensors(
254
+ lora_model_id=get_lora_id()
255
+ if lora_model_id is None else lora_model_id,
256
+ rank=rank,
257
+ lora_alpha=lora_alpha,
258
+ tensors=tensors,
259
+ device=device,
260
+ dtype=dtype,
261
+ embeddings=embeddings,
262
+ target_embedding_padding=target_embedding_padding,
263
+ )
264
+
265
+
266
+class LoRAModelManager:
267
+ """A manager that manages multiple LoRA-fine-tuned models."""
268
+
269
+ def __init__(
270
+ self,
271
+ model: nn.Module,
272
+ max_num_seqs: int,
273
+ max_num_batched_tokens: int,
274
+ vocab_size: int,
275
+ lora_config: LoRAConfig,
276
+ lora_target_modules: Union[str, List[str]] = TARGET_MODULES_QKV,
277
+ packed_modules_mapping: Dict[str, List[str]] = PACKED_MODULES_CFG,
278
+ ):
279
+ """Create a LoRAModelManager and adapter for a given model.
280
+
281
+ Args:
282
+ model: the model to be adapted.
283
+ max_num_seqs: the maximum number of sequences model can run in a
284
+ single batch.
285
+ max_num_batched_tokens: the maximum number of tokens model can run
286
+ in a single batch.
287
+ vocab_size: the vocab size of the model.
288
+ lora_config: the LoRA configuration.
289
+ lora_target_modules: the target modules patterns to be adapted.
290
+ Support both single module name and a list of module names.
291
+ packed_modules_mapping: the mapping for packed modules. vLLM
292
+ packs some modules into one module, e.g., qkv_proj
293
+ is packed of q_proj, k_proj, and v_proj. These modules
294
+ have a single layer in the original model, but they are split
295
+ into multiple layers in the adapted model.
296
+ """
297
+ self.lora_config = lora_config
298
+ self.max_num_seqs = max_num_seqs
299
+ assert self.capacity >= self.lora_slots
300
+ self.max_num_batched_tokens = math.ceil(max_num_batched_tokens / 8) * 8
301
+ self.lora_index_to_id: List[Optional[int]] = [None] * self.lora_slots
302
+ self.vocab_size = vocab_size
303
+ self.base_indices = torch.empty(self.max_num_batched_tokens,
304
+ dtype=torch.long,
305
+ device="cuda")
306
+ self.sampler_indices = torch.empty(self.max_num_batched_tokens,
307
+ dtype=torch.long,
308
+ device="cuda")
309
+ self.sampler_indices_padded = torch.empty(self.max_num_batched_tokens,
310
+ dtype=torch.long,
311
+ device="cuda")
312
+ self.embeddings_indices = torch.empty(2,
313
+ self.max_num_batched_tokens,
314
+ dtype=torch.long,
315
+ device="cuda")
316
+ self.offsets = []
317
+ # 4 is the number of indicies tensors defined above
318
+ # base_indices, sampler_indices, sampler_indices_padded,
319
+ # embeddings_indices
320
+ self.indices_len = [None] * 4
321
+
322
+ self.model: nn.Module = model
323
+ self.lora_target_modules: List[str] = ([
324
+ lora_target_modules
325
+ ] if isinstance(lora_target_modules, str) else lora_target_modules)
326
+ self.lora_target_modules = copy.deepcopy(lora_target_modules)
327
+ self.packed_modules_mapping = copy.deepcopy(packed_modules_mapping)
328
+ self.packed_modules: Dict[str, List[str]] = {}
329
+ self.modules: Dict[str, "BaseLayerWithLoRA"] = {}
330
+ self._registered_loras: Dict[int, LoRAModel] = {}
331
+ # Dict instead of a Set for compatibility with LRUCache.
332
+ self._active_loras: Dict[int, None] = {}
333
+ self._last_mapping = None
334
+ self._create_lora_modules()
335
+ self.model.lora_manager = self
336
+
337
+ @property
338
+ def capacity(self) -> int:
339
+ return self.lora_config.max_cpu_loras
340
+
341
+ @property
342
+ def lora_slots(self) -> int:
343
+ return self.lora_config.max_loras
344
+
345
+ def __len__(self) -> int:
346
+ return len(self._registered_loras)
347
+
348
+ def activate_lora(
349
+ self,
350
+ lora_id: int,
351
+ ) -> bool:
352
+ """Move LoRA into a GPU buffer to be used in the forward pass."""
353
+ if lora_id in self._active_loras:
354
+ return False
355
+ first_free_slot = next(
356
+ ((i, lora_id) for i, lora_id in enumerate(self.lora_index_to_id)
357
+ if lora_id is None), None)
358
+ if first_free_slot is None:
359
+ raise ValueError("No free lora slots")
360
+ index, _ = first_free_slot
361
+ self._active_loras[lora_id] = None
362
+ lora_model = self._registered_loras[lora_id]
363
+ logger.debug(
364
+ f"Activating LoRA. int id: {lora_model.id}, slot index: {index}")
365
+ self.lora_index_to_id[index] = lora_model.id
366
+ for module_name, module in self.modules.items():
367
+ module_lora = lora_model.get_lora(module_name)
368
+ if module_lora:
369
+ module_lora.optimize()
370
+ module.set_lora(index, module_lora.lora_a, module_lora.lora_b,
371
+ module_lora.embeddings_tensor)
372
+ else:
373
+ module.reset_lora(index)
374
+ return True
375
+
376
+ def _deactivate_lora(self, lora_id: int):
377
+ try:
378
+ index = self.lora_index_to_id.index(lora_id)
379
+ self.lora_index_to_id[index] = None
380
+ except ValueError:
381
+ pass
382
+
383
+ def deactivate_lora(self, lora_id: int) -> bool:
384
+ """Remove a LoRA from a GPU buffer."""
385
+ if lora_id in self._active_loras:
386
+ self._deactivate_lora(lora_id)
387
+ self._active_loras.pop(lora_id)
388
+ return True
389
+ return False
390
+
391
+ def _add_lora(self, lora: LoRAModel) -> bool:
392
+ self._create_merged_loras_inplace(lora)
393
+ self._registered_loras[lora.id] = lora
394
+
395
+ def add_lora(self, lora: LoRAModel) -> bool:
396
+ """Add a LoRAModel to the manager CPU cache."""
397
+ if lora.id not in self._registered_loras:
398
+ if len(self._registered_loras) >= self.capacity:
399
+ raise RuntimeError("No free LoRA slots.")
400
+ self._add_lora(lora)
401
+ return True
402
+ return False
403
+
404
+ def remove_lora(self, lora_id: int) -> bool:
405
+ """Remove a LoRAModel from the manager CPU cache."""
406
+ # TODO: should we check active lora?
407
+ self.deactivate_lora(lora_id)
408
+ return bool(self._registered_loras.pop(lora_id, None))
409
+
410
+ # TODO see if this can be vectorized
411
+ def _set_lora_mapping(self, mapping: LoRAMapping) -> None:
412
+ (base_indices, sampler_indices, sampler_indices_padded,
413
+ embeddings_indices,
414
+ indices_len) = convert_mapping(mapping, self.lora_index_to_id,
415
+ self.lora_slots + 1, self.vocab_size,
416
+ self.lora_config.lora_extra_vocab_size)
417
+ self.base_indices[:base_indices.shape[0]].copy_(base_indices)
418
+ self.sampler_indices[:sampler_indices.shape[0]].copy_(sampler_indices)
419
+ self.sampler_indices_padded[:sampler_indices_padded.shape[0]].copy_(
420
+ sampler_indices_padded)
421
+ self.embeddings_indices[:embeddings_indices.
422
+ shape[0], :embeddings_indices.shape[1]].copy_(
423
+ embeddings_indices)
424
+ # Maintain the reference
425
+ self.indices_len[:] = indices_len
426
+
427
+ def set_lora_mapping(self, lora_mapping: LoRAMapping) -> None:
428
+ if self._last_mapping != lora_mapping:
429
+ self._set_lora_mapping(lora_mapping)
430
+ self._last_mapping = lora_mapping
431
+
432
+ def list_loras(self) -> Dict[int, LoRAModel]:
433
+ """List all registered LoRAModels."""
434
+ return dict(self._registered_loras)
435
+
436
+ def get_lora(self, lora_id: int) -> Optional[LoRAModel]:
437
+ return self._registered_loras.get(lora_id, None)
438
+
439
+ def remove_all_loras(self) -> bool:
440
+ """Remove all LoRAModels from the manager."""
441
+ self._registered_loras.clear()
442
+ self.lora_index_to_id = [None] * self.lora_slots
443
+ self._active_loras.clear()
444
+
445
+ def _create_lora_modules(self):
446
+ for module_name, module in self.model.named_modules():
447
+ if not self._match_target_modules(module_name):
448
+ continue
449
+
450
+ new_module = replace_submodule(
451
+ self.model, module_name,
452
+ from_layer(module, self.lora_slots, self.lora_config,
453
+ self.model.config))
454
+ # (yard1): TODO make this more robust
455
+ if "lm_head" in module_name:
456
+ sampler_module = self.model.get_submodule("sampler")
457
+ new_module = replace_submodule(
458
+ self.model, "sampler",
459
+ from_layer_sampler(sampler_module, module, self.lora_slots,
460
+ self.lora_config, self.model.config))
461
+ self.register_module(module_name, new_module)
462
+ self._register_packed_modules(module_name)
463
+ new_module.set_mapping(self.base_indices, self.sampler_indices,
464
+ self.sampler_indices_padded,
465
+ self.embeddings_indices, self.indices_len)
466
+
467
+ def register_module(self, module_name: str, module: "BaseLayerWithLoRA"):
468
+ assert isinstance(module, BaseLayerWithLoRA)
469
+ self.modules[module_name] = module
470
+
471
+ def create_dummy_lora(self, lora_id: int, rank: int) -> LoRAModel:
472
+ """Create zero-initialized LoRAModel for warmup."""
473
+ model = LoRAModel(lora_id, rank, {})
474
+ for module_name, module in self.model.named_modules():
475
+ if not self._match_target_modules(module_name) or not isinstance(
476
+ module, BaseLayerWithLoRA):
477
+ continue
478
+ parts = module_name.split(".")
479
+ if module_name not in self.packed_modules:
480
+ if parts[-1] in EMBEDDING_MODULES:
481
+ input_dim = (module.base_layer.org_vocab_size +
482
+ self.lora_config.lora_extra_vocab_size if
483
+ hasattr(module.base_layer, "org_vocab_size")
484
+ else module.base_layer.weight.shape[1])
485
+ output_dim = module.base_layer.embedding_dim if hasattr(
486
+ module.base_layer,
487
+ "embedding_dim") else module.base_layer.weight.shape[0]
488
+ embeddings_tensor_dim = (module.base_layer.embedding_dim if
489
+ hasattr(module.base_layer,
490
+ "embedding_dim") else
491
+ module.base_layer.weight.shape[1])
492
+ lora = LoRALayerWeights.create_dummy_lora_weights(
493
+ module_name,
494
+ input_dim,
495
+ output_dim,
496
+ rank,
497
+ module.lora_a_stacked.dtype,
498
+ "cpu",
499
+ embeddings_tensor_dim=embeddings_tensor_dim)
500
+ else:
501
+ lora = LoRALayerWeights.create_dummy_lora_weights(
502
+ module_name,
503
+ module.lora_a_stacked.shape[-1],
504
+ module.lora_b_stacked.shape[-2],
505
+ rank,
506
+ module.lora_a_stacked.dtype,
507
+ "cpu",
508
+ )
509
+ lora.optimize()
510
+ else:
511
+ parts = module_name.split(".")
512
+ replacements = self.packed_modules_mapping[parts[-1]]
513
+ subloras = []
514
+ for i, r in enumerate(replacements):
515
+ lora = LoRALayerWeights.create_dummy_lora_weights(
516
+ module_name + "." + r,
517
+ module.lora_a_stacked[i].shape[-1],
518
+ module.lora_b_stacked[i].shape[-2],
519
+ rank,
520
+ module.lora_a_stacked[i].dtype,
521
+ "cpu",
522
+ )
523
+ lora.optimize()
524
+ subloras.append(lora)
525
+ lora = PackedLoRALayerWeights.pack(subloras)
526
+ model.loras[module_name] = lora
527
+ return model
528
+
529
+ def _match_target_modules(self, module_name: str):
530
+ return any(
531
+ re.match(
532
+ r".*\.{target_module}$".format(target_module=target_module),
533
+ module_name) or target_module == module_name
534
+ for target_module in self.lora_target_modules)
535
+
536
+ def _register_packed_modules(self, module_full_name: str) -> None:
537
+ parts = module_full_name.split(".")
538
+ module_name = parts[-1]
539
+ replacements = self.packed_modules_mapping.get(module_name)
540
+ if not replacements:
541
+ return
542
+ prefix = ".".join(parts[:-1])
543
+ self.packed_modules[module_full_name] = [
544
+ prefix + "." + r if prefix else r for r in replacements
545
+ ]
546
+
547
+ def _create_merged_loras_inplace(self, lora_model: LoRAModel) -> None:
548
+ for module_name, new_module_names in self.packed_modules.items():
549
+ replacement_loras = []
550
+ has_replacement = False
551
+ for r in new_module_names:
552
+ lora = lora_model.get_lora(r)
553
+ replacement_loras.append(lora)
554
+ if lora:
555
+ has_replacement = True
556
+ if not has_replacement:
557
+ continue
558
+ for i in range(len(replacement_loras)):
559
+ if replacement_loras[i]:
560
+ continue
561
+ replacement_loras[i] = None
562
+ lora_model.loras[module_name] = PackedLoRALayerWeights.pack(
563
+ replacement_loras)
564
+
565
+
566
+class LoRALRUCache(LRUCache):
567
+
568
+ def __init__(self, capacity: int, deactivate_lora_fn: Callable[[Hashable],
569
+ None]):
570
+ super().__init__(capacity)
571
+ self.deactivate_lora_fn = deactivate_lora_fn
572
+
573
+ def _on_remove(self, key: Hashable, value: Any):
574
+ logger.debug(f"Removing LoRA. int id: {key}")
575
+ self.deactivate_lora_fn(key)
576
+ return super()._on_remove(key, value)
577
+
578
+
579
+class LRUCacheLoRAModelManager(LoRAModelManager):
580
+ """A model manager that manages multiple LoRAs with LRU cache."""
581
+
582
+ def __init__(
583
+ self,
584
+ model: nn.Module,
585
+ max_num_seqs: int,
586
+ max_num_batched_tokens: int,
587
+ vocab_size: int,
588
+ lora_config: LoRAConfig,
589
+ lora_target_modules: Union[str, List[str]] = TARGET_MODULES_QKV,
590
+ packed_modules_mapping: Dict[str, List[str]] = PACKED_MODULES_CFG,
591
+ ):
592
+ super().__init__(model, max_num_seqs, max_num_batched_tokens,
593
+ vocab_size, lora_config, lora_target_modules,
594
+ packed_modules_mapping)
595
+ self._registered_loras: LoRALRUCache = LoRALRUCache(
596
+ self.capacity, self.deactivate_lora)
597
+ self._active_loras: LoRALRUCache = LoRALRUCache(
598
+ self.lora_slots, self._deactivate_lora)
599
+
600
+ def list_loras(self) -> Dict[int, LoRAModel]:
601
+ """List all registered LoRAModels."""
602
+ return dict(self._registered_loras.cache)
603
+
604
+ def add_lora(self, lora: LoRAModel) -> bool:
605
+ """Add a LoRAModel to the manager."""
606
+ if lora.id not in self._registered_loras:
607
+ self._add_lora(lora)
608
+ was_added = True
609
+ else:
610
+ # We always touch to update the LRU cache order
611
+ self._registered_loras.touch(lora.id)
612
+ was_added = False
613
+ return was_added
614
+
615
+ def activate_lora(
616
+ self,
617
+ lora_id: int,
618
+ ) -> bool:
619
+ if lora_id not in self._active_loras and len(
620
+ self._active_loras) >= self.lora_slots:
621
+ self._active_loras.remove_oldest()
622
+ result = super().activate_lora(lora_id)
623
+ # We always touch to update the LRU cache order
624
+ self._active_loras.touch(lora_id)
625
+ return result
626
+
627
+ def remove_oldest_lora(self) -> bool:
628
+ if len(self._registered_loras) > 0:
629
+ self._registered_loras.remove_oldest()
630
+ return True
631
+ return False
632
+
633
+
634
+def create_lora_manager(
635
+ model: nn.Module,
636
+ max_num_seqs: int,
637
+ max_num_batched_tokens: int,
638
+ vocab_size: int,
639
+ lora_config: LoRAConfig,
640
+ target_modules: Union[str, List[str]] = TARGET_MODULES_QKV,
641
+ lora_manager_cls: Type[LoRAModelManager] = LoRAModelManager,
642
+ **kwargs) -> LoRAModelManager:
643
+ """Create a LoRA adapter for a given model."""
644
+ if not getattr(model, "supports_lora", False):
645
+ raise ValueError(f"Model {type(model)} is not supported for LoRA.")
646
+ lora_manager = lora_manager_cls(
647
+ model=model,
648
+ max_num_seqs=max_num_seqs,
649
+ max_num_batched_tokens=max_num_batched_tokens,
650
+ vocab_size=vocab_size,
651
+ lora_config=lora_config,
652
+ lora_target_modules=target_modules,
653
+ **kwargs)
654
+ return lora_manager
vllm/lora/punica.py
new
+173
@@ -0,0 +1,173 @@
1
+# Based on code from https://github.com/punica-ai/punica
2
+
3
+from typing import Optional
4
+
5
+import torch
6
+
7
+import_exc = None
8
+
9
+try:
10
+ import vllm._punica_C as punica_kernels
11
+except ImportError as e:
12
+ import_exc = e
13
+
14
+if import_exc is None:
15
+
16
+ def bgmv(
17
+ y: torch.Tensor,
18
+ x: torch.Tensor,
19
+ w_t_all: torch.Tensor,
20
+ indicies: torch.LongTensor,
21
+ layer_idx: int,
22
+ scale: float,
23
+ ):
24
+ """
25
+ Semantics:
26
+ y[i] += (
27
+ x[i].unsqueeze(0)
28
+ @ w_t_all[indices[i], layer_idx, :, :].transpose(-1, -2)
29
+ * scale
30
+ ).squeeze(0)
31
+
32
+ Args:
33
+ y: Shape: `[B, H2]`. Output vectors. Will be changed in-place.
34
+ x: Shape: `[B, H1]`. Input vectors.
35
+ w_t_all: Shape: `[None, L, H2, H1]`. All of the transposed weight
36
+ matrices.
37
+ indicies: Shape: `[B]`. Indices of the weight matrices.
38
+ layer_idx: Layer index of the weight matrices.
39
+ scale: Scaling factor.
40
+ """
41
+ punica_kernels.dispatch_bgmv(y, x, w_t_all, indicies, layer_idx, scale)
42
+
43
+ def add_lora(y: torch.Tensor,
44
+ x: torch.Tensor,
45
+ wa_t_all: torch.Tensor,
46
+ wb_t_all: torch.Tensor,
47
+ indicies: torch.LongTensor,
48
+ layer_idx: int,
49
+ scale: float,
50
+ *,
51
+ buffer: Optional[torch.Tensor] = None):
52
+ """
53
+ Semantics:
54
+ y[i] += (
55
+ x[i].unsqueeze(0)
56
+ @ wa_t_all[indices[i], layer_idx, :, :].transpose(-1, -2)
57
+ @ wb_t_all[indices[i], layer_idx, :, :].transpose(-1, -2)
58
+ * scale
59
+ ).squeeze(0)
60
+
61
+ Args:
62
+ y: Shape: `[B, H2]`. Output vectors. Will be changed in-place.
63
+ x: Shape: `[B, H1]`. Input vectors.
64
+ wa_t_all: Shape: `[None, L, R, H1]`. All of the transposed
65
+ LoRA A matrices.
66
+ wb_t_all: Shape: `[None, L, H2, R]`. All of the transposed
67
+ LoRA B matrices.
68
+ indicies: Shape: `[B]`. Indices of the LoRA weights.
69
+ layer_idx: Layer index of LoRA weights.
70
+ scale: Scaling factor.
71
+ buffer: Optional. Shape: `[B, R]`. Temporary buffer.
72
+ """
73
+ r = wb_t_all.size(-1)
74
+ if buffer is None:
75
+ # We set the buffer to be float32 by default to avoid
76
+ # numerical innacuracies that would otherwise happen
77
+ # due to downcasting.
78
+ buffer = torch.zeros((x.size(0), r),
79
+ dtype=torch.float32,
80
+ device=x.device)
81
+ punica_kernels.dispatch_bgmv(buffer, x, wa_t_all, indicies, layer_idx,
82
+ 1.0)
83
+ punica_kernels.dispatch_bgmv(y, buffer, wb_t_all, indicies, layer_idx,
84
+ scale)
85
+
86
+ def add_lora_slice(y: torch.Tensor,
87
+ x: torch.Tensor,
88
+ wa_t_all: torch.Tensor,
89
+ wb_t_all: torch.Tensor,
90
+ indicies: torch.LongTensor,
91
+ layer_idx: int,
92
+ scale: float,
93
+ y_offset: int,
94
+ y_slice_size: int,
95
+ *,
96
+ buffer: Optional[torch.Tensor] = None):
97
+ """
98
+ Same as `add_lora` but you can operate on slices of y.
99
+ Pass whole y, define y_offset and y_slice_size.
100
+
101
+ Semantics:
102
+ y[i] += (
103
+ x[i].unsqueeze(0)
104
+ @ wa_t_all[indices[i], layer_idx, :, :].transpose(-1, -2)
105
+ @ wb_t_all[indices[i], layer_idx, :, :].transpose(-1, -2)
106
+ * scale
107
+ ).squeeze(0)
108
+
109
+ Args:
110
+ y: Shape: `[B, H2]`. Output vectors. Will be changed in-place.
111
+ x: Shape: `[B, H1]`. Input vectors.
112
+ wa_t_all: Shape: `[None, L, R, H1]`. All of the transposed
113
+ LoRA A matrices.
114
+ wb_t_all: Shape: `[None, L, H2, R]`. All of the transposed
115
+ LoRA B matrices.
116
+ indicies: Shape: `[B]`. Indices of the LoRA weights.
117
+ layer_idx: Layer index of LoRA weights.
118
+ scale: Scaling factor.
119
+ y_offset: Offset to apply to the starting column of y.
120
+ y_slice_size: Size of the y column slice.
121
+ """
122
+ r = wb_t_all.size(-1)
123
+ if buffer is None:
124
+ # We set the buffer to be float32 by default to avoid
125
+ # numerical inaccuracies that would otherwise happen
126
+ # due to downcasting.
127
+ buffer = torch.zeros((x.size(0), r),
128
+ dtype=torch.float32,
129
+ device=x.device)
130
+ punica_kernels.dispatch_bgmv_low_level(
131
+ buffer,
132
+ x,
133
+ wa_t_all,
134
+ indicies,
135
+ layer_idx,
136
+ 1.0,
137
+ x.size(1),
138
+ buffer.size(1),
139
+ 0,
140
+ )
141
+ punica_kernels.dispatch_bgmv_low_level(
142
+ y,
143
+ buffer,
144
+ wb_t_all,
145
+ indicies,
146
+ layer_idx,
147
+ scale,
148
+ buffer.size(1),
149
+ y_slice_size,
150
+ y_offset,
151
+ )
152
+
153
+else:
154
+
155
+ def _raise_exc(
156
+ *args, # pylint: disable=unused-argument
157
+ **kwargs # pylint: disable=unused-argument
158
+ ):
159
+ if torch.cuda.get_device_capability() < (8, 0):
160
+ raise ImportError(
161
+ "LoRA kernels require compute capability>=8.0") from import_exc
162
+ else:
163
+ raise import_exc
164
+
165
+ bgmv = _raise_exc
166
+ add_lora = _raise_exc
167
+ add_lora_slice = _raise_exc
168
+
169
+__all__ = [
170
+ "bgmv",
171
+ "add_lora",
172
+ "add_lora_slice",
173
+]
vllm/lora/request.py
new
+32
@@ -0,0 +1,32 @@
1
+from dataclasses import dataclass
2
+
3
+
4
+@dataclass
5
+class LoRARequest:
6
+ """
7
+ Request for a LoRA adapter.
8
+
9
+ Note that this class should be be used internally. For online
10
+ serving, it is recommended to not allow users to use this class but
11
+ instead provide another layer of abstraction to prevent users from
12
+ accessing unauthorized LoRA adapters.
13
+
14
+ lora_int_id must be globally unique for a given adapter.
15
+ This is currently not enforced in vLLM.
16
+ """
17
+
18
+ lora_name: str
19
+ lora_int_id: int
20
+ lora_local_path: str
21
+
22
+ def __post_init__(self):
23
+ if self.lora_int_id < 1:
24
+ raise ValueError(
25
+ f"lora_int_id must be > 0, got {self.lora_int_id}")
26
+
27
+ def __eq__(self, value: object) -> bool:
28
+ return isinstance(
29
+ value, LoRARequest) and self.lora_int_id == value.lora_int_id
30
+
31
+ def __hash__(self) -> int:
32
+ return self.lora_int_id
vllm/lora/utils.py
new
+39
@@ -0,0 +1,39 @@
1
+import logging
2
+from typing import Tuple
3
+
4
+from torch import nn
5
+
6
+logger = logging.getLogger(__name__)
7
+
8
+
9
+def replace_submodule(model: nn.Module, module_name: str,
10
+ new_module: nn.Module) -> nn.Module:
11
+ """Replace a submodule in a model with a new module."""
12
+ parent = model.get_submodule(".".join(module_name.split(".")[:-1]))
13
+ target_name = module_name.split(".")[-1]
14
+ setattr(parent, target_name, new_module)
15
+ return new_module
16
+
17
+
18
+def parse_fine_tuned_lora_name(name: str) -> Tuple[str, bool]:
19
+ """Parse the name of lora weights.
20
+
21
+ args:
22
+ name: the name of the fine-tuned LoRA, e.g.
23
+ base_model.model.dense1.weight
24
+ return:
25
+ Tuple(module_name, is_lora_a):
26
+ module_name: the name of the module, e.g. model.dense1,
27
+ is_lora_a whether the tensor is lora_a or lora_b.
28
+ """
29
+ parts = name.split(".")
30
+ assert parts[0] == "base_model"
31
+ assert parts[1] == "model"
32
+ if parts[-1] == "weight":
33
+ assert parts[-2] == "lora_A" or parts[-2] == "lora_B"
34
+ return ".".join(parts[2:-2]), parts[-2] == "lora_A"
35
+
36
+ if parts[-1] == "lora_embedding_A" or parts[-1] == "lora_embedding_B":
37
+ return ".".join(parts[2:-1]), parts[-1] == "lora_embedding_A"
38
+
39
+ raise ValueError(f"{name} is unsupported format")
vllm/lora/worker_manager.py
new
+237
@@ -0,0 +1,237 @@
1
+import logging
2
+from abc import ABC, abstractmethod, abstractproperty
3
+from typing import Any, List, Optional, Set, Type, Union
4
+
5
+import torch
6
+
7
+from vllm.lora.models import (TARGET_MODULES_QKV, LoRAModel, LoRAModelManager,
8
+ LRUCacheLoRAModelManager, create_lora_manager)
9
+from vllm.lora.request import LoRARequest
10
+from vllm.lora.layers import LoRAMapping
11
+from vllm.config import LoRAConfig
12
+
13
+logger = logging.getLogger(__name__)
14
+
15
+
16
+class WorkerLoRAManager(ABC):
17
+ """Abstract class for managing LoRA models on the worker side."""
18
+
19
+ def __init__(self, max_num_seqs: int, max_num_batched_tokens: int,
20
+ vocab_size: int, lora_config: LoRAConfig,
21
+ device: torch.device):
22
+ self.max_num_seqs = max_num_seqs
23
+ self.max_num_batched_tokens = max_num_batched_tokens
24
+ self.vocab_size = vocab_size
25
+ self.device = device
26
+ self.lora_config = lora_config
27
+
28
+ @abstractproperty
29
+ def is_enabled(self) -> bool:
30
+ ...
31
+
32
+ @abstractmethod
33
+ def create_lora_manager(
34
+ self,
35
+ model: torch.nn.Module,
36
+ target_modules: Union[str, List[str]] = TARGET_MODULES_QKV,
37
+ ) -> Any:
38
+ ...
39
+
40
+ @abstractmethod
41
+ def set_active_loras(self, lora_requests: List[LoRARequest],
42
+ lora_mapping: LoRAMapping) -> None:
43
+ ...
44
+
45
+ @abstractmethod
46
+ def add_lora(self, lora_request: LoRARequest) -> bool:
47
+ ...
48
+
49
+ @abstractmethod
50
+ def add_dummy_lora(self, lora_request: LoRARequest, rank: int) -> bool:
51
+ ...
52
+
53
+ @abstractmethod
54
+ def remove_lora(self, lora_id: int) -> bool:
55
+ ...
56
+
57
+ @abstractmethod
58
+ def remove_all_loras(self) -> bool:
59
+ ...
60
+
61
+ @abstractmethod
62
+ def list_loras(self) -> Set[int]:
63
+ ...
64
+
65
+
66
+class WorkerLoRAManager(WorkerLoRAManager):
67
+ """WorkerLoRAManager that manages LoRA models on the worker side.
68
+
69
+ Every request, the requested LoRAs will be loaded (unless they are already
70
+ loaded), and every other LoRA will be unloaded."""
71
+
72
+ _lora_manager_cls: Type[LoRAModelManager] = LoRAModelManager
73
+
74
+ def __init__(
75
+ self,
76
+ max_num_seqs: int,
77
+ max_num_batched_tokens: int,
78
+ vocab_size: int,
79
+ lora_config: LoRAConfig,
80
+ device: torch.device,
81
+ lora_model_cls: Type[LoRAModel] = LoRAModel,
82
+ ):
83
+ self._lora_manager: Optional[LoRAModelManager] = None
84
+ self._lora_model_cls = lora_model_cls
85
+ super().__init__(max_num_seqs, max_num_batched_tokens, vocab_size,
86
+ lora_config, device)
87
+
88
+ @property
89
+ def is_enabled(self) -> bool:
90
+ return True
91
+
92
+ def create_lora_manager(
93
+ self,
94
+ model: torch.nn.Module,
95
+ target_modules: Union[str, List[str]] = TARGET_MODULES_QKV,
96
+ ) -> Any:
97
+ lora_manager = create_lora_manager(
98
+ model,
99
+ max_num_seqs=self.max_num_seqs,
100
+ max_num_batched_tokens=self.max_num_batched_tokens,
101
+ target_modules=target_modules,
102
+ vocab_size=self.vocab_size,
103
+ lora_config=self.lora_config,
104
+ lora_manager_cls=self._lora_manager_cls,
105
+ )
106
+ self._lora_manager: LoRAModelManager = lora_manager
107
+ return lora_manager.model
108
+
109
+ def set_active_loras(self, lora_requests: List[LoRARequest],
110
+ lora_mapping: LoRAMapping) -> None:
111
+ self._apply_loras(lora_requests)
112
+ self._lora_manager.set_lora_mapping(lora_mapping)
113
+
114
+ def _apply_loras(self, lora_requests: List[LoRARequest]) -> None:
115
+ loras_that_exist = self.list_loras()
116
+ loras_map = {
117
+ lora_request.lora_int_id: lora_request
118
+ for lora_request in lora_requests if lora_request
119
+ }
120
+ if len(loras_map) > self._lora_manager.lora_slots:
121
+ raise RuntimeError(
122
+ f"Number of requested LoRAs ({len(loras_map)}) is greater "
123
+ "than the number of GPU LoRA slots "
124
+ f"({self._lora_manager.lora_slots}).")
125
+
126
+ new_loras = set(loras_map)
127
+ loras_to_add = new_loras - loras_that_exist
128
+ loras_to_remove = loras_that_exist - new_loras
129
+
130
+ for lora_id in loras_to_remove:
131
+ self.remove_lora(lora_id)
132
+
133
+ for lora_id in loras_to_add:
134
+ self.add_lora(loras_map[lora_id])
135
+
136
+ def _load_lora(self, lora_request: LoRARequest) -> LoRAModel:
137
+ try:
138
+ lora = self._lora_model_cls.from_local_checkpoint(
139
+ lora_request.lora_local_path,
140
+ lora_model_id=lora_request.lora_int_id,
141
+ device="cpu",
142
+ dtype=self.lora_config.lora_dtype,
143
+ target_embedding_padding=self.vocab_size +
144
+ self.lora_config.lora_extra_vocab_size,
145
+ )
146
+ except Exception as e:
147
+ raise RuntimeError(
148
+ f"Loading lora {lora_request.lora_local_path} failed") from e
149
+ if lora.rank > self.lora_config.max_lora_rank:
150
+ raise ValueError(
151
+ f"LoRA rank {lora.rank} is greater than max_lora_rank "
152
+ f"{self.lora_config.max_lora_rank}.")
153
+ if lora.extra_vocab_size > self.lora_config.lora_extra_vocab_size:
154
+ raise ValueError(
155
+ f"LoRA added vocab size {lora.extra_vocab_size} is greater than "
156
+ f"lora_extra_vocab_size {self.lora_config.lora_extra_vocab_size}."
157
+ )
158
+ return lora
159
+
160
+ def add_dummy_lora(self, lora_request: LoRARequest, rank: int) -> bool:
161
+ if lora_request.lora_int_id in self.list_loras():
162
+ return False
163
+ return self._lora_manager.add_lora(
164
+ self._lora_manager.create_dummy_lora(lora_request.lora_int_id,
165
+ rank))
166
+
167
+ def add_lora(self, lora_request: LoRARequest) -> bool:
168
+ if lora_request.lora_int_id in self.list_loras():
169
+ return False
170
+ lora = self._load_lora(lora_request)
171
+ loaded = self._lora_manager.add_lora(lora)
172
+ self._lora_manager.activate_lora(lora.id)
173
+ return loaded
174
+
175
+ def remove_lora(self, lora_id: int) -> bool:
176
+ return self._lora_manager.remove_lora(lora_id)
177
+
178
+ def remove_all_loras(self) -> bool:
179
+ self._lora_manager.remove_all_loras()
180
+
181
+ def list_loras(self) -> Set[int]:
182
+ return set(self._lora_manager.list_loras())
183
+
184
+
185
+class LRUCacheWorkerLoRAManager(WorkerLoRAManager):
186
+ """WorkerLoRAManager that manages LoRA models on the worker side.
187
+
188
+ Uses an LRU Cache. Every request, the requested LoRAs will be loaded
189
+ (unless they are already loaded) and least recently used LoRAs will
190
+ be unloaded if the cache is above capacity."""
191
+
192
+ _lora_manager_cls: Type[
193
+ LRUCacheLoRAModelManager] = LRUCacheLoRAModelManager
194
+
195
+ def create_lora_manager(
196
+ self,
197
+ model: torch.nn.Module,
198
+ target_modules: Union[str, List[str]] = TARGET_MODULES_QKV,
199
+ ) -> Any:
200
+ lora_manager = create_lora_manager(
201
+ model,
202
+ target_modules=target_modules,
203
+ lora_manager_cls=self._lora_manager_cls,
204
+ max_num_seqs=self.max_num_seqs,
205
+ vocab_size=self.vocab_size,
206
+ lora_config=self.lora_config,
207
+ max_num_batched_tokens=self.max_num_batched_tokens,
208
+ )
209
+ self._lora_manager: LRUCacheLoRAModelManager = lora_manager
210
+ return lora_manager.model
211
+
212
+ def _apply_loras(self, lora_requests: List[LoRARequest]) -> None:
213
+ loras_map = {
214
+ lora_request.lora_int_id: lora_request
215
+ for lora_request in lora_requests if lora_request
216
+ }
217
+ if len(loras_map) > self._lora_manager.lora_slots:
218
+ raise RuntimeError(
219
+ f"Number of requested LoRAs ({len(loras_map)}) is greater "
220
+ "than the number of GPU LoRA slots "
221
+ f"({self._lora_manager.lora_slots}).")
222
+ for lora in loras_map.values():
223
+ self.add_lora(lora)
224
+
225
+ def add_lora(self, lora_request: LoRARequest) -> bool:
226
+ if lora_request.lora_int_id not in self.list_loras():
227
+ # Remove before we load the new lora to save memory
228
+ if len(self._lora_manager) + 1 > self._lora_manager.capacity:
229
+ self._lora_manager.remove_oldest_lora()
230
+ lora = self._load_lora(lora_request)
231
+ loaded = self._lora_manager.add_lora(lora)
232
+ else:
233
+ # If the lora is already loaded, just touch it to
234
+ # update its position in the caches
235
+ loaded = self._lora_manager.get_lora(lora_request.lora_int_id)
236
+ self._lora_manager.activate_lora(lora_request.lora_int_id)
237
+ return loaded
vllm/model_executor/layers/sampler.py
+18
-17
@@ -27,9 +27,25 @@ class Sampler(nn.Module):
27
parameters (e.g., sampling method, temperature, top-p, top-k, etc.).
28
"""
29
30
- def __init__(self, vocab_size: int) -> None:
30
+ def __init__(self,
31
+ vocab_size: int,
32
+ org_vocab_size: Optional[int] = None) -> None:
33
super().__init__()
34
self.vocab_size = vocab_size
35
+ # original vocabulary size (without LoRA).
36
+ self.org_vocab_size = org_vocab_size or vocab_size
37
+
38
+ def _get_logits(self, hidden_states: torch.Tensor, embedding: torch.Tensor,
39
+ embedding_bias: Optional[torch.Tensor]) -> torch.Tensor:
40
+ # Get the logits for the next tokens.
41
+ logits = torch.matmul(hidden_states, embedding.t())
42
+ if embedding_bias is not None:
43
+ logits += embedding_bias
44
+ logits = tensor_model_parallel_gather(logits)
45
+ # Remove paddings in vocab (if any).
46
+ if logits is not None:
47
+ logits = logits[:, :self.org_vocab_size]
48
+ return logits
49
50
def forward(
51
self,
@@ -42,8 +58,7 @@ class Sampler(nn.Module):
58
hidden_states = _prune_hidden_states(hidden_states, sampling_metadata)
59
60
# Get the logits for the next tokens.
45
- logits = _get_logits(hidden_states, embedding, embedding_bias,
46
- self.vocab_size)
61
+ logits = self._get_logits(hidden_states, embedding, embedding_bias)
62
63
# Only perform sampling in the driver worker.
64
# Note: `_get_logits` is still distributed across TP workers because
@@ -98,20 +113,6 @@ class Sampler(nn.Module):
113
prompt_logprobs, sample_logprobs)
114
115
101
-def _get_logits(hidden_states: torch.Tensor, embedding: torch.Tensor,
102
- embedding_bias: Optional[torch.Tensor],
103
- vocab_size: int) -> Optional[torch.Tensor]:
104
- # Get the logits for the next tokens.
105
- logits = torch.matmul(hidden_states, embedding.t())
106
- if embedding_bias is not None:
107
- logits += embedding_bias
108
- logits = tensor_model_parallel_gather(logits)
109
- # Remove paddings in vocab (if any).
110
- if logits is not None:
111
- logits = logits[:, :vocab_size]
112
- return logits
113
-
114
-
116
def _prune_hidden_states(
117
hidden_states: torch.Tensor,
118
sampling_metadata: SamplingMetadata,
vllm/model_executor/layers/vocab_parallel_embedding.py
+20
-6
@@ -13,8 +13,11 @@ from vllm.model_executor.parallel_utils.communication_op import (
13
tensor_model_parallel_all_reduce)
14
from vllm.model_executor.utils import set_weight_attrs
15
16
+DEFAULT_VOCAB_PADDING_SIZE = 64
17
17
-def pad_vocab_size(vocab_size: int, pad_to: int = 64) -> int:
18
+
19
+def pad_vocab_size(vocab_size: int,
20
+ pad_to: int = DEFAULT_VOCAB_PADDING_SIZE) -> int:
21
"""Pad the vocab size to the given value."""
22
return ((vocab_size + pad_to - 1) // pad_to) * pad_to
23
@@ -43,17 +46,23 @@ class VocabParallelEmbedding(torch.nn.Module):
46
num_embeddings: vocabulary size.
47
embedding_dim: size of hidden state.
48
params_dtype: type of the parameters.
49
+ org_num_embeddings: original vocabulary size (without LoRA).
50
+ padding_size: padding size for the vocabulary.
51
"""
52
53
def __init__(self,
54
num_embeddings: int,
55
embedding_dim: int,
51
- params_dtype: Optional[torch.dtype] = None):
56
+ params_dtype: Optional[torch.dtype] = None,
57
+ org_num_embeddings: Optional[int] = None,
58
+ padding_size: int = DEFAULT_VOCAB_PADDING_SIZE):
59
super().__init__()
60
61
# Keep the input dimensions.
62
self.num_embeddings = num_embeddings
56
- self.num_embeddings_padded = pad_vocab_size(num_embeddings)
63
+ self.org_vocab_size = org_num_embeddings or num_embeddings
64
+ self.num_embeddings_padded = pad_vocab_size(num_embeddings,
65
+ padding_size)
66
self.embedding_dim = embedding_dim
67
if params_dtype is None:
68
params_dtype = torch.get_default_dtype()
@@ -77,7 +86,7 @@ class VocabParallelEmbedding(torch.nn.Module):
86
87
def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor):
88
parallel_dim = param.parallel_dim
80
- assert loaded_weight.shape[parallel_dim] == self.num_embeddings
89
+ assert loaded_weight.shape[parallel_dim] == self.org_vocab_size
90
loaded_weight = loaded_weight[self.vocab_start_index:self.
91
vocab_end_index]
92
param[:loaded_weight.shape[0]].data.copy_(loaded_weight)
@@ -114,14 +123,19 @@ class ParallelLMHead(VocabParallelEmbedding):
123
embedding_dim: size of hidden state.
124
bias: whether to use bias.
125
params_dtype: type of the parameters.
126
+ org_num_embeddings: original vocabulary size (without LoRA).
127
+ padding_size: padding size for the vocabulary.
128
"""
129
130
def __init__(self,
131
num_embeddings: int,
132
embedding_dim: int,
133
bias: bool = False,
123
- params_dtype: Optional[torch.dtype] = None):
124
- super().__init__(num_embeddings, embedding_dim, params_dtype)
134
+ params_dtype: Optional[torch.dtype] = None,
135
+ org_num_embeddings: Optional[int] = None,
136
+ padding_size: int = DEFAULT_VOCAB_PADDING_SIZE):
137
+ super().__init__(num_embeddings, embedding_dim, params_dtype,
138
+ org_num_embeddings, padding_size)
139
if bias:
140
self.bias = Parameter(
141
torch.empty(self.num_embeddings_per_partition,
vllm/model_executor/model_loader.py
+15
-4
@@ -1,12 +1,12 @@
1
"""Utilities for selecting and loading models."""
2
import contextlib
3
-from typing import Type
3
+from typing import Optional, Type
4
5
import torch
6
import torch.nn as nn
7
from transformers import PretrainedConfig
8
9
-from vllm.config import ModelConfig
9
+from vllm.config import ModelConfig, LoRAConfig
10
from vllm.model_executor.models import ModelRegistry
11
from vllm.model_executor.weight_utils import (get_quant_config,
12
initialize_dummy_weights)
@@ -32,7 +32,8 @@ def _get_model_architecture(config: PretrainedConfig) -> Type[nn.Module]:
32
f"Supported architectures: {ModelRegistry.get_supported_archs()}")
33
34
35
-def get_model(model_config: ModelConfig) -> nn.Module:
35
+def get_model(model_config: ModelConfig,
36
+ lora_config: Optional[LoRAConfig] = None) -> nn.Module:
37
model_class = _get_model_architecture(model_config.hf_config)
38
39
# Get the (maybe quantized) linear method.
@@ -62,7 +63,17 @@ def get_model(model_config: ModelConfig) -> nn.Module:
63
# Create a model instance.
64
# The weights will be initialized as empty tensors.
65
with torch.device("cuda"):
65
- model = model_class(model_config.hf_config, linear_method)
66
+ if getattr(model_class, "supports_lora", False):
67
+ model = model_class(model_config.hf_config, linear_method,
68
+ lora_config)
69
+ elif lora_config:
70
+ raise ValueError(
71
+ f"Model {model_class.__name__} does not support LoRA, "
72
+ "but LoRA is enabled. Support for this model may "
73
+ "be added in the future. If this is important to you, "
74
+ "please open an issue on github.")
75
+ else:
76
+ model = model_class(model_config.hf_config, linear_method)
77
if model_config.load_format == "dummy":
78
# NOTE(woosuk): For accurate performance evaluation, we assign
79
# random values to the weights.
vllm/model_executor/models/llama.py
+25
-6
@@ -38,13 +38,14 @@ from vllm.model_executor.layers.linear import (LinearMethodBase,
38
from vllm.model_executor.layers.rotary_embedding import get_rope
39
from vllm.model_executor.layers.sampler import Sampler
40
from vllm.model_executor.layers.vocab_parallel_embedding import (
41
- VocabParallelEmbedding, ParallelLMHead)
41
+ VocabParallelEmbedding, ParallelLMHead, DEFAULT_VOCAB_PADDING_SIZE)
42
from vllm.model_executor.parallel_utils.parallel_state import (
43
get_tensor_model_parallel_world_size)
44
from vllm.model_executor.sampling_metadata import SamplingMetadata
45
from vllm.model_executor.weight_utils import (default_weight_loader,
46
hf_model_weights_iterator)
47
from vllm.sequence import SamplerOutput
48
+from vllm.config import LoRAConfig
49
50
KVCache = Tuple[torch.Tensor, torch.Tensor]
51
@@ -225,14 +226,19 @@ class LlamaModel(nn.Module):
226
self,
227
config: LlamaConfig,
228
linear_method: Optional[LinearMethodBase] = None,
229
+ lora_config: Optional[LoRAConfig] = None,
230
) -> None:
231
super().__init__()
232
self.config = config
233
self.padding_idx = config.pad_token_id
232
- self.vocab_size = config.vocab_size
234
+ lora_vocab = (lora_config.lora_extra_vocab_size *
235
+ (lora_config.max_loras or 1)) if lora_config else 0
236
+ self.vocab_size = config.vocab_size + lora_vocab
237
+ self.org_vocab_size = config.vocab_size
238
self.embed_tokens = VocabParallelEmbedding(
234
- config.vocab_size,
239
+ self.vocab_size,
240
config.hidden_size,
241
+ org_num_embeddings=config.vocab_size,
242
)
243
self.layers = nn.ModuleList([
244
LlamaDecoderLayer(config, linear_method)
@@ -263,18 +269,31 @@ class LlamaModel(nn.Module):
269
270
271
class LlamaForCausalLM(nn.Module):
272
+ supports_lora = True
273
274
def __init__(
275
self,
276
config: LlamaConfig,
277
linear_method: Optional[LinearMethodBase] = None,
278
+ lora_config: Optional[LoRAConfig] = None,
279
) -> None:
280
super().__init__()
281
self.config = config
282
self.linear_method = linear_method
275
- self.model = LlamaModel(config, linear_method)
276
- self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
277
- self.sampler = Sampler(config.vocab_size)
283
+ self.model = LlamaModel(config, linear_method, lora_config=lora_config)
284
+ unpadded_vocab_size = config.vocab_size
285
+ if lora_config:
286
+ unpadded_vocab_size += lora_config.lora_extra_vocab_size
287
+ self.lm_head = ParallelLMHead(
288
+ unpadded_vocab_size,
289
+ config.hidden_size,
290
+ org_num_embeddings=config.vocab_size,
291
+ padding_size=DEFAULT_VOCAB_PADDING_SIZE
292
+ # We need bigger padding if using lora for kernel
293
+ # compatibility
294
+ if not lora_config else lora_config.lora_vocab_padding_size,
295
+ )
296
+ self.sampler = Sampler(unpadded_vocab_size, config.vocab_size)
297
298
def forward(
299
self,
vllm/model_executor/models/mistral.py
+27
-6
@@ -38,13 +38,14 @@ from vllm.model_executor.layers.linear import (LinearMethodBase,
38
from vllm.model_executor.layers.rotary_embedding import get_rope
39
from vllm.model_executor.layers.sampler import Sampler
40
from vllm.model_executor.layers.vocab_parallel_embedding import (
41
- VocabParallelEmbedding, ParallelLMHead)
41
+ VocabParallelEmbedding, ParallelLMHead, DEFAULT_VOCAB_PADDING_SIZE)
42
from vllm.model_executor.parallel_utils.parallel_state import (
43
get_tensor_model_parallel_world_size)
44
from vllm.model_executor.sampling_metadata import SamplingMetadata
45
from vllm.model_executor.weight_utils import (default_weight_loader,
46
hf_model_weights_iterator)
47
from vllm.sequence import SamplerOutput
48
+from vllm.config import LoRAConfig
49
50
KVCache = Tuple[torch.Tensor, torch.Tensor]
51
@@ -220,15 +221,20 @@ class MistralModel(nn.Module):
221
self,
222
config: MistralConfig,
223
linear_method: Optional[LinearMethodBase] = None,
224
+ lora_config: Optional[LoRAConfig] = None,
225
) -> None:
226
super().__init__()
227
self.config = config
228
self.padding_idx = config.pad_token_id
227
- self.vocab_size = config.vocab_size
229
+ lora_vocab = (lora_config.lora_extra_vocab_size *
230
+ (lora_config.max_loras or 1)) if lora_config else 0
231
+ self.vocab_size = config.vocab_size + lora_vocab
232
+ self.org_vocab_size = config.vocab_size
233
234
self.embed_tokens = VocabParallelEmbedding(
230
- config.vocab_size,
235
+ self.vocab_size,
236
config.hidden_size,
237
+ org_num_embeddings=config.vocab_size,
238
)
239
self.layers = nn.ModuleList([
240
MistralDecoderLayer(config, linear_method)
@@ -259,18 +265,33 @@ class MistralModel(nn.Module):
265
266
267
class MistralForCausalLM(nn.Module):
268
+ supports_lora = True
269
270
def __init__(
271
self,
272
config: MistralConfig,
273
linear_method: Optional[LinearMethodBase] = None,
274
+ lora_config: Optional[LoRAConfig] = None,
275
) -> None:
276
super().__init__()
277
self.config = config
278
self.linear_method = linear_method
271
- self.model = MistralModel(config, linear_method)
272
- self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
273
- self.sampler = Sampler(config.vocab_size)
279
+ self.model = MistralModel(config,
280
+ linear_method,
281
+ lora_config=lora_config)
282
+ unpadded_vocab_size = config.vocab_size
283
+ if lora_config:
284
+ unpadded_vocab_size += lora_config.lora_extra_vocab_size
285
+ self.lm_head = ParallelLMHead(
286
+ unpadded_vocab_size,
287
+ config.hidden_size,
288
+ org_num_embeddings=config.vocab_size,
289
+ padding_size=DEFAULT_VOCAB_PADDING_SIZE
290
+ # We need bigger padding if using lora for kernel
291
+ # compatibility
292
+ if not lora_config else lora_config.lora_vocab_padding_size,
293
+ )
294
+ self.sampler = Sampler(unpadded_vocab_size, config.vocab_size)
295
296
def forward(
297
self,
vllm/model_executor/parallel_utils/parallel_state.py
+5
-1
@@ -195,10 +195,14 @@ def get_pipeline_model_parallel_prev_rank():
195
196
197
def destroy_model_parallel():
198
- """Set the groups to none."""
198
+ """Set the groups to none and destroy them."""
199
global _TENSOR_MODEL_PARALLEL_GROUP
200
+ if _TENSOR_MODEL_PARALLEL_GROUP:
201
+ torch.distributed.destroy_process_group(_TENSOR_MODEL_PARALLEL_GROUP)
202
_TENSOR_MODEL_PARALLEL_GROUP = None
203
global _PIPELINE_MODEL_PARALLEL_GROUP
204
+ if _PIPELINE_MODEL_PARALLEL_GROUP:
205
+ torch.distributed.destroy_process_group(_PIPELINE_MODEL_PARALLEL_GROUP)
206
_PIPELINE_MODEL_PARALLEL_GROUP = None
207
global _PIPELINE_GLOBAL_RANKS
208
_PIPELINE_GLOBAL_RANKS = None
vllm/outputs.py
+16
-3
@@ -2,6 +2,7 @@ from typing import List, Optional
2
3
from vllm.sequence import (PromptLogprobs, SampleLogprobs, SequenceGroup,
4
SequenceStatus)
5
+from vllm.lora.request import LoRARequest
6
7
8
class CompletionOutput:
@@ -16,6 +17,7 @@ class CompletionOutput:
17
logprobs: The log probabilities of the top probability words at each
18
position if the logprobs are requested.
19
finish_reason: The reason why the sequence is finished.
20
+ lora_request: The LoRA request that was used to generate the output.
21
"""
22
23
def __init__(
@@ -26,6 +28,7 @@ class CompletionOutput:
28
cumulative_logprob: float,
29
logprobs: Optional[SampleLogprobs],
30
finish_reason: Optional[str] = None,
31
+ lora_request: Optional[LoRARequest] = None,
32
) -> None:
33
self.index = index
34
self.text = text
@@ -33,6 +36,7 @@ class CompletionOutput:
36
self.cumulative_logprob = cumulative_logprob
37
self.logprobs = logprobs
38
self.finish_reason = finish_reason
39
+ self.lora_request = lora_request
40
41
def finished(self) -> bool:
42
return self.finish_reason is not None
@@ -56,6 +60,7 @@ class RequestOutput:
60
prompt_logprobs: The log probabilities to return per prompt token.
61
outputs: The output sequences of the request.
62
finished: Whether the whole request is finished.
63
+ lora_request: The LoRA request that was used to generate the output.
64
"""
65
66
def __init__(
@@ -66,6 +71,7 @@ class RequestOutput:
71
prompt_logprobs: Optional[PromptLogprobs],
72
outputs: List[CompletionOutput],
73
finished: bool,
74
+ lora_request: Optional[LoRARequest] = None,
75
) -> None:
76
self.request_id = request_id
77
self.prompt = prompt
@@ -73,6 +79,7 @@ class RequestOutput:
79
self.prompt_logprobs = prompt_logprobs
80
self.outputs = outputs
81
self.finished = finished
82
+ self.lora_request = lora_request
83
84
@classmethod
85
def from_seq_group(cls, seq_group: SequenceGroup) -> "RequestOutput":
@@ -108,8 +115,13 @@ class RequestOutput:
115
prompt_token_ids = seq_group.prompt_token_ids
116
prompt_logprobs = seq_group.prompt_logprobs
117
finished = seq_group.is_finished()
111
- return cls(seq_group.request_id, prompt, prompt_token_ids,
112
- prompt_logprobs, outputs, finished)
118
+ return cls(seq_group.request_id,
119
+ prompt,
120
+ prompt_token_ids,
121
+ prompt_logprobs,
122
+ outputs,
123
+ finished,
124
+ lora_request=seq_group.lora_request)
125
126
def __repr__(self) -> str:
127
return (f"RequestOutput(request_id={self.request_id}, "
@@ -117,4 +129,5 @@ class RequestOutput:
129
f"prompt_token_ids={self.prompt_token_ids}, "
130
f"prompt_logprobs={self.prompt_logprobs}, "
131
f"outputs={self.outputs}, "
120
- f"finished={self.finished})")
132
+ f"finished={self.finished}, "
133
+ f"lora_request={self.lora_request})")
vllm/prefix.py
+3
-2
@@ -74,13 +74,14 @@ class PrefixPool:
74
new_length = len(token_ids) // self.block_size * self.block_size
75
return tuple(token_ids[:new_length])
76
77
- def add_or_get_prefix(self, token_ids: Sequence[int]) -> Optional[Prefix]:
77
+ def add_or_get_prefix(self, token_ids: Sequence[int],
78
+ lora_int_id: int) -> Optional[Prefix]:
79
token_ids = self._truncate_token_ids(token_ids)
80
if len(token_ids) == 0:
81
# Prefix is empty.
82
return None
83
prefix = Prefix(token_ids, self.block_size)
83
- prefix_hash = hash(prefix)
84
+ prefix_hash = hash((prefix, lora_int_id))
85
if prefix_hash not in self.prefixes:
86
self.prefixes[prefix_hash] = prefix
87
return self.prefixes[prefix_hash]
vllm/sequence.py
+22
@@ -6,6 +6,7 @@ from typing import Dict, List, Optional, Union
6
from vllm.block import LogicalTokenBlock
7
from vllm.prefix import Prefix
8
from vllm.sampling_params import SamplingParams
9
+from vllm.lora.request import LoRARequest
10
11
PromptLogprobs = List[Optional[Dict[int, float]]]
12
SampleLogprobs = List[Dict[int, float]]
@@ -106,6 +107,7 @@ class Sequence:
107
prompt_token_ids: The token IDs of the prompt.
108
block_size: The block size of the sequence. Should be the same as the
109
block size used by the block manager and cache engine.
110
+ lora_request: LoRA request.
111
"""
112
113
def __init__(
@@ -114,10 +116,12 @@ class Sequence:
116
prompt: str,
117
prompt_token_ids: List[int],
118
block_size: int,
119
+ lora_request: Optional[LoRARequest] = None,
120
) -> None:
121
self.seq_id = seq_id
122
self.prompt = prompt
123
self.block_size = block_size
124
+ self.lora_request = lora_request
125
126
self.data = SequenceData(prompt_token_ids)
127
self.output_logprobs: SampleLogprobs = []
@@ -134,6 +138,10 @@ class Sequence:
138
# Input + output tokens
139
self.tokens: Optional[List[str]] = None
140
141
+ @property
142
+ def lora_int_id(self) -> int:
143
+ return self.lora_request.lora_int_id if self.lora_request else 0
144
+
145
def _append_logical_block(self) -> None:
146
block = LogicalTokenBlock(
147
block_number=len(self.logical_token_blocks),
@@ -229,6 +237,7 @@ class SequenceGroup:
237
seqs: The list of sequences.
238
sampling_params: The sampling parameters used to generate the outputs.
239
arrival_time: The arrival time of the request.
240
+ lora_request: LoRA request.
241
prefix: The prefix of the prompt of the sequence group.
242
"""
243
@@ -238,12 +247,14 @@ class SequenceGroup:
247
seqs: List[Sequence],
248
sampling_params: SamplingParams,
249
arrival_time: float,
250
+ lora_request: Optional[LoRARequest] = None,
251
prefix: Optional[Prefix] = None,
252
) -> None:
253
self.request_id = request_id
254
self.seqs_dict = {seq.seq_id: seq for seq in seqs}
255
self.sampling_params = sampling_params
256
self.arrival_time = arrival_time
257
+ self.lora_request = lora_request
258
self.prefix: Optional[Prefix] = prefix
259
self.prompt_logprobs: Optional[PromptLogprobs] = None
260
@@ -259,6 +270,10 @@ class SequenceGroup:
270
# We use the prompt of an arbitrary sequence.
271
return next(iter(self.seqs_dict.values())).data.prompt_token_ids
272
273
+ @property
274
+ def lora_int_id(self) -> int:
275
+ return self.lora_request.lora_int_id if self.lora_request else 0
276
+
277
def get_max_num_running_seqs(self) -> int:
278
"""The maximum number of sequences running in parallel in the remaining
279
lifetime of the request."""
@@ -338,6 +353,7 @@ class SequenceGroupMetadata:
353
sampling_params: The sampling parameters used to generate the outputs.
354
block_tables: The block tables. (Seq id -> list of physical block
355
numbers)
356
+ lora_request: LoRA request.
357
prefix: The prefix of the prompt of the sequence group.
358
"""
359
@@ -348,6 +364,7 @@ class SequenceGroupMetadata:
364
seq_data: Dict[int, SequenceData],
365
sampling_params: SamplingParams,
366
block_tables: Dict[int, List[int]],
367
+ lora_request: Optional[LoRARequest] = None,
368
prefix: Optional[Prefix] = None,
369
) -> None:
370
self.request_id = request_id
@@ -355,8 +372,13 @@ class SequenceGroupMetadata:
372
self.seq_data = seq_data
373
self.sampling_params = sampling_params
374
self.block_tables = block_tables
375
+ self.lora_request = lora_request
376
self.prefix = prefix
377
378
+ @property
379
+ def lora_int_id(self) -> int:
380
+ return self.lora_request.lora_int_id if self.lora_request else 0
381
+
382
383
class SequenceOutput:
384
"""The model output associated with a sequence.
vllm/transformers_utils/tokenizer.py
+80
@@ -4,6 +4,8 @@ from transformers import (AutoTokenizer, PreTrainedTokenizer,
4
PreTrainedTokenizerFast)
5
6
from vllm.logger import init_logger
7
+from vllm.lora.request import LoRARequest
8
+from vllm.utils import make_async, LRUCache
9
from vllm.transformers_utils.tokenizers import *
10
11
logger = init_logger(__name__)
@@ -65,6 +67,84 @@ def get_tokenizer(
67
return tokenizer
68
69
70
+def get_lora_tokenizer(lora_request: LoRARequest, *args,
71
+ **kwargs) -> Optional[PreTrainedTokenizer]:
72
+ if lora_request is None:
73
+ return None
74
+ try:
75
+ tokenizer = get_tokenizer(lora_request.lora_local_path, *args,
76
+ **kwargs)
77
+ except OSError as e:
78
+ # No tokenizer was found in the LoRA folder,
79
+ # use base model tokenizer
80
+ logger.warning(
81
+ f"No tokenizer found in {lora_request.lora_local_path}, "
82
+ "using base model tokenizer instead. "
83
+ f"(Exception: {str(e)})")
84
+ tokenizer = None
85
+ return tokenizer
86
+
87
+
88
+get_lora_tokenizer_async = make_async(get_lora_tokenizer)
89
+
90
+
91
+class TokenizerGroup:
92
+ """A group of tokenizers that can be used for LoRA adapters."""
93
+
94
+ def __init__(self, tokenizer_id: str, enable_lora: bool, max_num_seqs: int,
95
+ max_input_length: Optional[int], **tokenizer_config):
96
+ self.tokenizer_id = tokenizer_id
97
+ self.tokenizer_config = tokenizer_config
98
+ self.enable_lora = enable_lora
99
+ self.max_input_length = max_input_length
100
+ self.tokenizer = get_tokenizer(self.tokenizer_id, **tokenizer_config)
101
+ if enable_lora:
102
+ self.lora_tokenizers = LRUCache(capacity=max_num_seqs)
103
+ else:
104
+ self.lora_tokenizers = None
105
+
106
+ def encode(self,
107
+ prompt: str,
108
+ request_id: Optional[str] = None,
109
+ lora_request: Optional[LoRARequest] = None) -> List[int]:
110
+ tokenizer = self.get_lora_tokenizer(lora_request)
111
+ return tokenizer.encode(prompt)
112
+
113
+ async def encode_async(
114
+ self,
115
+ prompt: str,
116
+ request_id: Optional[str] = None,
117
+ lora_request: Optional[LoRARequest] = None) -> List[int]:
118
+ tokenizer = await self.get_lora_tokenizer_async(lora_request)
119
+ return tokenizer.encode(prompt)
120
+
121
+ def get_lora_tokenizer(
122
+ self,
123
+ lora_request: Optional[LoRARequest]) -> "PreTrainedTokenizer":
124
+ if not lora_request or not self.enable_lora:
125
+ return self.tokenizer
126
+ if lora_request.lora_int_id not in self.lora_tokenizers:
127
+ tokenizer = (get_lora_tokenizer(
128
+ lora_request, **self.tokenizer_config) or self.tokenizer)
129
+ self.lora_tokenizers.put(lora_request.lora_int_id, tokenizer)
130
+ return tokenizer
131
+ else:
132
+ return self.lora_tokenizers.get(lora_request.lora_int_id)
133
+
134
+ async def get_lora_tokenizer_async(
135
+ self,
136
+ lora_request: Optional[LoRARequest]) -> "PreTrainedTokenizer":
137
+ if not lora_request or not self.enable_lora:
138
+ return self.tokenizer
139
+ if lora_request.lora_int_id not in self.lora_tokenizers:
140
+ tokenizer = (await get_lora_tokenizer_async(
141
+ lora_request, **self.tokenizer_config) or self.tokenizer)
142
+ self.lora_tokenizers.put(lora_request.lora_int_id, tokenizer)
143
+ return tokenizer
144
+ else:
145
+ return self.lora_tokenizers.get(lora_request.lora_int_id)
146
+
147
+
148
def _convert_tokens_to_string_with_added_encoders(
149
tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
150
output_tokens: List[str],
vllm/utils.py
+90
@@ -7,6 +7,17 @@ from typing import List
7
8
import psutil
9
import torch
10
+import asyncio
11
+from functools import partial
12
+from typing import (
13
+ Awaitable,
14
+ Callable,
15
+ TypeVar,
16
+)
17
+from collections import OrderedDict
18
+from typing import Any, Hashable, Optional
19
+
20
+T = TypeVar("T")
21
22
23
class Device(enum.Enum):
@@ -28,6 +39,69 @@ class Counter:
39
self.counter = 0
40
41
42
+class LRUCache:
43
+
44
+ def __init__(self, capacity: int):
45
+ self.cache = OrderedDict()
46
+ self.capacity = capacity
47
+
48
+ def __contains__(self, key: Hashable) -> bool:
49
+ return key in self.cache
50
+
51
+ def __len__(self) -> int:
52
+ return len(self.cache)
53
+
54
+ def __getitem__(self, key: Hashable) -> Any:
55
+ return self.get(key)
56
+
57
+ def __setitem__(self, key: Hashable, value: Any) -> None:
58
+ self.put(key, value)
59
+
60
+ def __delitem__(self, key: Hashable) -> None:
61
+ self.pop(key)
62
+
63
+ def touch(self, key: Hashable) -> None:
64
+ self.cache.move_to_end(key)
65
+
66
+ def get(self, key: Hashable, default_value: Optional[Any] = None) -> int:
67
+ if key in self.cache:
68
+ value = self.cache[key]
69
+ self.cache.move_to_end(key)
70
+ else:
71
+ value = default_value
72
+ return value
73
+
74
+ def put(self, key: Hashable, value: Any) -> None:
75
+ self.cache[key] = value
76
+ self.cache.move_to_end(key)
77
+ self._remove_old_if_needed()
78
+
79
+ def _on_remove(self, key: Hashable, value: Any):
80
+ pass
81
+
82
+ def remove_oldest(self):
83
+ if not self.cache:
84
+ return
85
+ key, value = self.cache.popitem(last=False)
86
+ self._on_remove(key, value)
87
+
88
+ def _remove_old_if_needed(self) -> None:
89
+ while len(self.cache) > self.capacity:
90
+ self.remove_oldest()
91
+
92
+ def pop(self, key: int, default_value: Optional[Any] = None) -> Any:
93
+ run_on_remove = key in self.cache
94
+ value = self.cache.pop(key, default_value)
95
+ if run_on_remove:
96
+ self._on_remove(key, value)
97
+ return value
98
+
99
+ def clear(self):
100
+ while len(self.cache) > 0:
101
+ self.remove_oldest()
102
+ self.cache.clear()
103
+
104
+
105
def is_hip() -> bool:
106
return torch.version.hip is not None
107
@@ -59,6 +133,22 @@ def in_wsl() -> bool:
133
return "microsoft" in " ".join(uname()).lower()
134
135
136
+def make_async(func: Callable[..., T]) -> Callable[..., Awaitable[T]]:
137
+ """Take a blocking function, and run it on in an executor thread.
138
+
139
+ This function prevents the blocking function from blocking the
140
+ asyncio event loop.
141
+ The code in this function needs to be thread safe.
142
+ """
143
+
144
+ def _async_wrapper(*args, **kwargs) -> asyncio.Future:
145
+ loop = asyncio.get_event_loop()
146
+ p_func = partial(func, *args, **kwargs)
147
+ return loop.run_in_executor(executor=None, func=p_func)
148
+
149
+ return _async_wrapper
150
+
151
+
152
def get_ip() -> str:
153
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
154
s.connect(("8.8.8.8", 80)) # Doesn't need to be reachable
vllm/worker/model_runner.py
+146
-15
@@ -1,23 +1,27 @@
1
import time
2
-from typing import Dict, List, Optional, Tuple, Union
2
+from typing import Dict, List, Optional, Tuple, Set, Union
3
4
import numpy as np
5
import torch
6
import torch.nn as nn
7
8
-from vllm.config import ModelConfig, ParallelConfig, SchedulerConfig
8
+from vllm.config import ModelConfig, LoRAConfig, ParallelConfig, SchedulerConfig
9
from vllm.logger import init_logger
10
from vllm.model_executor import get_model, InputMetadata, SamplingMetadata
11
from vllm.model_executor.parallel_utils.communication_op import (
12
broadcast_tensor_dict)
13
from vllm.sampling_params import SamplingParams, SamplingType
14
from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
15
+from vllm.lora.worker_manager import LRUCacheWorkerLoRAManager
16
+from vllm.lora.layers import LoRAMapping
17
+from vllm.lora.request import LoRARequest
18
from vllm.utils import in_wsl
19
20
logger = init_logger(__name__)
21
22
KVCache = Tuple[torch.Tensor, torch.Tensor]
23
_PAD_SLOT_ID = -1
24
+LORA_WARMUP_RANK = 8
25
# Capture graphs for batch size 1, 2, 4, 8, 16, 24, 32, 40, ..., 256.
26
# NOTE: _get_graph_batch_size needs to be updated if this list is changed.
27
_BATCH_SIZES_TO_CAPTURE = [1, 2, 4] + [8 * i for i in range(1, 33)]
@@ -30,19 +34,23 @@ class ModelRunner:
34
model_config: ModelConfig,
35
parallel_config: ParallelConfig,
36
scheduler_config: SchedulerConfig,
37
+ lora_config: Optional[LoRAConfig],
38
is_driver_worker: bool = False,
39
):
40
self.model_config = model_config
41
self.parallel_config = parallel_config
42
self.scheduler_config = scheduler_config
43
+ self.lora_config = lora_config
44
self.is_driver_worker = is_driver_worker
45
46
# model_config can be None in tests/samplers/test_sampler.py.
47
# FIXME(woosuk): This is a hack to make the tests work. Refactor this.
48
self.sliding_window = (model_config.get_sliding_window()
49
if model_config is not None else None)
50
+ self.device = torch.device(torch.cuda.current_device())
51
self.model = None
52
self.block_size = None # Set after initial profiling.
53
+ self.lora_manager = None
54
55
self.graph_runners: Dict[int, CUDAGraphRunner] = {}
56
self.graph_memory_pool = None # Set during graph capture.
@@ -61,7 +69,17 @@ class ModelRunner:
69
self.in_wsl = in_wsl()
70
71
def load_model(self) -> None:
64
- self.model = get_model(self.model_config)
72
+ self.model = get_model(self.model_config, self.lora_config)
73
+
74
+ vocab_size = self.model.config.vocab_size
75
+
76
+ if self.lora_config:
77
+ self.lora_manager = LRUCacheWorkerLoRAManager(
78
+ self.scheduler_config.max_num_seqs,
79
+ self.scheduler_config.max_num_batched_tokens +
80
+ self.scheduler_config.max_paddings, vocab_size,
81
+ self.lora_config, self.device)
82
+ self.model = self.lora_manager.create_lora_manager(self.model)
83
84
def set_block_size(self, block_size: int) -> None:
85
self.block_size = block_size
@@ -74,12 +92,15 @@ class ModelRunner:
92
def _prepare_prompt(
93
self,
94
seq_group_metadata_list: List[SequenceGroupMetadata],
77
- ) -> Tuple[torch.Tensor, torch.Tensor, InputMetadata, List[int],
78
- List[int]]:
95
+ ) -> Tuple[torch.Tensor, torch.Tensor, InputMetadata, List[int], List[int],
96
+ List[int], List[int], Set[LoRARequest]]:
97
assert len(seq_group_metadata_list) > 0
98
input_tokens: List[List[int]] = []
99
input_positions: List[List[int]] = []
100
slot_mapping: List[List[int]] = []
101
+ lora_index_mapping: List[int] = []
102
+ lora_prompt_mapping: List[int] = []
103
+ lora_requests: Set[LoRARequest] = set()
104
105
prompt_lens: List[int] = []
106
context_lens: List[int] = []
@@ -113,6 +134,17 @@ class ModelRunner:
134
input_positions.append(
135
list(range(prefix_len, prefix_len + len(prompt_tokens))))
136
137
+ lora_id = seq_group_metadata.lora_int_id
138
+
139
+ if lora_id > 0:
140
+ lora_requests.add(seq_group_metadata.lora_request)
141
+
142
+ lora_index_mapping.append([lora_id] * prompt_len)
143
+ lora_prompt_mapping.extend(
144
+ [lora_id] *
145
+ (prompt_len
146
+ if seq_group_metadata.sampling_params.prompt_logprobs else 1))
147
+
148
if seq_group_metadata.block_tables is None:
149
# During memory profiling, the block tables are not initialized
150
# yet. In this case, we just use a dummy slot mapping.
@@ -156,6 +188,10 @@ class ModelRunner:
188
max_prompt_len,
189
pad=_PAD_SLOT_ID,
190
dtype=torch.long)
191
+ lora_index_mapping = [
192
+ _pad_to_max(mapping, max_prompt_len, pad=0)
193
+ for mapping in lora_index_mapping
194
+ ]
195
context_lens_tensor = torch.tensor(context_lens,
196
dtype=torch.int,
197
device='cuda')
@@ -188,23 +224,33 @@ class ModelRunner:
224
use_cuda_graph=False,
225
)
226
return (input_tokens, input_positions, input_metadata, prompt_lens,
191
- subquery_lens)
227
+ subquery_lens, lora_index_mapping, lora_prompt_mapping,
228
+ lora_requests)
229
230
def _prepare_decode(
231
self,
232
seq_group_metadata_list: List[SequenceGroupMetadata],
196
- ) -> Tuple[torch.Tensor, torch.Tensor, InputMetadata]:
233
+ ) -> Tuple[torch.Tensor, torch.Tensor, InputMetadata, List[int], List[int],
234
+ Set[LoRARequest]]:
235
assert len(seq_group_metadata_list) > 0
236
input_tokens: List[List[int]] = []
237
input_positions: List[List[int]] = []
238
slot_mapping: List[List[int]] = []
239
context_lens: List[int] = []
240
block_tables: List[List[int]] = []
241
+ lora_index_mapping: List[int] = []
242
+ lora_prompt_mapping: List[int] = []
243
+ lora_requests: Set[LoRARequest] = set()
244
245
for seq_group_metadata in seq_group_metadata_list:
246
assert not seq_group_metadata.is_prompt
247
248
seq_ids = list(seq_group_metadata.seq_data.keys())
249
+ lora_id = seq_group_metadata.lora_int_id
250
+
251
+ if lora_id > 0:
252
+ lora_requests.add(seq_group_metadata.lora_request)
253
+
254
for seq_id in seq_ids:
255
seq_data = seq_group_metadata.seq_data[seq_id]
256
generation_token = seq_data.get_last_token_id()
@@ -223,6 +269,8 @@ class ModelRunner:
269
block_offset = position % self.block_size
270
slot = block_number * self.block_size + block_offset
271
slot_mapping.append([slot])
272
+ lora_index_mapping.append([lora_id])
273
+ lora_prompt_mapping.append(lora_id)
274
275
if self.sliding_window is not None:
276
sliding_window_blocks = (self.sliding_window //
@@ -287,6 +335,10 @@ class ModelRunner:
335
device="cuda",
336
)
337
338
+ lora_index_mapping = [
339
+ _pad_to_max(mapping, 1, pad=0) for mapping in lora_index_mapping
340
+ ]
341
+
342
input_metadata = InputMetadata(
343
is_prompt=False,
344
slot_mapping=slot_mapping,
@@ -298,7 +350,7 @@ class ModelRunner:
350
block_tables=block_tables,
351
use_cuda_graph=use_captured_graph,
352
)
301
- return input_tokens, input_positions, input_metadata
353
+ return input_tokens, input_positions, input_metadata, lora_index_mapping, lora_prompt_mapping, lora_requests
354
355
def _prepare_sample(
356
self,
@@ -375,7 +427,8 @@ class ModelRunner:
427
def prepare_input_tensors(
428
self,
429
seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
378
- ) -> Tuple[torch.Tensor, torch.Tensor, InputMetadata, SamplingMetadata]:
430
+ ) -> Tuple[torch.Tensor, torch.Tensor, InputMetadata, SamplingMetadata,
431
+ Set[int], LoRAMapping]:
432
if self.is_driver_worker:
433
# NOTE: We assume that all sequences in the group are all prompts or
434
# all decodes.
@@ -383,16 +436,29 @@ class ModelRunner:
436
# Prepare input tensors.
437
if is_prompt:
438
(input_tokens, input_positions, input_metadata, prompt_lens,
386
- subquery_lens) = self._prepare_prompt(seq_group_metadata_list)
439
+ subquery_lens, lora_index_mapping, lora_prompt_mapping,
440
+ lora_requests) = self._prepare_prompt(seq_group_metadata_list)
441
else:
388
- (input_tokens, input_positions, input_metadata
389
- ) = self._prepare_decode(seq_group_metadata_list)
390
- subquery_lens = None
442
+ (input_tokens, input_positions, input_metadata,
443
+ lora_index_mapping, lora_prompt_mapping,
444
+ lora_requests) = self._prepare_decode(seq_group_metadata_list)
445
prompt_lens = []
446
+ subquery_lens = None
447
sampling_metadata = self._prepare_sample(seq_group_metadata_list,
448
prompt_lens,
449
subquery_lens)
450
451
+ if self.lora_config:
452
+ flat_lora_index_mapping = [
453
+ item for sublist in lora_index_mapping for item in sublist
454
+ ]
455
+ lora_mapping = LoRAMapping(
456
+ flat_lora_index_mapping,
457
+ lora_prompt_mapping,
458
+ )
459
+ else:
460
+ lora_mapping = None
461
+
462
# Broadcast the metadata.
463
metadata_dict = {
464
"input_tokens": input_tokens,
@@ -408,12 +474,16 @@ class ModelRunner:
474
"use_cuda_graph": input_metadata.use_cuda_graph,
475
"selected_token_indices":
476
sampling_metadata.selected_token_indices,
477
+ "lora_requests": lora_requests,
478
+ "lora_mapping": lora_mapping,
479
}
480
broadcast_tensor_dict(metadata_dict, src=0)
481
else:
482
metadata_dict = broadcast_tensor_dict(src=0)
483
input_tokens = metadata_dict["input_tokens"]
484
input_positions = metadata_dict["input_positions"]
485
+ lora_mapping = metadata_dict["lora_mapping"]
486
+ lora_requests = metadata_dict["lora_requests"]
487
input_metadata = InputMetadata(
488
is_prompt=metadata_dict["is_prompt"],
489
slot_mapping=metadata_dict["slot_mapping"],
@@ -434,7 +504,7 @@ class ModelRunner:
504
perform_sampling=False,
505
)
506
437
- return input_tokens, input_positions, input_metadata, sampling_metadata
507
+ return input_tokens, input_positions, input_metadata, sampling_metadata, lora_requests, lora_mapping
508
509
@torch.inference_mode()
510
def execute_model(
@@ -442,8 +512,12 @@ class ModelRunner:
512
seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
513
kv_caches: List[Tuple[torch.Tensor, torch.Tensor]],
514
) -> Optional[SamplerOutput]:
445
- input_tokens, input_positions, input_metadata, sampling_metadata = (
515
+ input_tokens, input_positions, input_metadata, sampling_metadata, lora_requests, lora_mapping = (
516
self.prepare_input_tensors(seq_group_metadata_list))
517
+
518
+ if self.lora_config:
519
+ self.set_active_loras(lora_requests, lora_mapping)
520
+
521
# Execute the model.
522
if input_metadata.use_cuda_graph:
523
graph_batch_size = input_tokens.shape[0]
@@ -472,6 +546,28 @@ class ModelRunner:
546
max_num_batched_tokens = self.scheduler_config.max_num_batched_tokens
547
max_num_seqs = self.scheduler_config.max_num_seqs
548
549
+ # This represents the maximum number of different requests
550
+ # that will have unique loras, an therefore the max amount of memory
551
+ # consumption create dummy lora request copies from the lora request
552
+ # passed in, which contains a lora from the lora warmup path.
553
+ dummy_lora_requests = []
554
+ dummy_lora_requests_per_seq = []
555
+ if self.lora_config:
556
+ for idx in range(self.lora_config.max_loras):
557
+ lora_id = idx + 1
558
+ dummy_lora_request = LoRARequest(
559
+ lora_name=f"warmup_{lora_id}",
560
+ lora_int_id=lora_id,
561
+ lora_local_path="/not/a/real/path",
562
+ )
563
+ self.lora_manager.add_dummy_lora(dummy_lora_request,
564
+ rank=LORA_WARMUP_RANK)
565
+ dummy_lora_requests.append(dummy_lora_request)
566
+ dummy_lora_requests_per_seq = [
567
+ dummy_lora_requests[idx % len(dummy_lora_requests)]
568
+ for idx in range(max_num_seqs)
569
+ ]
570
+
571
# Profile memory usage with max_num_sequences sequences and the total
572
# number of tokens equal to max_num_batched_tokens.
573
seqs: List[SequenceGroupMetadata] = []
@@ -485,6 +581,8 @@ class ModelRunner:
581
seq_data={group_id: seq_data},
582
sampling_params=sampling_params,
583
block_tables=None,
584
+ lora_request=dummy_lora_requests_per_seq[group_id]
585
+ if dummy_lora_requests_per_seq else None,
586
)
587
seqs.append(seq)
588
@@ -495,6 +593,32 @@ class ModelRunner:
593
torch.cuda.synchronize()
594
return
595
596
+ def remove_all_loras(self) -> bool:
597
+ if not self.lora_manager:
598
+ raise RuntimeError("LoRA is not enabled.")
599
+ return self.lora_manager.remove_all_loras()
600
+
601
+ def set_active_loras(self, lora_requests: List[LoRARequest],
602
+ lora_mapping: LoRAMapping) -> None:
603
+ if not self.lora_manager:
604
+ raise RuntimeError("LoRA is not enabled.")
605
+ self.lora_manager.set_active_loras(lora_requests, lora_mapping)
606
+
607
+ def add_lora(self, lora_request: LoRARequest) -> bool:
608
+ if not self.lora_manager:
609
+ raise RuntimeError("LoRA is not enabled.")
610
+ return self.lora_manager.add_lora(lora_request)
611
+
612
+ def remove_lora(self, lora_id: int) -> bool:
613
+ if not self.lora_manager:
614
+ raise RuntimeError("LoRA is not enabled.")
615
+ return self.lora_manager.remove_lora(lora_id)
616
+
617
+ def list_loras(self) -> Set[int]:
618
+ if not self.lora_manager:
619
+ raise RuntimeError("LoRA is not enabled.")
620
+ return self.lora_manager.list_loras()
621
+
622
@torch.inference_mode()
623
def capture_model(self, kv_caches: List[KVCache]) -> None:
624
assert not self.model_config.enforce_eager
@@ -541,6 +665,13 @@ class ModelRunner:
665
use_cuda_graph=True,
666
)
667
668
+ if self.lora_config:
669
+ lora_mapping = LoRAMapping(
670
+ [0] * batch_size,
671
+ [0] * batch_size,
672
+ )
673
+ self.set_active_loras(set(), lora_mapping)
674
+
675
graph_runner = CUDAGraphRunner(self.model)
676
graph_runner.capture(
677
input_tokens[:batch_size],
vllm/worker/worker.py
+23
-4
@@ -1,12 +1,13 @@
1
"""A GPU worker class."""
2
+import gc
3
import os
3
-from typing import Dict, List, Optional, Tuple
4
+from typing import Dict, List, Tuple, Set, Optional
5
6
import torch
7
import torch.distributed
8
9
from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
9
- SchedulerConfig)
10
+ SchedulerConfig, LoRAConfig)
11
from vllm.model_executor import set_random_seed
12
from vllm.model_executor.parallel_utils.communication_op import (
13
broadcast_tensor_dict)
@@ -15,6 +16,7 @@ from vllm.model_executor.parallel_utils.parallel_state import (
16
from vllm.sequence import SamplerOutput, SequenceGroupMetadata
17
from vllm.worker.cache_engine import CacheEngine
18
from vllm.worker.model_runner import ModelRunner
19
+from vllm.lora.request import LoRARequest
20
21
22
class Worker:
@@ -33,6 +35,7 @@ class Worker:
35
local_rank: int,
36
rank: int,
37
distributed_init_method: str,
38
+ lora_config: Optional[LoRAConfig] = None,
39
is_driver_worker: bool = False,
40
) -> None:
41
self.model_config = model_config
@@ -41,12 +44,16 @@ class Worker:
44
self.local_rank = local_rank
45
self.rank = rank
46
self.distributed_init_method = distributed_init_method
47
+ self.lora_config = lora_config
48
self.is_driver_worker = is_driver_worker
49
if self.is_driver_worker:
50
assert self.rank == 0, "The driver worker must have rank 0."
51
48
- self.model_runner = ModelRunner(model_config, parallel_config,
49
- scheduler_config, is_driver_worker)
52
+ self.model_runner = ModelRunner(model_config,
53
+ parallel_config,
54
+ scheduler_config,
55
+ lora_config=self.lora_config,
56
+ is_driver_worker=is_driver_worker)
57
# Uninitialized cache engine. Will be initialized by
58
# self.init_cache_engine().
59
self.cache_config = None
@@ -117,6 +124,9 @@ class Worker:
124
num_cpu_blocks = int(cpu_swap_space // cache_block_size)
125
num_gpu_blocks = max(num_gpu_blocks, 0)
126
num_cpu_blocks = max(num_cpu_blocks, 0)
127
+ if self.model_runner.lora_manager:
128
+ self.model_runner.remove_all_loras()
129
+ gc.collect()
130
torch.cuda.empty_cache()
131
return num_gpu_blocks, num_cpu_blocks
132
@@ -199,6 +209,15 @@ class Worker:
209
self.gpu_cache)
210
return output
211
212
+ def add_lora(self, lora_request: LoRARequest) -> bool:
213
+ return self.model_runner.add_lora(lora_request)
214
+
215
+ def remove_lora(self, lora_id: int) -> bool:
216
+ return self.model_runner.remove_lora(lora_id)
217
+
218
+ def list_loras(self) -> Set[int]:
219
+ return self.model_runner.list_loras()
220
+
221
222
def _init_distributed_environment(
223
parallel_config: ParallelConfig,