@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9b945daaf1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

[Experimental] Add multi-LoRA support (#1804)

Co-authored-by: Chen Shen <scv119@gmail.com> Co-authored-by: Shreyas Krishnaswamy <shrekris@anyscale.com> Co-authored-by: Avnish Narayan <avnish@anyscale.com>

Antoni Baum committed Jan 24, 2024 at 00:26 UTC 9b945daaf1ce03b8b02d68b37c59baf28566b535
52 files changed +8035 -126
.buildkite/test-pipeline.yaml
+3
@@ -41,6 +41,9 @@ steps:
41 - label: Worker Test
42 command: pytest -v -s worker
43
44 +- label: LoRA Test
45 + command: pytest -v -s lora
46 +
47 - label: Benchmarks
48 working_dir: "/vllm-workspace/.buildkite"
49 commands:
benchmarks/benchmark_latency.py
+5 -5
@@ -65,7 +65,9 @@ def main(args: argparse.Namespace):
65 if args.profile:
66 profile_dir = args.profile_result_dir
67 if not profile_dir:
68 - profile_dir = Path(".") / "vllm_benchmark_result" / f"latency_result_{time.time()}"
68 + profile_dir = Path(
69 + "."
70 + ) / "vllm_benchmark_result" / f"latency_result_{time.time()}"
71 print(f"Profiling (results will be saved to '{profile_dir}')...")
72 run_to_completion(profile_dir=args.profile_result_dir)
73 return
@@ -123,9 +125,7 @@ if __name__ == '__main__':
125 '--profile-result-dir',
126 type=str,
127 default=None,
126 - help=(
127 - 'path to save the pytorch profiler output. Can be visualized '
128 - 'with ui.perfetto.dev or Tensorboard.'
129 - ))
128 + help=('path to save the pytorch profiler output. Can be visualized '
129 + 'with ui.perfetto.dev or Tensorboard.'))
130 args = parser.parse_args()
131 main(args)
csrc/punica/LICENSE new
+217
@@ -0,0 +1,217 @@
1 +Contains code from https://github.com/punica-ai/punica
2 +
3 + Apache License
4 + Version 2.0, January 2004
5 + http://www.apache.org/licenses/
6 +
7 + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
8 +
9 + 1. Definitions.
10 +
11 + "License" shall mean the terms and conditions for use, reproduction,
12 + and distribution as defined by Sections 1 through 9 of this document.
13 +
14 + "Licensor" shall mean the copyright owner or entity authorized by
15 + the copyright owner that is granting the License.
16 +
17 + "Legal Entity" shall mean the union of the acting entity and all
18 + other entities that control, are controlled by, or are under common
19 + control with that entity. For the purposes of this definition,
20 + "control" means (i) the power, direct or indirect, to cause the
21 + direction or management of such entity, whether by contract or
22 + otherwise, or (ii) ownership of fifty percent (50%) or more of the
23 + outstanding shares, or (iii) beneficial ownership of such entity.
24 +
25 + "You" (or "Your") shall mean an individual or Legal Entity
26 + exercising permissions granted by this License.
27 +
28 + "Source" form shall mean the preferred form for making modifications,
29 + including but not limited to software source code, documentation
30 + source, and configuration files.
31 +
32 + "Object" form shall mean any form resulting from mechanical
33 + transformation or translation of a Source form, including but
34 + not limited to compiled object code, generated documentation,
35 + and conversions to other media types.
36 +
37 + "Work" shall mean the work of authorship, whether in Source or
38 + Object form, made available under the License, as indicated by a
39 + copyright notice that is included in or attached to the work
40 + (an example is provided in the Appendix below).
41 +
42 + "Derivative Works" shall mean any work, whether in Source or Object
43 + form, that is based on (or derived from) the Work and for which the
44 + editorial revisions, annotations, elaborations, or other modifications
45 + represent, as a whole, an original work of authorship. For the purposes
46 + of this License, Derivative Works shall not include works that remain
47 + separable from, or merely link (or bind by name) to the interfaces of,
48 + the Work and Derivative Works thereof.
49 +
50 + "Contribution" shall mean any work of authorship, including
51 + the original version of the Work and any modifications or additions
52 + to that Work or Derivative Works thereof, that is intentionally
53 + submitted to Licensor for inclusion in the Work by the copyright owner
54 + or by an individual or Legal Entity authorized to submit on behalf of
55 + the copyright owner. For the purposes of this definition, "submitted"
56 + means any form of electronic, verbal, or written communication sent
57 + to the Licensor or its representatives, including but not limited to
58 + communication on electronic mailing lists, source code control systems,
59 + and issue tracking systems that are managed by, or on behalf of, the
60 + Licensor for the purpose of discussing and improving the Work, but
61 + excluding communication that is conspicuously marked or otherwise
62 + designated in writing by the copyright owner as "Not a Contribution."
63 +
64 + "Contributor" shall mean Licensor and any individual or Legal Entity
65 + on behalf of whom a Contribution has been received by Licensor and
66 + subsequently incorporated within the Work.
67 +
68 + 2. Grant of Copyright License. Subject to the terms and conditions of
69 + this License, each Contributor hereby grants to You a perpetual,
70 + worldwide, non-exclusive, no-charge, royalty-free, irrevocable
71 + copyright license to reproduce, prepare Derivative Works of,
72 + publicly display, publicly perform, sublicense, and distribute the
73 + Work and such Derivative Works in Source or Object form.
74 +
75 + 3. Grant of Patent License. Subject to the terms and conditions of
76 + this License, each Contributor hereby grants to You a perpetual,
77 + worldwide, non-exclusive, no-charge, royalty-free, irrevocable
78 + (except as stated in this section) patent license to make, have made,
79 + use, offer to sell, sell, import, and otherwise transfer the Work,
80 + where such license applies only to those patent claims licensable
81 + by such Contributor that are necessarily infringed by their
82 + Contribution(s) alone or by combination of their Contribution(s)
83 + with the Work to which such Contribution(s) was submitted. If You
84 + institute patent litigation against any entity (including a
85 + cross-claim or counterclaim in a lawsuit) alleging that the Work
86 + or a Contribution incorporated within the Work constitutes direct
87 + or contributory patent infringement, then any patent licenses
88 + granted to You under this License for that Work shall terminate
89 + as of the date such litigation is filed.
90 +
91 + 4. Redistribution. You may reproduce and distribute copies of the
92 + Work or Derivative Works thereof in any medium, with or without
93 + modifications, and in Source or Object form, provided that You
94 + meet the following conditions:
95 +
96 + (a) You must give any other recipients of the Work or
97 + Derivative Works a copy of this License; and
98 +
99 + (b) You must cause any modified files to carry prominent notices
100 + stating that You changed the files; and
101 +
102 + (c) You must retain, in the Source form of any Derivative Works
103 + that You distribute, all copyright, patent, trademark, and
104 + attribution notices from the Source form of the Work,
105 + excluding those notices that do not pertain to any part of
106 + the Derivative Works; and
107 +
108 + (d) If the Work includes a "NOTICE" text file as part of its
109 + distribution, then any Derivative Works that You distribute must
110 + include a readable copy of the attribution notices contained
111 + within such NOTICE file, excluding those notices that do not
112 + pertain to any part of the Derivative Works, in at least one
113 + of the following places: within a NOTICE text file distributed
114 + as part of the Derivative Works; within the Source form or
115 + documentation, if provided along with the Derivative Works; or,
116 + within a display generated by the Derivative Works, if and
117 + wherever such third-party notices normally appear. The contents
118 + of the NOTICE file are for informational purposes only and
119 + do not modify the License. You may add Your own attribution
120 + notices within Derivative Works that You distribute, alongside
121 + or as an addendum to the NOTICE text from the Work, provided
122 + that such additional attribution notices cannot be construed
123 + as modifying the License.
124 +
125 + You may add Your own copyright statement to Your modifications and
126 + may provide additional or different license terms and conditions
127 + for use, reproduction, or distribution of Your modifications, or
128 + for any such Derivative Works as a whole, provided Your use,
129 + reproduction, and distribution of the Work otherwise complies with
130 + the conditions stated in this License.
131 +
132 + 5. Submission of Contributions. Unless You explicitly state otherwise,
133 + any Contribution intentionally submitted for inclusion in the Work
134 + by You to the Licensor shall be under the terms and conditions of
135 + this License, without any additional terms or conditions.
136 + Notwithstanding the above, nothing herein shall supersede or modify
137 + the terms of any separate license agreement you may have executed
138 + with Licensor regarding such Contributions.
139 +
140 + 6. Trademarks. This License does not grant permission to use the trade
141 + names, trademarks, service marks, or product names of the Licensor,
142 + except as required for reasonable and customary use in describing the
143 + origin of the Work and reproducing the content of the NOTICE file.
144 +
145 + 7. Disclaimer of Warranty. Unless required by applicable law or
146 + agreed to in writing, Licensor provides the Work (and each
147 + Contributor provides its Contributions) on an "AS IS" BASIS,
148 + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
149 + implied, including, without limitation, any warranties or conditions
150 + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
151 + PARTICULAR PURPOSE. You are solely responsible for determining the
152 + appropriateness of using or redistributing the Work and assume any
153 + risks associated with Your exercise of permissions under this License.
154 +
155 + 8. Limitation of Liability. In no event and under no legal theory,
156 + whether in tort (including negligence), contract, or otherwise,
157 + unless required by applicable law (such as deliberate and grossly
158 + negligent acts) or agreed to in writing, shall any Contributor be
159 + liable to You for damages, including any direct, indirect, special,
160 + incidental, or consequential damages of any character arising as a
161 + result of this License or out of the use or inability to use the
162 + Work (including but not limited to damages for loss of goodwill,
163 + work stoppage, computer failure or malfunction, or any and all
164 + other commercial damages or losses), even if such Contributor
165 + has been advised of the possibility of such damages.
166 +
167 + 9. Accepting Warranty or Additional Liability. While redistributing
168 + the Work or Derivative Works thereof, You may choose to offer,
169 + and charge a fee for, acceptance of support, warranty, indemnity,
170 + or other liability obligations and/or rights consistent with this
171 + License. However, in accepting such obligations, You may act only
172 + on Your own behalf and on Your sole responsibility, not on behalf
173 + of any other Contributor, and only if You agree to indemnify,
174 + defend, and hold each Contributor harmless for any liability
175 + incurred by, or claims asserted against, such Contributor by reason
176 + of your accepting any such warranty or additional liability.
177 +
178 + END OF TERMS AND CONDITIONS
179 +
180 + APPENDIX: How to apply the Apache License to your work.
181 +
182 + To apply the Apache License to your work, attach the following
183 + boilerplate notice, with the fields enclosed by brackets "{}"
184 + replaced with your own identifying information. (Don't include
185 + the brackets!) The text should be enclosed in the appropriate
186 + comment syntax for the file format. We also recommend that a
187 + file or class name and description of purpose be included on the
188 + same "printed page" as the copyright notice for easier
189 + identification within third-party archives.
190 +
191 + Copyright {yyyy} {name of copyright owner}
192 +
193 + Licensed under the Apache License, Version 2.0 (the "License");
194 + you may not use this file except in compliance with the License.
195 + You may obtain a copy of the License at
196 +
197 + http://www.apache.org/licenses/LICENSE-2.0
198 +
199 + Unless required by applicable law or agreed to in writing, software
200 + distributed under the License is distributed on an "AS IS" BASIS,
201 + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
202 + See the License for the specific language governing permissions and
203 + limitations under the License.
204 +
205 +------------------------------------------------------------------------------------
206 +
207 +This product bundles various third-party components under other open source licenses.
208 +This section summarizes those components and their licenses. See licenses/
209 +for text of these licenses.
210 +
211 +
212 +Apache-2.0
213 +* third_party/nvbench (with LLVM exception)
214 +* third_party/flashinfer
215 +
216 +BSD-3-Clause:
217 +* third_party/cutlass
\ No newline at end of file
csrc/punica/bgmv/bgmv_all.cu new
+21
@@ -0,0 +1,21 @@
1 +#include "bgmv_config.h"
2 +#include "bgmv_impl.cuh"
3 +
4 +FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_half, nv_half, nv_half)
5 +FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_half, nv_half, nv_bfloat16)
6 +FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_bfloat16, nv_bfloat16, nv_bfloat16)
7 +FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_bfloat16, nv_bfloat16, nv_half)
8 +FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_half, nv_bfloat16, nv_half)
9 +FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_bfloat16, nv_half, nv_half)
10 +FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_half, nv_bfloat16, nv_bfloat16)
11 +FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_bfloat16, nv_half, nv_bfloat16)
12 +FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, float, nv_half, nv_half)
13 +FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, float, nv_half, nv_bfloat16)
14 +FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, float, nv_bfloat16, nv_bfloat16)
15 +FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, float, nv_bfloat16, nv_half)
16 +FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_half, float, nv_half)
17 +FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_half, float, nv_bfloat16)
18 +FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_bfloat16, float, nv_bfloat16)
19 +FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_bfloat16, float, nv_half)
20 +FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, float, float, nv_half)
21 +FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, float, float, nv_bfloat16)
csrc/punica/bgmv/bgmv_config.h new
+59
@@ -0,0 +1,59 @@
1 +#pragma once
2 +
3 +template <int feat_in, int feat_out, typename in_T, typename out_T,
4 + typename W_T>
5 +void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
6 + const W_T *__restrict__ W,
7 + const int64_t *__restrict__ indicies, int64_t y_offset,
8 + int64_t full_y_size, int64_t batch_size, int64_t num_layers,
9 + int64_t layer_idx, float scale);
10 +
11 +// clang-format off
12 +
13 +#define FOR_BGMV_WIDE(f, in_T, out_T, W_T, narrow) \
14 + f(in_T, out_T, W_T, narrow, 128) \
15 + f(in_T, out_T, W_T, narrow, 256) \
16 + f(in_T, out_T, W_T, narrow, 512) \
17 + f(in_T, out_T, W_T, narrow, 1024) \
18 + f(in_T, out_T, W_T, narrow, 1280) \
19 + f(in_T, out_T, W_T, narrow, 1728) \
20 + f(in_T, out_T, W_T, narrow, 1792) \
21 + f(in_T, out_T, W_T, narrow, 2048) \
22 + f(in_T, out_T, W_T, narrow, 2560) \
23 + f(in_T, out_T, W_T, narrow, 2752) \
24 + f(in_T, out_T, W_T, narrow, 3072) \
25 + f(in_T, out_T, W_T, narrow, 3456) \
26 + f(in_T, out_T, W_T, narrow, 3584) \
27 + f(in_T, out_T, W_T, narrow, 4096) \
28 + f(in_T, out_T, W_T, narrow, 5120) \
29 + f(in_T, out_T, W_T, narrow, 5504) \
30 + f(in_T, out_T, W_T, narrow, 5632) \
31 + f(in_T, out_T, W_T, narrow, 6912) \
32 + f(in_T, out_T, W_T, narrow, 7168) \
33 + f(in_T, out_T, W_T, narrow, 8192) \
34 + f(in_T, out_T, W_T, narrow, 9216) \
35 + f(in_T, out_T, W_T, narrow, 10240) \
36 + f(in_T, out_T, W_T, narrow, 11008) \
37 + f(in_T, out_T, W_T, narrow, 12288) \
38 + f(in_T, out_T, W_T, narrow, 13824) \
39 + f(in_T, out_T, W_T, narrow, 14336) \
40 + f(in_T, out_T, W_T, narrow, 16384) \
41 + f(in_T, out_T, W_T, narrow, 20480) \
42 + f(in_T, out_T, W_T, narrow, 28672) \
43 + f(in_T, out_T, W_T, narrow, 32000) \
44 + f(in_T, out_T, W_T, narrow, 32256) \
45 + f(in_T, out_T, W_T, narrow, 32512) \
46 + f(in_T, out_T, W_T, narrow, 32768) \
47 + f(in_T, out_T, W_T, narrow, 33024) \
48 + f(in_T, out_T, W_T, narrow, 36864) \
49 + f(in_T, out_T, W_T, narrow, 49152) \
50 +// Keep above in sync with vllm/lora/layers::SamplerWithLoRA
51 +
52 +// Keep this in sync with vllm/config::LoRAConfig
53 +#define FOR_BGMV_WIDE_NARROW(f, in_T, out_T, W_T) \
54 + FOR_BGMV_WIDE(f, in_T, out_T, W_T, 8) \
55 + FOR_BGMV_WIDE(f, in_T, out_T, W_T, 16) \
56 + FOR_BGMV_WIDE(f, in_T, out_T, W_T, 32) \
57 + FOR_BGMV_WIDE(f, in_T, out_T, W_T, 64)
58 +
59 +// clang-format on
csrc/punica/bgmv/bgmv_impl.cuh new
+294
@@ -0,0 +1,294 @@
1 +#pragma once
2 +
3 +#include <ATen/cuda/CUDAContext.h>
4 +#include <cooperative_groups.h>
5 +#include <cuda/pipeline>
6 +#include <cuda_runtime.h>
7 +#include <iostream>
8 +#include <stdio.h>
9 +
10 +#include "vec_dtypes.cuh"
11 +
12 +namespace cg = cooperative_groups;
13 +
14 +// nthrs = (32, 4)
15 +template <int feat_in, int feat_out, size_t vec_size, size_t X_copy_size,
16 + size_t W_copy_size, int tx, int ty, int tz, typename in_T,
17 + typename out_T, typename W_T>
18 +__global__ void
19 +bgmv_shrink_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
20 + const W_T *__restrict__ W,
21 + const int64_t *__restrict__ indicies, int64_t y_offset,
22 + int64_t full_y_size, int64_t num_layers, int64_t layer_idx,
23 + float scale) {
24 + size_t batch_idx = blockIdx.y;
25 + int64_t idx = indicies[batch_idx] * num_layers + layer_idx;
26 + if (idx < 0) {
27 + return;
28 + }
29 +
30 + auto block = cg::this_thread_block();
31 + size_t j = blockIdx.x;
32 + constexpr size_t num_pipeline_stages = 2;
33 + constexpr size_t tile_size = tx * ty * vec_size;
34 + __shared__ W_T W_shared[num_pipeline_stages * tile_size];
35 + __shared__ in_T X_shared[num_pipeline_stages * tile_size];
36 + __shared__ float y_warpwise[ty];
37 +
38 + size_t W_shared_offset[num_pipeline_stages] = {0U, 1U * tile_size};
39 + size_t X_shared_offset[num_pipeline_stages] = {0U, 1U * tile_size};
40 + auto pipe = cuda::make_pipeline();
41 +
42 + // pipeline load W/X and compute WX;
43 + pipe.producer_acquire();
44 + cuda::memcpy_async(W_shared + (threadIdx.y * tx + threadIdx.x) * vec_size,
45 + W + (idx * feat_out + j) * feat_in +
46 + (threadIdx.y * tx + threadIdx.x) * vec_size,
47 + cuda::aligned_size_t<W_copy_size>(W_copy_size), pipe);
48 + cuda::memcpy_async(X_shared + (threadIdx.y * tx + threadIdx.x) * vec_size,
49 + X + (batch_idx * feat_in) +
50 + (threadIdx.y * tx + threadIdx.x) * vec_size,
51 + cuda::aligned_size_t<X_copy_size>(X_copy_size), pipe);
52 + pipe.producer_commit();
53 + size_t copy_idx, compute_idx;
54 + float y = 0.f;
55 + vec_t<in_T, vec_size> x_vec;
56 + vec_t<W_T, vec_size> w_vec;
57 + size_t tile_idx;
58 +
59 +#pragma unroll
60 + for (tile_idx = 1; tile_idx < (feat_in + tile_size - 1) / tile_size;
61 + ++tile_idx) {
62 + copy_idx = tile_idx % num_pipeline_stages;
63 + // pipeline stage: async copy W fragment
64 + pipe.producer_acquire();
65 + if (tile_idx * tile_size + threadIdx.y * tx * vec_size < feat_in) {
66 + cuda::memcpy_async(W_shared + W_shared_offset[copy_idx] +
67 + (threadIdx.y * tx + threadIdx.x) * vec_size,
68 + W + (idx * feat_out + j) * feat_in +
69 + tile_idx * tile_size +
70 + (threadIdx.y * tx + threadIdx.x) * vec_size,
71 + cuda::aligned_size_t<W_copy_size>(W_copy_size), pipe);
72 + cuda::memcpy_async(X_shared + X_shared_offset[copy_idx] +
73 + (threadIdx.y * tx + threadIdx.x) * vec_size,
74 + X + (batch_idx * feat_in) + tile_idx * tile_size +
75 + (threadIdx.y * tx + threadIdx.x) * vec_size,
76 + cuda::aligned_size_t<X_copy_size>(X_copy_size), pipe);
77 + }
78 + pipe.producer_commit();
79 +
80 + compute_idx = (tile_idx - 1) % num_pipeline_stages;
81 + // pipeline stage: compute WX
82 + pipe.consumer_wait();
83 + block.sync();
84 + x_vec.load(X_shared + X_shared_offset[compute_idx] +
85 + (threadIdx.y * tx + threadIdx.x) * vec_size);
86 + w_vec.load(W_shared + W_shared_offset[compute_idx] +
87 + (threadIdx.y * tx + threadIdx.x) * vec_size);
88 + float sum = 0.f;
89 +#pragma unroll
90 + for (size_t i = 0; i < vec_size; ++i) {
91 + sum += float(w_vec[i]) * float(x_vec[i]) * scale;
92 + }
93 +#pragma unroll
94 + for (size_t offset = tx / 2; offset > 0; offset /= 2) {
95 + sum += __shfl_down_sync(0xffffffff, sum, offset);
96 + }
97 + y_warpwise[threadIdx.y] = sum;
98 + block.sync();
99 +#pragma unroll
100 + for (size_t i = 0; i < ty; ++i) {
101 + y += y_warpwise[i];
102 + }
103 +
104 + block.sync();
105 + pipe.consumer_release();
106 + }
107 +
108 + compute_idx = (tile_idx - 1) % num_pipeline_stages;
109 + // final pipeline stage
110 + pipe.consumer_wait();
111 + block.sync();
112 + x_vec.load(X_shared + X_shared_offset[compute_idx] +
113 + (threadIdx.y * tx + threadIdx.x) * vec_size);
114 + w_vec.load(W_shared + W_shared_offset[compute_idx] +
115 + (threadIdx.y * tx + threadIdx.x) * vec_size);
116 + float sum = 0.f;
117 +#pragma unroll
118 + for (size_t i = 0; i < vec_size; ++i) {
119 + sum += float(w_vec[i]) * float(x_vec[i]) * scale;
120 + }
121 +#pragma unroll
122 + for (size_t offset = tx / 2; offset > 0; offset /= 2) {
123 + sum += __shfl_down_sync(0xffffffff, sum, offset);
124 + }
125 + y_warpwise[threadIdx.y] =
126 + ((tile_idx - 1) * tile_size + threadIdx.y * tx * vec_size < feat_in)
127 + ? sum
128 + : 0.f;
129 + block.sync();
130 +#pragma unroll
131 + for (size_t i = 0; i < ty; ++i) {
132 + y += y_warpwise[i];
133 + }
134 +
135 + block.sync();
136 + pipe.consumer_release();
137 +
138 + // write Y;
139 + if (block.thread_rank() == 0) {
140 + Y[batch_idx * full_y_size + y_offset + j] += static_cast<out_T>(y);
141 + }
142 +}
143 +
144 +// nthrs = (2, 16, 4)
145 +template <int feat_in, int feat_out, size_t vec_size, int tx, int ty, int tz,
146 + typename in_T, typename out_T, typename W_T>
147 +__global__ void
148 +bgmv_expand_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
149 + const W_T *__restrict__ W,
150 + const int64_t *__restrict__ indicies, int64_t y_offset,
151 + int64_t full_y_size, int64_t num_layers, int64_t layer_idx,
152 + float scale) {
153 + size_t batch_idx = blockIdx.y;
154 + int64_t idx = indicies[batch_idx] * num_layers + layer_idx;
155 +
156 + if (idx < 0) {
157 + return;
158 + }
159 +
160 + auto block = cg::this_thread_block();
161 + size_t tile_idx = blockIdx.x;
162 +
163 + // load X;
164 + vec_t<in_T, vec_size> x_vec;
165 + x_vec.load(X + batch_idx * feat_in + threadIdx.x * vec_size);
166 +
167 + // load W;
168 + vec_t<W_T, vec_size> w_vec;
169 + w_vec.load(W + (idx * feat_out + tile_idx * tz * ty) * feat_in +
170 + block.thread_rank() * vec_size);
171 +
172 + float sum = 0.f;
173 +#pragma unroll
174 + for (size_t i = 0; i < vec_size; ++i) {
175 + sum += float(w_vec[i]) * float(x_vec[i]) * scale;
176 + }
177 +
178 + cg::thread_block_tile g = cg::tiled_partition<tx>(block);
179 +#pragma unroll
180 + for (size_t offset = tx / 2; offset > 0; offset /= 2) {
181 + sum += g.shfl_down(sum, offset);
182 + }
183 + sum = g.shfl(sum, 0);
184 +
185 + if (threadIdx.x == 0) {
186 + Y[batch_idx * full_y_size + y_offset + tile_idx * (tz * ty) +
187 + threadIdx.z * ty + threadIdx.y] += static_cast<out_T>(sum);
188 + }
189 +}
190 +
191 +template <int feat_in, int feat_out, typename in_T, typename out_T,
192 + typename W_T>
193 +void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
194 + const W_T *__restrict__ W,
195 + const int64_t *__restrict__ indicies, int64_t y_offset,
196 + int64_t full_y_size, int64_t batch_size, int64_t num_layers,
197 + int64_t layer_idx, float scale) {
198 + constexpr size_t vec_size = 8;
199 + constexpr int tz = 4;
200 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
201 +
202 + if constexpr (feat_in < feat_out) {
203 + static_assert(feat_in % vec_size == 0);
204 + constexpr int tx = feat_in / vec_size;
205 +
206 + static_assert((32 % tx == 0 && feat_out % (32 / tx * tz) == 0) ||
207 + (16 % tx == 0 && feat_out % (16 / tx * tz) == 0) ||
208 + (8 % tx == 0 && feat_out % (8 / tx * tz) == 0));
209 +
210 + if constexpr (32 % tx == 0 && feat_out % (32 / tx * tz) == 0) {
211 + constexpr int ty = 32 / tx;
212 + dim3 nblks(feat_out / (ty * tz), batch_size);
213 + dim3 nthrs(tx, ty, tz);
214 +
215 + bgmv_expand_kernel<feat_in, feat_out, vec_size, tx, ty, tz>
216 + <<<nblks, nthrs, 0, stream>>>(Y, X, W, indicies, y_offset,
217 + full_y_size, num_layers, layer_idx,
218 + scale);
219 + } else if (16 % tx == 0 && feat_out % (16 / tx * tz) == 0) {
220 + constexpr int ty = 16 / tx;
221 + dim3 nblks(feat_out / (ty * tz), batch_size);
222 + dim3 nthrs(tx, ty, tz);
223 +
224 + bgmv_expand_kernel<feat_in, feat_out, vec_size, tx, ty, tz>
225 + <<<nblks, nthrs, 0, stream>>>(Y, X, W, indicies, y_offset,
226 + full_y_size, num_layers, layer_idx,
227 + scale);
228 + } else {
229 + constexpr int ty = 8 / tx;
230 + dim3 nblks(feat_out / (ty * tz), batch_size);
231 + dim3 nthrs(tx, ty, tz);
232 +
233 + bgmv_expand_kernel<feat_in, feat_out, vec_size, tx, ty, tz>
234 + <<<nblks, nthrs, 0, stream>>>(Y, X, W, indicies, y_offset,
235 + full_y_size, num_layers, layer_idx,
236 + scale);
237 + }
238 + } else {
239 + static_assert(feat_in % (vec_size * 32) == 0 ||
240 + feat_in % (vec_size * 16) == 0 ||
241 + feat_in % (vec_size * 8) == 0);
242 +
243 + if constexpr (feat_in % (vec_size * 32) == 0) {
244 + constexpr int tx = 32;
245 + constexpr int ty = 4;
246 +
247 + dim3 nblks(feat_out, batch_size);
248 + dim3 nthrs(tx, ty);
249 +
250 + bgmv_shrink_kernel<feat_in, feat_out, vec_size, vec_size * sizeof(in_T),
251 + vec_size * sizeof(W_T), tx, ty, tz>
252 + <<<nblks, nthrs, 0, stream>>>(Y, X, W, indicies, y_offset,
253 + full_y_size, num_layers, layer_idx,
254 + scale);
255 + } else if constexpr (feat_in % (vec_size / 2 * 32) == 0) {
256 + constexpr int tx = 32;
257 + constexpr int ty = 4;
258 +
259 + dim3 nblks(feat_out, batch_size);
260 + dim3 nthrs(tx, ty);
261 +
262 + bgmv_shrink_kernel<feat_in, feat_out, vec_size / 2,
263 + vec_size * sizeof(in_T) / 2,
264 + vec_size * sizeof(W_T) / 2, tx, ty, tz>
265 + <<<nblks, nthrs, 0, stream>>>(Y, X, W, indicies, y_offset,
266 + full_y_size, num_layers, layer_idx,
267 + scale);
268 + } else if constexpr (feat_in % (vec_size / 2 * 16) == 0) {
269 + constexpr int tx = 16;
270 + constexpr int ty = 4;
271 +
272 + dim3 nblks(feat_out, batch_size);
273 + dim3 nthrs(tx, ty);
274 +
275 + bgmv_shrink_kernel<feat_in, feat_out, vec_size / 2,
276 + vec_size * sizeof(in_T) / 2,
277 + vec_size * sizeof(W_T) / 2, tx, ty, tz>
278 + <<<nblks, nthrs, 0, stream>>>(Y, X, W, indicies, y_offset,
279 + full_y_size, num_layers, layer_idx,
280 + scale);
281 + }
282 + }
283 +}
284 +
285 +#define INST_BGMV(feat_in, feat_out, in_T, out_T, W_T) \
286 + template void bgmv_kernel<feat_in, feat_out>( \
287 + out_T * __restrict__ Y, const in_T *__restrict__ X, \
288 + const W_T *__restrict__ W, const int64_t *__restrict__ indicies, \
289 + int64_t y_offset, int64_t full_y_size, int64_t batch_size, \
290 + int64_t num_layers, int64_t layer_idx, float scale);
291 +
292 +#define INST_BGMV_TWOSIDE(in_T, out_T, W_T, narrow, wide) \
293 + INST_BGMV(narrow, wide, in_T, out_T, W_T) \
294 + INST_BGMV(wide, narrow, in_T, out_T, W_T)
csrc/punica/bgmv/vec_dtypes.cuh new
+1324
@@ -0,0 +1,1324 @@
1 +#ifndef VEC_DTYPES_CUH_
2 +#define VEC_DTYPES_CUH_
3 +
4 +#include <cuda_bf16.h>
5 +#include <cuda_fp16.h>
6 +#ifdef FLASHINFER_USE_FP8
7 +#include <cuda_fp8.h>
8 +#endif
9 +#include <cuda_runtime.h>
10 +
11 +#include <type_traits>
12 +
13 +#define FLASHINFER_INLINE \
14 + inline __attribute__((always_inline)) __device__ __host__
15 +
16 +template <typename float_t, size_t vec_size>
17 +struct vec_t {
18 + FLASHINFER_INLINE float_t &operator[](size_t i);
19 + FLASHINFER_INLINE const float_t &operator[](size_t i) const;
20 + FLASHINFER_INLINE void fill(float_t val);
21 + FLASHINFER_INLINE void load(const float_t *ptr);
22 + FLASHINFER_INLINE void store(float_t *ptr) const;
23 + template <typename T>
24 + FLASHINFER_INLINE void cast_from(const vec_t<T, vec_size> &src);
25 + template <typename T>
26 + FLASHINFER_INLINE void cast_load(const T *ptr);
27 + template <typename T>
28 + FLASHINFER_INLINE void cast_store(T *ptr) const;
29 + FLASHINFER_INLINE static void memcpy(float_t *dst, const float_t *src);
30 +};
31 +
32 +template <typename src_float_t, typename tgt_float_t, size_t vec_size>
33 +FLASHINFER_INLINE void cast_from_impl(const vec_t<src_float_t, vec_size> &src,
34 + vec_t<tgt_float_t, vec_size> &dst) {
35 +#pragma unroll
36 + for (size_t i = 0; i < vec_size; ++i) {
37 + dst[i] = tgt_float_t(src[i]);
38 + }
39 +}
40 +
41 +template <typename src_float_t, typename tgt_float_t, size_t vec_size>
42 +FLASHINFER_INLINE void cast_load_impl(const src_float_t *src_ptr,
43 + vec_t<tgt_float_t, vec_size> &dst) {
44 + if constexpr (std::is_same<src_float_t, tgt_float_t>::value) {
45 + dst.load(src_ptr);
46 + } else {
47 + vec_t<src_float_t, vec_size> tmp;
48 + tmp.load(src_ptr);
49 + dst.cast_from(tmp);
50 + }
51 +}
52 +
53 +template <typename src_float_t, typename tgt_float_t, size_t vec_size>
54 +FLASHINFER_INLINE void cast_store_impl(const vec_t<src_float_t, vec_size> &src,
55 + tgt_float_t *dst_ptr) {
56 + if constexpr (std::is_same<src_float_t, tgt_float_t>::value) {
57 + src.store(dst_ptr);
58 + } else {
59 + vec_t<tgt_float_t, vec_size> tmp;
60 + tmp.cast_from(src);
61 + tmp.store(dst_ptr);
62 + }
63 +}
64 +
65 +#ifdef FLASHINFER_USE_FP8
66 +/******************* vec_t<__nv_fp8_e4m3> *******************/
67 +
68 +// __nv_fp8_e4m3 x 1
69 +template <>
70 +struct vec_t<__nv_fp8_e4m3, 1> {
71 + __nv_fp8_e4m3 data;
72 +
73 + FLASHINFER_INLINE __nv_fp8_e4m3 &operator[](size_t i) {
74 + return ((__nv_fp8_e4m3 *)(&data))[i];
75 + }
76 + FLASHINFER_INLINE const __nv_fp8_e4m3 &operator[](size_t i) const {
77 + return ((const __nv_fp8_e4m3 *)(&data))[i];
78 + }
79 + FLASHINFER_INLINE void fill(__nv_fp8_e4m3 val);
80 + FLASHINFER_INLINE void load(const __nv_fp8_e4m3 *ptr);
81 + FLASHINFER_INLINE void store(__nv_fp8_e4m3 *ptr) const;
82 + template <typename T>
83 + FLASHINFER_INLINE void cast_from(const vec_t<T, 1> &src) {
84 + cast_from_impl(src, *this);
85 + }
86 + template <typename T>
87 + FLASHINFER_INLINE void cast_load(const T *ptr) {
88 + cast_load_impl(ptr, *this);
89 + }
90 + template <typename T>
91 + FLASHINFER_INLINE void cast_store(T *ptr) const {
92 + cast_store_impl(*this, ptr);
93 + }
94 +
95 + FLASHINFER_INLINE static void memcpy(__nv_fp8_e4m3 *dst,
96 + const __nv_fp8_e4m3 *src);
97 +};
98 +
99 +FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 1>::fill(__nv_fp8_e4m3 val) {
100 + data = val;
101 +}
102 +
103 +FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 1>::load(const __nv_fp8_e4m3 *ptr) {
104 + data = *ptr;
105 +}
106 +
107 +FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 1>::store(
108 + __nv_fp8_e4m3 *ptr) const {
109 + *ptr = data;
110 +}
111 +
112 +FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 1>::memcpy(
113 + __nv_fp8_e4m3 *dst, const __nv_fp8_e4m3 *src) {
114 + *dst = *src;
115 +}
116 +
117 +// __nv_fp8_e4m3 x 2
118 +template <>
119 +struct vec_t<__nv_fp8_e4m3, 2> {
120 + __nv_fp8x2_e4m3 data;
121 +
122 + FLASHINFER_INLINE __nv_fp8_e4m3 &operator[](size_t i) {
123 + return ((__nv_fp8_e4m3 *)(&data))[i];
124 + }
125 + FLASHINFER_INLINE const __nv_fp8_e4m3 &operator[](size_t i) const {
126 + return ((const __nv_fp8_e4m3 *)(&data))[i];
127 + }
128 + FLASHINFER_INLINE void fill(__nv_fp8_e4m3 val);
129 + FLASHINFER_INLINE void load(const __nv_fp8_e4m3 *ptr);
130 + FLASHINFER_INLINE void store(__nv_fp8_e4m3 *ptr) const;
131 + template <typename T>
132 + FLASHINFER_INLINE void cast_from(const vec_t<T, 2> &src) {
133 + cast_from_impl(src, *this);
134 + }
135 + template <typename T>
136 + FLASHINFER_INLINE void cast_load(const T *ptr) {
137 + cast_load_impl(ptr, *this);
138 + }
139 + template <typename T>
140 + FLASHINFER_INLINE void cast_store(T *ptr) const {
141 + cast_store_impl(*this, ptr);
142 + }
143 +
144 + FLASHINFER_INLINE static void memcpy(__nv_fp8_e4m3 *dst,
145 + const __nv_fp8_e4m3 *src);
146 +};
147 +
148 +FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 2>::fill(__nv_fp8_e4m3 val) {
149 + data.__x =
150 + (__nv_fp8x2_storage_t(val.__x) << 8) | __nv_fp8x2_storage_t(val.__x);
151 +}
152 +
153 +FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 2>::load(const __nv_fp8_e4m3 *ptr) {
154 + data = *((__nv_fp8x2_e4m3 *)ptr);
155 +}
156 +
157 +FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 2>::store(
158 + __nv_fp8_e4m3 *ptr) const {
159 + *((__nv_fp8x2_e4m3 *)ptr) = data;
160 +}
161 +
162 +FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 2>::memcpy(
163 + __nv_fp8_e4m3 *dst, const __nv_fp8_e4m3 *src) {
164 + *((__nv_fp8x2_e4m3 *)dst) = *((__nv_fp8x2_e4m3 *)src);
165 +}
166 +
167 +// __nv_fp8_e4m3 x 4
168 +
169 +template <>
170 +struct vec_t<__nv_fp8_e4m3, 4> {
171 + __nv_fp8x4_e4m3 data;
172 +
173 + FLASHINFER_INLINE __nv_fp8_e4m3 &operator[](size_t i) {
174 + return ((__nv_fp8_e4m3 *)(&data))[i];
175 + }
176 + FLASHINFER_INLINE const __nv_fp8_e4m3 &operator[](size_t i) const {
177 + return ((const __nv_fp8_e4m3 *)(&data))[i];
178 + }
179 + FLASHINFER_INLINE void fill(__nv_fp8_e4m3 val);
180 + FLASHINFER_INLINE void load(const __nv_fp8_e4m3 *ptr);
181 + FLASHINFER_INLINE void store(__nv_fp8_e4m3 *ptr) const;
182 + template <typename T>
183 + FLASHINFER_INLINE void cast_from(const vec_t<T, 4> &src) {
184 + cast_from_impl(src, *this);
185 + }
186 + template <typename T>
187 + FLASHINFER_INLINE void cast_load(const T *ptr) {
188 + cast_load_impl(ptr, *this);
189 + }
190 + template <typename T>
191 + FLASHINFER_INLINE void cast_store(T *ptr) const {
192 + cast_store_impl(*this, ptr);
193 + }
194 +
195 + FLASHINFER_INLINE static void memcpy(__nv_fp8_e4m3 *dst,
196 + const __nv_fp8_e4m3 *src);
197 +};
198 +
199 +FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 4>::fill(__nv_fp8_e4m3 val) {
200 + data.__x = (__nv_fp8x4_storage_t(val.__x) << 24) |
201 + (__nv_fp8x4_storage_t(val.__x) << 16) |
202 + (__nv_fp8x4_storage_t(val.__x) << 8) |
203 + __nv_fp8x4_storage_t(val.__x);
204 +}
205 +
206 +FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 4>::load(const __nv_fp8_e4m3 *ptr) {
207 + data = *((__nv_fp8x4_e4m3 *)ptr);
208 +}
209 +
210 +FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 4>::store(
211 + __nv_fp8_e4m3 *ptr) const {
212 + *((__nv_fp8x4_e4m3 *)ptr) = data;
213 +}
214 +
215 +FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 4>::memcpy(
216 + __nv_fp8_e4m3 *dst, const __nv_fp8_e4m3 *src) {
217 + *((__nv_fp8x4_e4m3 *)dst) = *((__nv_fp8x4_e4m3 *)src);
218 +}
219 +
220 +// __nv_fp8_e4m3 x 8
221 +
222 +template <>
223 +struct vec_t<__nv_fp8_e4m3, 8> {
224 + uint2 data;
225 +
226 + FLASHINFER_INLINE __nv_fp8_e4m3 &operator[](size_t i) {
227 + return ((__nv_fp8_e4m3 *)(&data))[i];
228 + }
229 + FLASHINFER_INLINE const __nv_fp8_e4m3 &operator[](size_t i) const {
230 + return ((const __nv_fp8_e4m3 *)(&data))[i];
231 + }
232 + FLASHINFER_INLINE void fill(__nv_fp8_e4m3 val);
233 + FLASHINFER_INLINE void load(const __nv_fp8_e4m3 *ptr);
234 + FLASHINFER_INLINE void store(__nv_fp8_e4m3 *ptr) const;
235 + template <typename T>
236 + FLASHINFER_INLINE void cast_from(const vec_t<T, 8> &src) {
237 + cast_from_impl(src, *this);
238 + }
239 + template <typename T>
240 + FLASHINFER_INLINE void cast_load(const T *ptr) {
241 + cast_load_impl(ptr, *this);
242 + }
243 + template <typename T>
244 + FLASHINFER_INLINE void cast_store(T *ptr) const {
245 + cast_store_impl(*this, ptr);
246 + }
247 +
248 + FLASHINFER_INLINE static void memcpy(__nv_fp8_e4m3 *dst,
249 + const __nv_fp8_e4m3 *src);
250 +};
251 +
252 +FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 8>::fill(__nv_fp8_e4m3 val) {
253 + ((__nv_fp8x4_e4m3 *)(&data.x))->__x = (__nv_fp8x4_storage_t(val.__x) << 24) |
254 + (__nv_fp8x4_storage_t(val.__x) << 16) |
255 + (__nv_fp8x4_storage_t(val.__x) << 8) |
256 + __nv_fp8x4_storage_t(val.__x);
257 + ((__nv_fp8x4_e4m3 *)(&data.y))->__x = (__nv_fp8x4_storage_t(val.__x) << 24) |
258 + (__nv_fp8x4_storage_t(val.__x) << 16) |
259 + (__nv_fp8x4_storage_t(val.__x) << 8) |
260 + __nv_fp8x4_storage_t(val.__x);
261 +}
262 +
263 +FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 8>::load(const __nv_fp8_e4m3 *ptr) {
264 + data = *((uint2 *)ptr);
265 +}
266 +
267 +FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 8>::store(
268 + __nv_fp8_e4m3 *ptr) const {
269 + *((uint2 *)ptr) = data;
270 +}
271 +
272 +FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 8>::memcpy(
273 + __nv_fp8_e4m3 *dst, const __nv_fp8_e4m3 *src) {
274 + *((__nv_fp8_e4m3 *)dst) = *((__nv_fp8_e4m3 *)src);
275 +}
276 +
277 +// __nv_fp8_e4m3 x 16 or more
278 +template <size_t vec_size>
279 +struct vec_t<__nv_fp8_e4m3, vec_size> {
280 + uint4 data[vec_size / 16];
281 +
282 + FLASHINFER_INLINE __nv_fp8_e4m3 &operator[](size_t i) {
283 + return ((__nv_fp8_e4m3 *)data)[i];
284 + }
285 + FLASHINFER_INLINE const __nv_fp8_e4m3 &operator[](size_t i) const {
286 + return ((const __nv_fp8_e4m3 *)data)[i];
287 + }
288 + FLASHINFER_INLINE void fill(__nv_fp8_e4m3 val) {
289 +#pragma unroll
290 + for (size_t i = 0; i < vec_size / 16; ++i) {
291 + ((__nv_fp8x4_e4m3 *)(&(data[i].x)))->__x =
292 + (__nv_fp8x4_storage_t(val.__x) << 24) |
293 + (__nv_fp8x4_storage_t(val.__x) << 16) |
294 + (__nv_fp8x4_storage_t(val.__x) << 8) | __nv_fp8x4_storage_t(val.__x);
295 + ((__nv_fp8x4_e4m3 *)(&(data[i].y)))->__x =
296 + (__nv_fp8x4_storage_t(val.__x) << 24) |
297 + (__nv_fp8x4_storage_t(val.__x) << 16) |
298 + (__nv_fp8x4_storage_t(val.__x) << 8) | __nv_fp8x4_storage_t(val.__x);
299 + ((__nv_fp8x4_e4m3 *)(&(data[i].z)))->__x =
300 + (__nv_fp8x4_storage_t(val.__x) << 24) |
301 + (__nv_fp8x4_storage_t(val.__x) << 16) |
302 + (__nv_fp8x4_storage_t(val.__x) << 8) | __nv_fp8x4_storage_t(val.__x);
303 + ((__nv_fp8x4_e4m3 *)(&(data[i].w)))->__x =
304 + (__nv_fp8x4_storage_t(val.__x) << 24) |
305 + (__nv_fp8x4_storage_t(val.__x) << 16) |
306 + (__nv_fp8x4_storage_t(val.__x) << 8) | __nv_fp8x4_storage_t(val.__x);
307 + }
308 + }
309 + FLASHINFER_INLINE void load(const __nv_fp8_e4m3 *ptr) {
310 +#pragma unroll
311 + for (size_t i = 0; i < vec_size / 16; ++i) {
312 + data[i] = ((uint4 *)ptr)[i];
313 + }
314 + }
315 + FLASHINFER_INLINE void store(__nv_fp8_e4m3 *ptr) const {
316 +#pragma unroll
317 + for (size_t i = 0; i < vec_size / 16; ++i) {
318 + ((uint4 *)ptr)[i] = data[i];
319 + }
320 + }
321 + template <typename T>
322 + FLASHINFER_INLINE void cast_from(const vec_t<T, vec_size> &src) {
323 + cast_from_impl(src, *this);
324 + }
325 + template <typename T>
326 + FLASHINFER_INLINE void cast_load(const T *ptr) {
327 + cast_load_impl(ptr, *this);
328 + }
329 + template <typename T>
330 + FLASHINFER_INLINE void cast_store(T *ptr) const {
331 + cast_store_impl(*this, ptr);
332 + }
333 +
334 + FLASHINFER_INLINE static void memcpy(__nv_fp8_e4m3 *dst,
335 + const __nv_fp8_e4m3 *src) {
336 +#pragma unroll
337 + for (size_t i = 0; i < vec_size / 16; ++i) {
338 + ((uint4 *)dst)[i] = ((uint4 *)src)[i];
339 + }
340 + }
341 +};
342 +
343 +/******************* vec_t<__nv_fp8_e5m2> *******************/
344 +
345 +// __nv_fp8_e5m2 x 1
346 +template <>
347 +struct vec_t<__nv_fp8_e5m2, 1> {
348 + __nv_fp8_e5m2 data;
349 +
350 + FLASHINFER_INLINE __nv_fp8_e5m2 &operator[](size_t i) {
351 + return ((__nv_fp8_e5m2 *)(&data))[i];
352 + }
353 + FLASHINFER_INLINE const __nv_fp8_e5m2 &operator[](size_t i) const {
354 + return ((const __nv_fp8_e5m2 *)(&data))[i];
355 + }
356 + FLASHINFER_INLINE void fill(__nv_fp8_e5m2 val);
357 + FLASHINFER_INLINE void load(const __nv_fp8_e5m2 *ptr);
358 + FLASHINFER_INLINE void store(__nv_fp8_e5m2 *ptr) const;
359 + template <typename T>
360 + FLASHINFER_INLINE void cast_from(const vec_t<T, 1> &src) {
361 + cast_from_impl(src, *this);
362 + }
363 + template <typename T>
364 + FLASHINFER_INLINE void cast_load(const T *ptr) {
365 + cast_load_impl(ptr, *this);
366 + }
367 + template <typename T>
368 + FLASHINFER_INLINE void cast_store(T *ptr) const {
369 + cast_store_impl(*this, ptr);
370 + }
371 +
372 + FLASHINFER_INLINE static void memcpy(__nv_fp8_e5m2 *dst,
373 + const __nv_fp8_e5m2 *src);
374 +};
375 +
376 +FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 1>::fill(__nv_fp8_e5m2 val) {
377 + data = val;
378 +}
379 +
380 +FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 1>::load(const __nv_fp8_e5m2 *ptr) {
381 + data = *ptr;
382 +}
383 +
384 +FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 1>::store(
385 + __nv_fp8_e5m2 *ptr) const {
386 + *ptr = data;
387 +}
388 +
389 +FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 1>::memcpy(
390 + __nv_fp8_e5m2 *dst, const __nv_fp8_e5m2 *src) {
391 + *dst = *src;
392 +}
393 +
394 +// __nv_fp8_e5m2 x 2
395 +template <>
396 +struct vec_t<__nv_fp8_e5m2, 2> {
397 + __nv_fp8x2_e5m2 data;
398 +
399 + FLASHINFER_INLINE __nv_fp8_e5m2 &operator[](size_t i) {
400 + return ((__nv_fp8_e5m2 *)(&data))[i];
401 + }
402 + FLASHINFER_INLINE const __nv_fp8_e5m2 &operator[](size_t i) const {
403 + return ((const __nv_fp8_e5m2 *)(&data))[i];
404 + }
405 + FLASHINFER_INLINE void fill(__nv_fp8_e5m2 val);
406 + FLASHINFER_INLINE void load(const __nv_fp8_e5m2 *ptr);
407 + FLASHINFER_INLINE void store(__nv_fp8_e5m2 *ptr) const;
408 + template <typename T>
409 + FLASHINFER_INLINE void cast_from(const vec_t<T, 2> &src) {
410 + cast_from_impl(src, *this);
411 + }
412 + template <typename T>
413 + FLASHINFER_INLINE void cast_load(const T *ptr) {
414 + cast_load_impl(ptr, *this);
415 + }
416 + template <typename T>
417 + FLASHINFER_INLINE void cast_store(T *ptr) const {
418 + cast_store_impl(*this, ptr);
419 + }
420 +
421 + FLASHINFER_INLINE static void memcpy(__nv_fp8_e5m2 *dst,
422 + const __nv_fp8_e5m2 *src);
423 +};
424 +
425 +FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 2>::fill(__nv_fp8_e5m2 val) {
426 + data.__x =
427 + (__nv_fp8x2_storage_t(val.__x) << 8) | __nv_fp8x2_storage_t(val.__x);
428 +}
429 +
430 +FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 2>::load(const __nv_fp8_e5m2 *ptr) {
431 + data = *((__nv_fp8x2_e5m2 *)ptr);
432 +}
433 +
434 +FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 2>::store(
435 + __nv_fp8_e5m2 *ptr) const {
436 + *((__nv_fp8x2_e5m2 *)ptr) = data;
437 +}
438 +
439 +FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 2>::memcpy(
440 + __nv_fp8_e5m2 *dst, const __nv_fp8_e5m2 *src) {
441 + *((__nv_fp8x2_e5m2 *)dst) = *((__nv_fp8x2_e5m2 *)src);
442 +}
443 +
444 +// __nv_fp8_e5m2 x 4
445 +
446 +template <>
447 +struct vec_t<__nv_fp8_e5m2, 4> {
448 + __nv_fp8x4_e5m2 data;
449 +
450 + FLASHINFER_INLINE __nv_fp8_e5m2 &operator[](size_t i) {
451 + return ((__nv_fp8_e5m2 *)(&data))[i];
452 + }
453 + FLASHINFER_INLINE const __nv_fp8_e5m2 &operator[](size_t i) const {
454 + return ((const __nv_fp8_e5m2 *)(&data))[i];
455 + }
456 + FLASHINFER_INLINE void fill(__nv_fp8_e5m2 val);
457 + FLASHINFER_INLINE void load(const __nv_fp8_e5m2 *ptr);
458 + FLASHINFER_INLINE void store(__nv_fp8_e5m2 *ptr) const;
459 + template <typename T>
460 + FLASHINFER_INLINE void cast_from(const vec_t<T, 4> &src) {
461 + cast_from_impl(src, *this);
462 + }
463 + template <typename T>
464 + FLASHINFER_INLINE void cast_load(const T *ptr) {
465 + cast_load_impl(ptr, *this);
466 + }
467 + template <typename T>
468 + FLASHINFER_INLINE void cast_store(T *ptr) const {
469 + cast_store_impl(*this, ptr);
470 + }
471 +
472 + FLASHINFER_INLINE static void memcpy(__nv_fp8_e5m2 *dst,
473 + const __nv_fp8_e5m2 *src);
474 +};
475 +
476 +FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 4>::fill(__nv_fp8_e5m2 val) {
477 + data.__x = (__nv_fp8x4_storage_t(val.__x) << 24) |
478 + (__nv_fp8x4_storage_t(val.__x) << 16) |
479 + (__nv_fp8x4_storage_t(val.__x) << 8) |
480 + __nv_fp8x4_storage_t(val.__x);
481 +}
482 +
483 +FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 4>::load(const __nv_fp8_e5m2 *ptr) {
484 + data = *((__nv_fp8x4_e5m2 *)ptr);
485 +}
486 +
487 +FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 4>::store(
488 + __nv_fp8_e5m2 *ptr) const {
489 + *((__nv_fp8x4_e5m2 *)ptr) = data;
490 +}
491 +
492 +FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 4>::memcpy(
493 + __nv_fp8_e5m2 *dst, const __nv_fp8_e5m2 *src) {
494 + *((__nv_fp8x4_e5m2 *)dst) = *((__nv_fp8x4_e5m2 *)src);
495 +}
496 +
497 +// __nv_fp8_e5m2 x 8
498 +
499 +template <>
500 +struct vec_t<__nv_fp8_e5m2, 8> {
501 + uint2 data;
502 +
503 + FLASHINFER_INLINE __nv_fp8_e5m2 &operator[](size_t i) {
504 + return ((__nv_fp8_e5m2 *)(&data))[i];
505 + }
506 + FLASHINFER_INLINE const __nv_fp8_e5m2 &operator[](size_t i) const {
507 + return ((const __nv_fp8_e5m2 *)(&data))[i];
508 + }
509 + FLASHINFER_INLINE void fill(__nv_fp8_e5m2 val);
510 + FLASHINFER_INLINE void load(const __nv_fp8_e5m2 *ptr);
511 + FLASHINFER_INLINE void store(__nv_fp8_e5m2 *ptr) const;
512 + template <typename T>
513 + FLASHINFER_INLINE void cast_from(const vec_t<T, 8> &src) {
514 + cast_from_impl(src, *this);
515 + }
516 + template <typename T>
517 + FLASHINFER_INLINE void cast_load(const T *ptr) {
518 + cast_load_impl(ptr, *this);
519 + }
520 + template <typename T>
521 + FLASHINFER_INLINE void cast_store(T *ptr) const {
522 + cast_store_impl(*this, ptr);
523 + }
524 +
525 + FLASHINFER_INLINE static void memcpy(__nv_fp8_e5m2 *dst,
526 + const __nv_fp8_e5m2 *src);
527 +};
528 +
529 +FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 8>::fill(__nv_fp8_e5m2 val) {
530 + ((__nv_fp8x4_e5m2 *)(&data.x))->__x = (__nv_fp8x4_storage_t(val.__x) << 24) |
531 + (__nv_fp8x4_storage_t(val.__x) << 16) |
532 + (__nv_fp8x4_storage_t(val.__x) << 8) |
533 + __nv_fp8x4_storage_t(val.__x);
534 + ((__nv_fp8x4_e5m2 *)(&data.y))->__x = (__nv_fp8x4_storage_t(val.__x) << 24) |
535 + (__nv_fp8x4_storage_t(val.__x) << 16) |
536 + (__nv_fp8x4_storage_t(val.__x) << 8) |
537 + __nv_fp8x4_storage_t(val.__x);
538 +}
539 +
540 +FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 8>::load(const __nv_fp8_e5m2 *ptr) {
541 + data = *((uint2 *)ptr);
542 +}
543 +
544 +FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 8>::store(
545 + __nv_fp8_e5m2 *ptr) const {
546 + *((uint2 *)ptr) = data;
547 +}
548 +
549 +FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 8>::memcpy(
550 + __nv_fp8_e5m2 *dst, const __nv_fp8_e5m2 *src) {
551 + *((__nv_fp8_e5m2 *)dst) = *((__nv_fp8_e5m2 *)src);
552 +}
553 +
554 +// __nv_fp8_e5m2 x 16 or more
555 +
556 +template <size_t vec_size>
557 +struct vec_t<__nv_fp8_e5m2, vec_size> {
558 + uint4 data[vec_size / 16];
559 +
560 + FLASHINFER_INLINE __nv_fp8_e5m2 &operator[](size_t i) {
561 + return ((__nv_fp8_e5m2 *)data)[i];
562 + }
563 + FLASHINFER_INLINE const __nv_fp8_e5m2 &operator[](size_t i) const {
564 + return ((const __nv_fp8_e5m2 *)data)[i];
565 + }
566 + FLASHINFER_INLINE void fill(__nv_fp8_e5m2 val) {
567 +#pragma unroll
568 + for (size_t i = 0; i < vec_size / 16; ++i) {
569 + ((__nv_fp8x4_e5m2 *)(&(data[i].x)))->__x =
570 + (__nv_fp8x4_storage_t(val.__x) << 24) |
571 + (__nv_fp8x4_storage_t(val.__x) << 16) |
572 + (__nv_fp8x4_storage_t(val.__x) << 8) | __nv_fp8x4_storage_t(val.__x);
573 + ((__nv_fp8x4_e5m2 *)(&(data[i].y)))->__x =
574 + (__nv_fp8x4_storage_t(val.__x) << 24) |
575 + (__nv_fp8x4_storage_t(val.__x) << 16) |
576 + (__nv_fp8x4_storage_t(val.__x) << 8) | __nv_fp8x4_storage_t(val.__x);
577 + ((__nv_fp8x4_e5m2 *)(&(data[i].z)))->__x =
578 + (__nv_fp8x4_storage_t(val.__x) << 24) |
579 + (__nv_fp8x4_storage_t(val.__x) << 16) |
580 + (__nv_fp8x4_storage_t(val.__x) << 8) | __nv_fp8x4_storage_t(val.__x);
581 + ((__nv_fp8x4_e5m2 *)(&(data[i].w)))->__x =
582 + (__nv_fp8x4_storage_t(val.__x) << 24) |
583 + (__nv_fp8x4_storage_t(val.__x) << 16) |
584 + (__nv_fp8x4_storage_t(val.__x) << 8) | __nv_fp8x4_storage_t(val.__x);
585 + }
586 + }
587 + FLASHINFER_INLINE void load(const __nv_fp8_e5m2 *ptr) {
588 +#pragma unroll
589 + for (size_t i = 0; i < vec_size / 16; ++i) {
590 + data[i] = ((uint4 *)ptr)[i];
591 + }
592 + }
593 + FLASHINFER_INLINE void store(__nv_fp8_e5m2 *ptr) const {
594 +#pragma unroll
595 + for (size_t i = 0; i < vec_size / 16; ++i) {
596 + ((uint4 *)ptr)[i] = data[i];
597 + }
598 + }
599 + template <typename T>
600 + FLASHINFER_INLINE void cast_from(const vec_t<T, vec_size> &src) {
601 + cast_from_impl(src, *this);
602 + }
603 + template <typename T>
604 + FLASHINFER_INLINE void cast_load(const T *ptr) {
605 + cast_load_impl(ptr, *this);
606 + }
607 + template <typename T>
608 + FLASHINFER_INLINE void cast_store(T *ptr) const {
609 + cast_store_impl(*this, ptr);
610 + }
611 +
612 + FLASHINFER_INLINE static void memcpy(__nv_fp8_e5m2 *dst,
613 + const __nv_fp8_e5m2 *src) {
614 +#pragma unroll
615 + for (size_t i = 0; i < vec_size / 16; ++i) {
616 + ((uint4 *)dst)[i] = ((uint4 *)src)[i];
617 + }
618 + }
619 +};
620 +#endif
621 +
622 +/******************* vec_t<half> *******************/
623 +
624 +// half x 1
625 +template <>
626 +struct vec_t<half, 1> {
627 + half data;
628 +
629 + FLASHINFER_INLINE half &operator[](size_t i) { return ((half *)(&data))[i]; }
630 + FLASHINFER_INLINE const half &operator[](size_t i) const {
631 + return ((const half *)(&data))[i];
632 + }
633 + FLASHINFER_INLINE void fill(half val);
634 + FLASHINFER_INLINE void load(const half *ptr);
635 + FLASHINFER_INLINE void store(half *ptr) const;
636 + template <typename T>
637 + FLASHINFER_INLINE void cast_from(const vec_t<T, 1> &src) {
638 + cast_from_impl(src, *this);
639 + }
640 + template <typename T>
641 + FLASHINFER_INLINE void cast_load(const T *ptr) {
642 + cast_load_impl(ptr, *this);
643 + }
644 + template <typename T>
645 + FLASHINFER_INLINE void cast_store(T *ptr) const {
646 + cast_store_impl(*this, ptr);
647 + }
648 +
649 + FLASHINFER_INLINE static void memcpy(half *dst, const half *src);
650 +};
651 +
652 +FLASHINFER_INLINE void vec_t<half, 1>::fill(half val) { data = val; }
653 +
654 +FLASHINFER_INLINE void vec_t<half, 1>::load(const half *ptr) { data = *ptr; }
655 +
656 +FLASHINFER_INLINE void vec_t<half, 1>::store(half *ptr) const { *ptr = data; }
657 +
658 +FLASHINFER_INLINE void vec_t<half, 1>::memcpy(half *dst, const half *src) {
659 + *dst = *src;
660 +}
661 +
662 +// half x 2
663 +template <>
664 +struct vec_t<half, 2> {
665 + half2 data;
666 +
667 + FLASHINFER_INLINE half &operator[](size_t i) { return ((half *)(&data))[i]; }
668 + FLASHINFER_INLINE const half &operator[](size_t i) const {
669 + return ((const half *)(&data))[i];
670 + }
671 + FLASHINFER_INLINE void fill(half val);
672 + FLASHINFER_INLINE void load(const half *ptr);
673 + FLASHINFER_INLINE void store(half *ptr) const;
674 + template <typename T>
675 + FLASHINFER_INLINE void cast_from(const vec_t<T, 2> &src) {
676 + cast_from_impl(src, *this);
677 + }
678 + template <typename T>
679 + FLASHINFER_INLINE void cast_load(const T *ptr) {
680 + cast_load_impl(ptr, *this);
681 + }
682 + template <typename T>
683 + FLASHINFER_INLINE void cast_store(T *ptr) const {
684 + cast_store_impl(*this, ptr);
685 + }
686 +
687 + FLASHINFER_INLINE static void memcpy(half *dst, const half *src);
688 +};
689 +
690 +FLASHINFER_INLINE void vec_t<half, 2>::fill(half val) {
691 + data = make_half2(val, val);
692 +}
693 +
694 +FLASHINFER_INLINE void vec_t<half, 2>::load(const half *ptr) {
695 + data = *((half2 *)ptr);
696 +}
697 +
698 +FLASHINFER_INLINE void vec_t<half, 2>::store(half *ptr) const {
699 + *((half2 *)ptr) = data;
700 +}
701 +
702 +FLASHINFER_INLINE void vec_t<half, 2>::memcpy(half *dst, const half *src) {
703 + *((half2 *)dst) = *((half2 *)src);
704 +}
705 +
706 +// half x 4
707 +
708 +template <>
709 +struct vec_t<half, 4> {
710 + uint2 data;
711 +
712 + FLASHINFER_INLINE half &operator[](size_t i) { return ((half *)(&data))[i]; }
713 + FLASHINFER_INLINE const half &operator[](size_t i) const {
714 + return ((const half *)(&data))[i];
715 + }
716 + FLASHINFER_INLINE void fill(half val);
717 + FLASHINFER_INLINE void load(const half *ptr);
718 + FLASHINFER_INLINE void store(half *ptr) const;
719 + template <typename T>
720 + FLASHINFER_INLINE void cast_from(const vec_t<T, 4> &src) {
721 + cast_from_impl(src, *this);
722 + }
723 + template <typename T>
724 + FLASHINFER_INLINE void cast_load(const T *ptr) {
725 + cast_load_impl(ptr, *this);
726 + }
727 + template <typename T>
728 + FLASHINFER_INLINE void cast_store(T *ptr) const {
729 + cast_store_impl(*this, ptr);
730 + }
731 +
732 + FLASHINFER_INLINE static void memcpy(half *dst, const half *src);
733 +};
734 +
735 +FLASHINFER_INLINE void vec_t<half, 4>::fill(half val) {
736 + *(half2 *)(&data.x) = make_half2(val, val);
737 + *(half2 *)(&data.y) = make_half2(val, val);
738 +}
739 +
740 +FLASHINFER_INLINE void vec_t<half, 4>::load(const half *ptr) {
741 + data = *((uint2 *)ptr);
742 +}
743 +
744 +FLASHINFER_INLINE void vec_t<half, 4>::store(half *ptr) const {
745 + *((uint2 *)ptr) = data;
746 +}
747 +
748 +FLASHINFER_INLINE void vec_t<half, 4>::memcpy(half *dst, const half *src) {
749 + *((uint2 *)dst) = *((uint2 *)src);
750 +}
751 +
752 +// half x 8 or more
753 +
754 +template <size_t vec_size>
755 +struct vec_t<half, vec_size> {
756 + uint4 data[vec_size / 8];
757 + FLASHINFER_INLINE half &operator[](size_t i) { return ((half *)data)[i]; }
758 + FLASHINFER_INLINE const half &operator[](size_t i) const {
759 + return ((const half *)data)[i];
760 + }
761 + FLASHINFER_INLINE void fill(half val) {
762 +#pragma unroll
763 + for (size_t i = 0; i < vec_size; ++i) {
764 + *(half2 *)(&(data[i].x)) = make_half2(val, val);
765 + *(half2 *)(&(data[i].y)) = make_half2(val, val);
766 + *(half2 *)(&(data[i].z)) = make_half2(val, val);
767 + *(half2 *)(&(data[i].w)) = make_half2(val, val);
768 + }
769 + }
770 + FLASHINFER_INLINE void load(const half *ptr) {
771 +#pragma unroll
772 + for (size_t i = 0; i < vec_size / 8; ++i) {
773 + data[i] = ((uint4 *)ptr)[i];
774 + }
775 + }
776 + FLASHINFER_INLINE void store(half *ptr) const {
777 +#pragma unroll
778 + for (size_t i = 0; i < vec_size / 8; ++i) {
779 + ((uint4 *)ptr)[i] = data[i];
780 + }
781 + }
782 + template <typename T>
783 + FLASHINFER_INLINE void cast_from(const vec_t<T, vec_size> &src) {
784 + cast_from_impl(src, *this);
785 + }
786 + template <typename T>
787 + FLASHINFER_INLINE void cast_load(const T *ptr) {
788 + cast_load_impl(ptr, *this);
789 + }
790 + template <typename T>
791 + FLASHINFER_INLINE void cast_store(T *ptr) const {
792 + cast_store_impl(*this, ptr);
793 + }
794 +
795 + FLASHINFER_INLINE static void memcpy(half *dst, const half *src) {
796 +#pragma unroll
797 + for (size_t i = 0; i < vec_size / 8; ++i) {
798 + ((uint4 *)dst)[i] = ((uint4 *)src)[i];
799 + }
800 + }
801 +};
802 +
803 +/******************* vec_t<nv_bfloat16> *******************/
804 +
805 +// nv_bfloat16 x 1
806 +template <>
807 +struct vec_t<nv_bfloat16, 1> {
808 + nv_bfloat16 data;
809 +
810 + FLASHINFER_INLINE nv_bfloat16 &operator[](size_t i) {
811 + return ((nv_bfloat16 *)(&data))[i];
812 + }
813 + FLASHINFER_INLINE const nv_bfloat16 &operator[](size_t i) const {
814 + return ((const nv_bfloat16 *)(&data))[i];
815 + }
816 + FLASHINFER_INLINE void fill(nv_bfloat16 val);
817 + FLASHINFER_INLINE void load(const nv_bfloat16 *ptr);
818 + FLASHINFER_INLINE void store(nv_bfloat16 *ptr) const;
819 + template <typename T>
820 + FLASHINFER_INLINE void cast_from(const vec_t<T, 1> &src) {
821 + cast_from_impl(src, *this);
822 + }
823 + template <typename T>
824 + FLASHINFER_INLINE void cast_load(const T *ptr) {
825 + cast_load_impl(ptr, *this);
826 + }
827 + template <typename T>
828 + FLASHINFER_INLINE void cast_store(T *ptr) const {
829 + cast_store_impl(*this, ptr);
830 + }
831 +
832 + FLASHINFER_INLINE static void memcpy(nv_bfloat16 *dst,
833 + const nv_bfloat16 *src);
834 +};
835 +
836 +FLASHINFER_INLINE void vec_t<nv_bfloat16, 1>::fill(nv_bfloat16 val) {
837 + data = val;
838 +}
839 +
840 +FLASHINFER_INLINE void vec_t<nv_bfloat16, 1>::load(const nv_bfloat16 *ptr) {
841 + data = *ptr;
842 +}
843 +
844 +FLASHINFER_INLINE void vec_t<nv_bfloat16, 1>::store(nv_bfloat16 *ptr) const {
845 + *ptr = data;
846 +}
847 +
848 +FLASHINFER_INLINE void vec_t<nv_bfloat16, 1>::memcpy(nv_bfloat16 *dst,
849 + const nv_bfloat16 *src) {
850 + *dst = *src;
851 +}
852 +
853 +// nv_bfloat16 x 2
854 +template <>
855 +struct vec_t<nv_bfloat16, 2> {
856 + nv_bfloat162 data;
857 +
858 + FLASHINFER_INLINE nv_bfloat16 &operator[](size_t i) {
859 + return ((nv_bfloat16 *)(&data))[i];
860 + }
861 + FLASHINFER_INLINE const nv_bfloat16 &operator[](size_t i) const {
862 + return ((const nv_bfloat16 *)(&data))[i];
863 + }
864 + FLASHINFER_INLINE void fill(nv_bfloat16 val);
865 + FLASHINFER_INLINE void load(const nv_bfloat16 *ptr);
866 + FLASHINFER_INLINE void store(nv_bfloat16 *ptr) const;
867 + template <typename T>
868 + FLASHINFER_INLINE void cast_from(const vec_t<T, 2> &src) {
869 + cast_from_impl(src, *this);
870 + }
871 + template <typename T>
872 + FLASHINFER_INLINE void cast_load(const T *ptr) {
873 + cast_load_impl(ptr, *this);
874 + }
875 + template <typename T>
876 + FLASHINFER_INLINE void cast_store(T *ptr) const {
877 + cast_store_impl(*this, ptr);
878 + }
879 +
880 + FLASHINFER_INLINE static void memcpy(nv_bfloat16 *dst,
881 + const nv_bfloat16 *src);
882 +};
883 +
884 +FLASHINFER_INLINE void vec_t<nv_bfloat16, 2>::fill(nv_bfloat16 val) {
885 + data = make_bfloat162(val, val);
886 +}
887 +
888 +FLASHINFER_INLINE void vec_t<nv_bfloat16, 2>::load(const nv_bfloat16 *ptr) {
889 + data = *((nv_bfloat162 *)ptr);
890 +}
891 +
892 +FLASHINFER_INLINE void vec_t<nv_bfloat16, 2>::store(nv_bfloat16 *ptr) const {
893 + *((nv_bfloat162 *)ptr) = data;
894 +}
895 +
896 +FLASHINFER_INLINE void vec_t<nv_bfloat16, 2>::memcpy(nv_bfloat16 *dst,
897 + const nv_bfloat16 *src) {
898 + *((nv_bfloat162 *)dst) = *((nv_bfloat162 *)src);
899 +}
900 +
901 +// nv_bfloat16 x 4
902 +
903 +template <>
904 +struct vec_t<nv_bfloat16, 4> {
905 + uint2 data;
906 +
907 + FLASHINFER_INLINE nv_bfloat16 &operator[](size_t i) {
908 + return ((nv_bfloat16 *)(&data))[i];
909 + }
910 + FLASHINFER_INLINE const nv_bfloat16 &operator[](size_t i) const {
911 + return ((const nv_bfloat16 *)(&data))[i];
912 + }
913 + FLASHINFER_INLINE void fill(nv_bfloat16 val);
914 + FLASHINFER_INLINE void load(const nv_bfloat16 *ptr);
915 + FLASHINFER_INLINE void store(nv_bfloat16 *ptr) const;
916 + template <typename T>
917 + FLASHINFER_INLINE void cast_from(const vec_t<T, 4> &src) {
918 + cast_from_impl(src, *this);
919 + }
920 + template <typename T>
921 + FLASHINFER_INLINE void cast_load(const T *ptr) {
922 + cast_load_impl(ptr, *this);
923 + }
924 + template <typename T>
925 + FLASHINFER_INLINE void cast_store(T *ptr) const {
926 + cast_store_impl(*this, ptr);
927 + }
928 +
929 + FLASHINFER_INLINE static void memcpy(nv_bfloat16 *dst,
930 + const nv_bfloat16 *src);
931 +};
932 +
933 +FLASHINFER_INLINE void vec_t<nv_bfloat16, 4>::fill(nv_bfloat16 val) {
934 + *(nv_bfloat162 *)(&data.x) = make_bfloat162(val, val);
935 + *(nv_bfloat162 *)(&data.y) = make_bfloat162(val, val);
936 +}
937 +
938 +FLASHINFER_INLINE void vec_t<nv_bfloat16, 4>::load(const nv_bfloat16 *ptr) {
939 + data = *((uint2 *)ptr);
940 +}
941 +
942 +FLASHINFER_INLINE void vec_t<nv_bfloat16, 4>::store(nv_bfloat16 *ptr) const {
943 + *((uint2 *)ptr) = data;
944 +}
945 +
946 +FLASHINFER_INLINE void vec_t<nv_bfloat16, 4>::memcpy(nv_bfloat16 *dst,
947 + const nv_bfloat16 *src) {
948 + *((uint2 *)dst) = *((uint2 *)src);
949 +}
950 +
951 +// nv_bfloat16 x 8 or more
952 +
953 +template <size_t vec_size>
954 +struct vec_t<nv_bfloat16, vec_size> {
955 + uint4 data[vec_size / 8];
956 +
957 + FLASHINFER_INLINE nv_bfloat16 &operator[](size_t i) {
958 + return ((nv_bfloat16 *)data)[i];
959 + }
960 + FLASHINFER_INLINE const nv_bfloat16 &operator[](size_t i) const {
961 + return ((const nv_bfloat16 *)data)[i];
962 + }
963 + FLASHINFER_INLINE void fill(nv_bfloat16 val) {
964 +#pragma unoll
965 + for (size_t i = 0; i < vec_size / 8; ++i) {
966 + *(nv_bfloat162 *)(&(data[i].x)) = make_bfloat162(val, val);
967 + *(nv_bfloat162 *)(&(data[i].y)) = make_bfloat162(val, val);
968 + *(nv_bfloat162 *)(&(data[i].z)) = make_bfloat162(val, val);
969 + *(nv_bfloat162 *)(&(data[i].w)) = make_bfloat162(val, val);
970 + }
971 + }
972 + FLASHINFER_INLINE void load(const nv_bfloat16 *ptr) {
973 +#pragma unoll
974 + for (size_t i = 0; i < vec_size / 8; ++i) {
975 + data[i] = ((uint4 *)ptr)[i];
976 + }
977 + }
978 + FLASHINFER_INLINE void store(nv_bfloat16 *ptr) const {
979 +#pragma unoll
980 + for (size_t i = 0; i < vec_size / 8; ++i) {
981 + ((uint4 *)ptr)[i] = data[i];
982 + }
983 + }
984 + template <typename T>
985 + FLASHINFER_INLINE void cast_from(const vec_t<T, vec_size> &src) {
986 + cast_from_impl(src, *this);
987 + }
988 + template <typename T>
989 + FLASHINFER_INLINE void cast_load(const T *ptr) {
990 + cast_load_impl(ptr, *this);
991 + }
992 + template <typename T>
993 + FLASHINFER_INLINE void cast_store(T *ptr) const {
994 + cast_store_impl(*this, ptr);
995 + }
996 +
997 + FLASHINFER_INLINE static void memcpy(nv_bfloat16 *dst,
998 + const nv_bfloat16 *src) {
999 +#pragma unoll
1000 + for (size_t i = 0; i < vec_size / 8; ++i) {
1001 + ((uint4 *)dst)[i] = ((uint4 *)src)[i];
1002 + }
1003 + }
1004 +};
1005 +
1006 +/******************* vec_t<float> *******************/
1007 +
1008 +// float x 1
1009 +
1010 +template <>
1011 +struct vec_t<float, 1> {
1012 + float data;
1013 +
1014 + FLASHINFER_INLINE float &operator[](size_t i) {
1015 + return ((float *)(&data))[i];
1016 + }
1017 + FLASHINFER_INLINE const float &operator[](size_t i) const {
1018 + return ((const float *)(&data))[i];
1019 + }
1020 + FLASHINFER_INLINE void fill(float val);
1021 + FLASHINFER_INLINE void load(const float *ptr);
1022 + FLASHINFER_INLINE void store(float *ptr) const;
1023 + template <typename T>
1024 + FLASHINFER_INLINE void cast_from(const vec_t<T, 1> &src) {
1025 + cast_from_impl(src, *this);
1026 + }
1027 + template <typename T>
1028 + FLASHINFER_INLINE void cast_load(const T *ptr) {
1029 + cast_load_impl(ptr, *this);
1030 + }
1031 + template <typename T>
1032 + FLASHINFER_INLINE void cast_store(T *ptr) const {
1033 + cast_store_impl(*this, ptr);
1034 + }
1035 +
1036 + FLASHINFER_INLINE static void memcpy(float *dst, const float *src);
1037 +};
1038 +
1039 +FLASHINFER_INLINE void vec_t<float, 1>::fill(float val) { data = val; }
1040 +
1041 +FLASHINFER_INLINE void vec_t<float, 1>::load(const float *ptr) { data = *ptr; }
1042 +
1043 +FLASHINFER_INLINE void vec_t<float, 1>::store(float *ptr) const { *ptr = data; }
1044 +
1045 +FLASHINFER_INLINE void vec_t<float, 1>::memcpy(float *dst, const float *src) {
1046 + *dst = *src;
1047 +}
1048 +
1049 +// float x 2
1050 +
1051 +template <>
1052 +struct vec_t<float, 2> {
1053 + float2 data;
1054 +
1055 + FLASHINFER_INLINE float &operator[](size_t i) {
1056 + return ((float *)(&data))[i];
1057 + }
1058 + FLASHINFER_INLINE const float &operator[](size_t i) const {
1059 + return ((const float *)(&data))[i];
1060 + }
1061 + FLASHINFER_INLINE void fill(float val);
1062 + FLASHINFER_INLINE void load(const float *ptr);
1063 + FLASHINFER_INLINE void store(float *ptr) const;
1064 + template <typename T>
1065 + FLASHINFER_INLINE void cast_from(const vec_t<T, 2> &src) {
1066 + cast_from_impl(src, *this);
1067 + }
1068 + template <typename T>
1069 + FLASHINFER_INLINE void cast_load(const T *ptr) {
1070 + cast_load_impl(ptr, *this);
1071 + }
1072 + template <typename T>
1073 + FLASHINFER_INLINE void cast_store(T *ptr) const {
1074 + cast_store_impl(*this, ptr);
1075 + }
1076 + FLASHINFER_INLINE static void memcpy(float *dst, const float *src);
1077 +};
1078 +
1079 +FLASHINFER_INLINE void vec_t<float, 2>::fill(float val) {
1080 + data = make_float2(val, val);
1081 +}
1082 +
1083 +FLASHINFER_INLINE void vec_t<float, 2>::load(const float *ptr) {
1084 + data = *((float2 *)ptr);
1085 +}
1086 +
1087 +FLASHINFER_INLINE void vec_t<float, 2>::store(float *ptr) const {
1088 + *((float2 *)ptr) = data;
1089 +}
1090 +
1091 +FLASHINFER_INLINE void vec_t<float, 2>::memcpy(float *dst, const float *src) {
1092 + *((float2 *)dst) = *((float2 *)src);
1093 +}
1094 +
1095 +// float x 4 or more
1096 +template <size_t vec_size>
1097 +struct vec_t<float, vec_size> {
1098 + float4 data[vec_size / 4];
1099 +
1100 + FLASHINFER_INLINE float &operator[](size_t i) { return ((float *)(data))[i]; }
1101 + FLASHINFER_INLINE const float &operator[](size_t i) const {
1102 + return ((const float *)(data))[i];
1103 + }
1104 + FLASHINFER_INLINE void fill(float val) {
1105 +#pragma unroll
1106 + for (size_t i = 0; i < vec_size / 4; ++i) {
1107 + data[i] = make_float4(val, val, val, val);
1108 + }
1109 + }
1110 + FLASHINFER_INLINE void load(const float *ptr) {
1111 +#pragma unroll
1112 + for (size_t i = 0; i < vec_size / 4; ++i) {
1113 + data[i] = ((float4 *)ptr)[i];
1114 + }
1115 + }
1116 + FLASHINFER_INLINE void store(float *ptr) const {
1117 +#pragma unroll
1118 + for (size_t i = 0; i < vec_size / 4; ++i) {
1119 + ((float4 *)ptr)[i] = data[i];
1120 + }
1121 + }
1122 + template <typename T>
1123 + FLASHINFER_INLINE void cast_from(const vec_t<T, vec_size> &src) {
1124 + cast_from_impl(src, *this);
1125 + }
1126 + template <typename T>
1127 + FLASHINFER_INLINE void cast_load(const T *ptr) {
1128 + cast_load_impl(ptr, *this);
1129 + }
1130 + template <typename T>
1131 + FLASHINFER_INLINE void cast_store(T *ptr) const {
1132 + cast_store_impl(*this, ptr);
1133 + }
1134 + FLASHINFER_INLINE static void memcpy(float *dst, const float *src) {
1135 +#pragma unroll
1136 + for (size_t i = 0; i < vec_size / 4; ++i) {
1137 + ((float4 *)dst)[i] = ((float4 *)src)[i];
1138 + }
1139 + }
1140 +};
1141 +
1142 +/******************* vec_t type cast *******************/
1143 +
1144 +template <size_t vec_size>
1145 +FLASHINFER_INLINE void cast_from_impl(const vec_t<half, vec_size> &src,
1146 + vec_t<float, vec_size> &dst) {
1147 + if constexpr (vec_size == 1) {
1148 + dst.data = float(src.data);
1149 + } else {
1150 +#pragma unroll
1151 + for (size_t i = 0; i < vec_size / 2; ++i) {
1152 + ((float2 *)(&dst.data))[i] = __half22float2(((half2 *)(&src.data))[i]);
1153 + }
1154 + }
1155 +}
1156 +
1157 +template <size_t vec_size>
1158 +FLASHINFER_INLINE void cast_from_impl(const vec_t<float, vec_size> &src,
1159 + vec_t<half, vec_size> &dst) {
1160 + if constexpr (vec_size == 1) {
1161 + dst.data = half(src.data);
1162 + } else {
1163 +#pragma unroll
1164 + for (size_t i = 0; i < vec_size / 2; ++i) {
1165 + ((half2 *)(&dst.data))[i] = __float22half2_rn(((float2 *)(&src.data))[i]);
1166 + }
1167 + }
1168 +}
1169 +
1170 +template <size_t vec_size>
1171 +FLASHINFER_INLINE void cast_from_impl(const vec_t<nv_bfloat16, vec_size> &src,
1172 + vec_t<float, vec_size> &dst) {
1173 + if constexpr (vec_size == 1) {
1174 + dst.data = float(src.data);
1175 + } else {
1176 +#pragma unroll
1177 + for (size_t i = 0; i < vec_size / 2; ++i) {
1178 + ((float2 *)(&dst.data))[i] =
1179 + __bfloat1622float2(((nv_bfloat162 *)(&src.data))[i]);
1180 + }
1181 + }
1182 +}
1183 +
1184 +template <size_t vec_size>
1185 +FLASHINFER_INLINE void cast_from_impl(const vec_t<float, vec_size> &src,
1186 + vec_t<nv_bfloat16, vec_size> &dst) {
1187 + if constexpr (vec_size == 1) {
1188 + dst.data = nv_bfloat16(src.data);
1189 + } else {
1190 +#pragma unroll
1191 + for (size_t i = 0; i < vec_size / 2; ++i) {
1192 + ((nv_bfloat162 *)(&dst.data))[i] =
1193 + __float22bfloat162_rn(((float2 *)(&src.data))[i]);
1194 + }
1195 + }
1196 +}
1197 +
1198 +#ifdef FLASHINFER_USE_FP8
1199 +
1200 +template <size_t vec_size>
1201 +FLASHINFER_INLINE void cast_from_impl(const vec_t<__nv_fp8_e4m3, vec_size> &src,
1202 + vec_t<float, vec_size> &dst) {
1203 + if constexpr (vec_size == 1) {
1204 + dst.data = float(src.data);
1205 + } else if constexpr (vec_size == 2) {
1206 + *(float2 *)(&dst.data) = float2(*(__nv_fp8x2_e4m3 *)(&src.data));
1207 + } else {
1208 +#pragma unroll
1209 + for (size_t i = 0; i < vec_size / 4; ++i) {
1210 + ((float4 *)(&dst.data))[i] = float4(((__nv_fp8x4_e4m3 *)(&src.data))[i]);
1211 + }
1212 + }
1213 +}
1214 +
1215 +template <size_t vec_size>
1216 +FLASHINFER_INLINE void cast_from_impl(const vec_t<__nv_fp8_e4m3, vec_size> &src,
1217 + vec_t<half, vec_size> &dst) {
1218 + if constexpr (vec_size == 1) {
1219 + dst.data = float(src.data);
1220 + } else {
1221 +#pragma unroll
1222 + for (size_t i = 0; i < vec_size / 2; ++i) {
1223 + ((half2 *)(&dst.data))[i] = half2(((__nv_fp8x2_e4m3 *)(&src.data))[i]);
1224 + }
1225 + }
1226 +}
1227 +
1228 +template <size_t vec_size>
1229 +FLASHINFER_INLINE void cast_from_impl(const vec_t<float, vec_size> &src,
1230 + vec_t<__nv_fp8_e4m3, vec_size> &dst) {
1231 + if constexpr (vec_size == 1) {
1232 + dst.data = __nv_fp8_e4m3(src.data);
1233 + } else if constexpr (vec_size == 2) {
1234 + *(__nv_fp8x2_e4m3 *)(&dst.data) = __nv_fp8x2_e4m3(*(float2 *)(&src.data));
1235 + } else {
1236 +#pragma unroll
1237 + for (size_t i = 0; i < vec_size / 4; ++i) {
1238 + ((__nv_fp8x4_e4m3 *)(&dst.data))[i] =
1239 + __nv_fp8x4_e4m3(((float4 *)(&src.data))[i]);
1240 + }
1241 + }
1242 +}
1243 +
1244 +template <size_t vec_size>
1245 +FLASHINFER_INLINE void cast_from_impl(const vec_t<half, vec_size> &src,
1246 + vec_t<__nv_fp8_e4m3, vec_size> &dst) {
1247 + if constexpr (vec_size == 1) {
1248 + dst.data = __nv_fp8_e4m3(src.data);
1249 + } else if constexpr (vec_size == 2) {
1250 + *(__nv_fp8x2_e4m3 *)(&dst.data) = __nv_fp8x2_e4m3(*(half2 *)(&src.data));
1251 + } else {
1252 +#pragma unroll
1253 + for (size_t i = 0; i < vec_size / 4; ++i) {
1254 + // NOTE(Zihao): need to double check if we properly handle flo and fhi
1255 + ((__nv_fp8x4_e4m3 *)(&dst.data))[i] = __nv_fp8x4_e4m3(
1256 + ((half2 *)(&src.data))[i * 2], ((half2 *)(&src.data))[i * 2 + 1]);
1257 + }
1258 + }
1259 +}
1260 +
1261 +template <size_t vec_size>
1262 +FLASHINFER_INLINE void cast_from_impl(const vec_t<__nv_fp8_e5m2, vec_size> &src,
1263 + vec_t<float, vec_size> &dst) {
1264 + if constexpr (vec_size == 1) {
1265 + dst.data = float(src.data);
1266 + } else if constexpr (vec_size == 2) {
1267 + *(float2 *)(&dst.data) = float2(*(__nv_fp8x2_e5m2 *)(&src.data));
1268 + } else {
1269 +#pragma unroll
1270 + for (size_t i = 0; i < vec_size / 4; ++i) {
1271 + ((float4 *)(&dst.data))[i] = float4(((__nv_fp8x4_e5m2 *)(&src.data))[i]);
1272 + }
1273 + }
1274 +}
1275 +
1276 +template <size_t vec_size>
1277 +FLASHINFER_INLINE void cast_from_impl(const vec_t<__nv_fp8_e5m2, vec_size> &src,
1278 + vec_t<half, vec_size> &dst) {
1279 + if constexpr (vec_size == 1) {
1280 + dst.data = float(src.data);
1281 + } else {
1282 +#pragma unroll
1283 + for (size_t i = 0; i < vec_size / 2; ++i) {
1284 + ((half2 *)(&dst.data))[i] = half2(((__nv_fp8x2_e5m2 *)(&src.data))[i]);
1285 + }
1286 + }
1287 +}
1288 +
1289 +template <size_t vec_size>
1290 +FLASHINFER_INLINE void cast_from_impl(const vec_t<float, vec_size> &src,
1291 + vec_t<__nv_fp8_e5m2, vec_size> &dst) {
1292 + if constexpr (vec_size == 1) {
1293 + dst.data = __nv_fp8_e5m2(src.data);
1294 + } else if constexpr (vec_size == 2) {
1295 + *(__nv_fp8x2_e5m2 *)(&dst.data) = __nv_fp8x2_e5m2(*(float2 *)(&src.data));
1296 + } else {
1297 +#pragma unroll
1298 + for (size_t i = 0; i < vec_size / 4; ++i) {
1299 + ((__nv_fp8x4_e5m2 *)(&dst.data))[i] =
1300 + __nv_fp8x4_e5m2(((float4 *)(&src.data))[i]);
1301 + }
1302 + }
1303 +}
1304 +
1305 +template <size_t vec_size>
1306 +FLASHINFER_INLINE void cast_from_impl(const vec_t<half, vec_size> &src,
1307 + vec_t<__nv_fp8_e5m2, vec_size> &dst) {
1308 + if constexpr (vec_size == 1) {
1309 + dst.data = __nv_fp8_e4m3(src.data);
1310 + } else if constexpr (vec_size == 2) {
1311 + *(__nv_fp8x2_e5m2 *)(&dst.data) = __nv_fp8x2_e5m2(*(half2 *)(&src.data));
1312 + } else {
1313 +#pragma unroll
1314 + for (size_t i = 0; i < vec_size / 4; ++i) {
1315 + // NOTE(Zihao): need to double check if we properly handle flo and fhi
1316 + ((__nv_fp8x4_e5m2 *)(&dst.data))[i] = __nv_fp8x4_e5m2(
1317 + ((half2 *)(&src.data))[i * 2], ((half2 *)(&src.data))[i * 2 + 1]);
1318 + }
1319 + }
1320 +}
1321 +
1322 +#endif // FLASHINFER_USE_FP8
1323 +
1324 +#endif // VEC_DTYPES_CUH_
csrc/punica/punica_ops.cc new
+563
@@ -0,0 +1,563 @@
1 +#include <cuda_bf16.h>
2 +#include <cuda_fp16.h>
3 +#include <torch/extension.h>
4 +
5 +#include <cstdint>
6 +
7 +#include "bgmv/bgmv_config.h"
8 +
9 +namespace {
10 +
11 +//====== utils ======
12 +
13 +inline void check_shape(const torch::Tensor &a, const torch::Tensor &b,
14 + const char *a_name, const char *b_name) {
15 + TORCH_CHECK(a.dim() == b.dim(), a_name, ".dim() != ", b_name, ".dim(). ",
16 + a.dim(), " vs ", b.dim());
17 + for (int i = 0; i < a.dim(); ++i) {
18 + TORCH_CHECK(a.size(i) == b.size(i), a_name, ".size(", i, ") != ", b_name,
19 + ".size(", i, ")");
20 + }
21 +}
22 +
23 +inline constexpr uint32_t pack_u16(uint16_t a, uint16_t b) {
24 + return (uint32_t(a) << 16) | uint32_t(b);
25 +}
26 +
27 +#define CHECK_CUDA(x) TORCH_CHECK(x.is_cuda(), #x " must be a CUDA tensor")
28 +
29 +#define CHECK_CONTIGUOUS(x) \
30 + TORCH_CHECK(x.is_contiguous(), #x " must be contiguous")
31 +
32 +#define CHECK_INPUT(x) \
33 + CHECK_CUDA(x); \
34 + CHECK_CONTIGUOUS(x)
35 +
36 +#define CHECK_DIM(d, x) \
37 + TORCH_CHECK(x.dim() == d, #x " must be a " #d "D tensor")
38 +
39 +#define CHECK_SHAPE(a, b) check_shape(a, b, #a, #b)
40 +
41 +#define CHECK_EQ(a, b) \
42 + TORCH_CHECK(a == b, "CHECK_EQ(" #a ", " #b ") failed. ", a, " vs ", b)
43 +
44 +//====== bgmv ======
45 +
46 +template <typename in_T, typename out_T, typename W_T>
47 +inline bool launch_bgmv_kernel(out_T *Y, const in_T *X, const W_T *W,
48 + const int64_t *lora_indices,
49 + uint16_t in_features, uint16_t out_features,
50 + int64_t y_offset, int64_t full_y_size,
51 + int64_t batch_size, int64_t num_layers,
52 + int64_t layer_idx, float scale) {
53 + switch (pack_u16(in_features, out_features)) {
54 +#define CASE_ONESIDE(_in_T, _out_T, _W_T, feat_in, feat_out) \
55 + case pack_u16(feat_in, feat_out): \
56 + bgmv_kernel<feat_in, feat_out>(Y, X, W, lora_indices, y_offset, \
57 + full_y_size, batch_size, num_layers, \
58 + layer_idx, scale); \
59 + break;
60 +#define CASE(_in_T, _out_T, _W_T, narrow, wide) \
61 + CASE_ONESIDE(in_T, out_T, W_T, narrow, wide) \
62 + CASE_ONESIDE(in_T, out_T, W_T, wide, narrow)
63 +
64 + FOR_BGMV_WIDE_NARROW(CASE, _, _, _)
65 +#undef CASE
66 +#undef CASE_ONESIDE
67 + default:
68 + return false;
69 + }
70 +
71 + return true;
72 +}
73 +
74 +void dispatch_bgmv(torch::Tensor y, torch::Tensor x, torch::Tensor w,
75 + torch::Tensor indicies, int64_t layer_idx, float scale) {
76 + CHECK_INPUT(y);
77 + CHECK_INPUT(x);
78 + CHECK_INPUT(w);
79 + CHECK_INPUT(indicies);
80 +
81 + CHECK_DIM(2, y);
82 + CHECK_DIM(2, x);
83 + CHECK_DIM(4, w);
84 + CHECK_DIM(1, indicies);
85 +
86 + int64_t B = x.size(0);
87 + int64_t h_in = x.size(1);
88 + int64_t h_out = y.size(1);
89 + int64_t num_layers = w.size(1);
90 + CHECK_EQ(w.size(3), h_in);
91 + CHECK_EQ(w.size(2), h_out);
92 + CHECK_EQ(indicies.size(0), x.size(0));
93 + CHECK_EQ(y.size(0), x.size(0));
94 + bool ok = false;
95 + if (h_in < 65536 && h_out < 65536) {
96 + // TODO: See if we can get rid of this massive nested switch
97 + switch (x.scalar_type()) {
98 + case at::ScalarType::Half:
99 + switch (y.scalar_type()) {
100 + case at::ScalarType::Half:
101 + switch (w.scalar_type()) {
102 + case at::ScalarType::Half:
103 + ok = launch_bgmv_kernel(static_cast<nv_half *>(y.data_ptr()),
104 + static_cast<nv_half *>(x.data_ptr()),
105 + static_cast<nv_half *>(w.data_ptr()),
106 + indicies.data_ptr<int64_t>(), h_in, h_out, 0,
107 + h_out, B, num_layers, layer_idx, scale);
108 + break;
109 + case at::ScalarType::BFloat16:
110 + ok = launch_bgmv_kernel(static_cast<nv_half *>(y.data_ptr()),
111 + static_cast<nv_half *>(x.data_ptr()),
112 + static_cast<nv_bfloat16 *>(w.data_ptr()),
113 + indicies.data_ptr<int64_t>(), h_in, h_out, 0,
114 + h_out, B, num_layers, layer_idx, scale);
115 + break;
116 + default:
117 + break;
118 + }
119 + break;
120 + case at::ScalarType::BFloat16:
121 + switch (w.scalar_type()) {
122 + case at::ScalarType::Half:
123 + ok = launch_bgmv_kernel(static_cast<nv_bfloat16 *>(y.data_ptr()),
124 + static_cast<nv_half *>(x.data_ptr()),
125 + static_cast<nv_half *>(w.data_ptr()),
126 + indicies.data_ptr<int64_t>(), h_in, h_out, 0,
127 + h_out, B, num_layers, layer_idx, scale);
128 + break;
129 + case at::ScalarType::BFloat16:
130 + ok = launch_bgmv_kernel(static_cast<nv_bfloat16 *>(y.data_ptr()),
131 + static_cast<nv_half *>(x.data_ptr()),
132 + static_cast<nv_bfloat16 *>(w.data_ptr()),
133 + indicies.data_ptr<int64_t>(), h_in, h_out, 0,
134 + h_out, B, num_layers, layer_idx, scale);
135 + break;
136 + default:
137 + break;
138 + }
139 + break;
140 + case at::ScalarType::Float:
141 + switch (w.scalar_type()) {
142 + case at::ScalarType::Half:
143 + ok = launch_bgmv_kernel(static_cast<float *>(y.data_ptr()),
144 + static_cast<nv_half *>(x.data_ptr()),
145 + static_cast<nv_half *>(w.data_ptr()),
146 + indicies.data_ptr<int64_t>(), h_in, h_out, 0,
147 + h_out, B, num_layers, layer_idx, scale);
148 + break;
149 + case at::ScalarType::BFloat16:
150 + ok = launch_bgmv_kernel(static_cast<float *>(y.data_ptr()),
151 + static_cast<nv_half *>(x.data_ptr()),
152 + static_cast<nv_bfloat16 *>(w.data_ptr()),
153 + indicies.data_ptr<int64_t>(), h_in, h_out, 0,
154 + h_out, B, num_layers, layer_idx, scale);
155 + break;
156 + default:
157 + break;
158 + }
159 + break;
160 + default:
161 + break;
162 + }
163 + break;
164 + case at::ScalarType::BFloat16:
165 + switch (y.scalar_type()) {
166 + case at::ScalarType::Half:
167 + switch (w.scalar_type()) {
168 + case at::ScalarType::Half:
169 + ok = launch_bgmv_kernel(static_cast<nv_half *>(y.data_ptr()),
170 + static_cast<nv_bfloat16 *>(x.data_ptr()),
171 + static_cast<nv_half *>(w.data_ptr()),
172 + indicies.data_ptr<int64_t>(), h_in, h_out, 0,
173 + h_out, B, num_layers, layer_idx, scale);
174 + break;
175 + case at::ScalarType::BFloat16:
176 + ok = launch_bgmv_kernel(static_cast<nv_half *>(y.data_ptr()),
177 + static_cast<nv_bfloat16 *>(x.data_ptr()),
178 + static_cast<nv_bfloat16 *>(w.data_ptr()),
179 + indicies.data_ptr<int64_t>(), h_in, h_out, 0,
180 + h_out, B, num_layers, layer_idx, scale);
181 + break;
182 + default:
183 + break;
184 + }
185 + break;
186 + case at::ScalarType::BFloat16:
187 + switch (w.scalar_type()) {
188 + case at::ScalarType::Half:
189 + ok = launch_bgmv_kernel(static_cast<nv_bfloat16 *>(y.data_ptr()),
190 + static_cast<nv_bfloat16 *>(x.data_ptr()),
191 + static_cast<nv_half *>(w.data_ptr()),
192 + indicies.data_ptr<int64_t>(), h_in, h_out, 0,
193 + h_out, B, num_layers, layer_idx, scale);
194 + break;
195 + case at::ScalarType::BFloat16:
196 + ok = launch_bgmv_kernel(static_cast<nv_bfloat16 *>(y.data_ptr()),
197 + static_cast<nv_bfloat16 *>(x.data_ptr()),
198 + static_cast<nv_bfloat16 *>(w.data_ptr()),
199 + indicies.data_ptr<int64_t>(), h_in, h_out, 0,
200 + h_out, B, num_layers, layer_idx, scale);
201 + break;
202 + default:
203 + break;
204 + }
205 + break;
206 + case at::ScalarType::Float:
207 + switch (w.scalar_type()) {
208 + case at::ScalarType::Half:
209 + ok = launch_bgmv_kernel(static_cast<float *>(y.data_ptr()),
210 + static_cast<nv_bfloat16 *>(x.data_ptr()),
211 + static_cast<nv_half *>(w.data_ptr()),
212 + indicies.data_ptr<int64_t>(), h_in, h_out, 0,
213 + h_out, B, num_layers, layer_idx, scale);
214 + break;
215 + case at::ScalarType::BFloat16:
216 + ok = launch_bgmv_kernel(static_cast<float *>(y.data_ptr()),
217 + static_cast<nv_bfloat16 *>(x.data_ptr()),
218 + static_cast<nv_bfloat16 *>(w.data_ptr()),
219 + indicies.data_ptr<int64_t>(), h_in, h_out, 0,
220 + h_out, B, num_layers, layer_idx, scale);
221 + break;
222 + default:
223 + break;
224 + }
225 + break;
226 + default:
227 + break;
228 + }
229 + break;
230 + case at::ScalarType::Float:
231 + switch (y.scalar_type()) {
232 + case at::ScalarType::Half:
233 + switch (w.scalar_type()) {
234 + case at::ScalarType::Half:
235 + ok = launch_bgmv_kernel(static_cast<nv_half *>(y.data_ptr()),
236 + static_cast<float *>(x.data_ptr()),
237 + static_cast<nv_half *>(w.data_ptr()),
238 + indicies.data_ptr<int64_t>(), h_in, h_out, 0,
239 + h_out, B, num_layers, layer_idx, scale);
240 + break;
241 + case at::ScalarType::BFloat16:
242 + ok = launch_bgmv_kernel(static_cast<nv_half *>(y.data_ptr()),
243 + static_cast<float *>(x.data_ptr()),
244 + static_cast<nv_bfloat16 *>(w.data_ptr()),
245 + indicies.data_ptr<int64_t>(), h_in, h_out, 0,
246 + h_out, B, num_layers, layer_idx, scale);
247 + break;
248 + default:
249 + break;
250 + }
251 + break;
252 + case at::ScalarType::BFloat16:
253 + switch (w.scalar_type()) {
254 + case at::ScalarType::Half:
255 + ok = launch_bgmv_kernel(static_cast<nv_bfloat16 *>(y.data_ptr()),
256 + static_cast<float *>(x.data_ptr()),
257 + static_cast<nv_half *>(w.data_ptr()),
258 + indicies.data_ptr<int64_t>(), h_in, h_out, 0,
259 + h_out, B, num_layers, layer_idx, scale);
260 + break;
261 + case at::ScalarType::BFloat16:
262 + ok = launch_bgmv_kernel(static_cast<nv_bfloat16 *>(y.data_ptr()),
263 + static_cast<float *>(x.data_ptr()),
264 + static_cast<nv_bfloat16 *>(w.data_ptr()),
265 + indicies.data_ptr<int64_t>(), h_in, h_out, 0,
266 + h_out, B, num_layers, layer_idx, scale);
267 + break;
268 + default:
269 + break;
270 + }
271 + break;
272 + case at::ScalarType::Float:
273 + switch (w.scalar_type()) {
274 + case at::ScalarType::Half:
275 + ok = launch_bgmv_kernel(static_cast<float *>(y.data_ptr()),
276 + static_cast<float *>(x.data_ptr()),
277 + static_cast<nv_half *>(w.data_ptr()),
278 + indicies.data_ptr<int64_t>(), h_in, h_out, 0,
279 + h_out, B, num_layers, layer_idx, scale);
280 + break;
281 + case at::ScalarType::BFloat16:
282 + ok = launch_bgmv_kernel(static_cast<float *>(y.data_ptr()),
283 + static_cast<float *>(x.data_ptr()),
284 + static_cast<nv_bfloat16 *>(w.data_ptr()),
285 + indicies.data_ptr<int64_t>(), h_in, h_out, 0,
286 + h_out, B, num_layers, layer_idx, scale);
287 + break;
288 + default:
289 + break;
290 + }
291 + break;
292 + default:
293 + break;
294 + }
295 + break;
296 + default:
297 + break;
298 + }
299 + }
300 + TORCH_CHECK(ok, "No suitable kernel.", " h_in=", h_in, " h_out=", h_out,
301 + " dtype=", x.scalar_type(), " out_dtype=", y.scalar_type());
302 +}
303 +
304 +void dispatch_bgmv_low_level(torch::Tensor y, torch::Tensor x, torch::Tensor w,
305 + torch::Tensor indicies, int64_t layer_idx,
306 + float scale, int64_t h_in, int64_t h_out,
307 + int64_t y_offset) {
308 + CHECK_INPUT(y);
309 + CHECK_INPUT(x);
310 + CHECK_INPUT(w);
311 + CHECK_INPUT(indicies);
312 +
313 + CHECK_DIM(2, y);
314 + CHECK_DIM(2, x);
315 + CHECK_DIM(4, w);
316 + CHECK_DIM(1, indicies);
317 +
318 + int64_t B = x.size(0);
319 + int64_t num_layers = w.size(1);
320 + int64_t full_y_size = y.size(1);
321 + CHECK_EQ(w.size(3), h_in);
322 + CHECK_EQ(w.size(2), h_out);
323 + CHECK_EQ(indicies.size(0), x.size(0));
324 + CHECK_EQ(y.size(0), x.size(0));
325 + bool ok = false;
326 + if (h_in < 65536 && h_out < 65536) {
327 + // TODO: See if we can get rid of this massive nested switch
328 + switch (x.scalar_type()) {
329 + case at::ScalarType::Half:
330 + switch (y.scalar_type()) {
331 + case at::ScalarType::Half:
332 + switch (w.scalar_type()) {
333 + case at::ScalarType::Half:
334 + ok = launch_bgmv_kernel(static_cast<nv_half *>(y.data_ptr()),
335 + static_cast<nv_half *>(x.data_ptr()),
336 + static_cast<nv_half *>(w.data_ptr()),
337 + indicies.data_ptr<int64_t>(), h_in, h_out,
338 + y_offset, full_y_size, B, num_layers,
339 + layer_idx, scale);
340 + break;
341 + case at::ScalarType::BFloat16:
342 + ok = launch_bgmv_kernel(static_cast<nv_half *>(y.data_ptr()),
343 + static_cast<nv_half *>(x.data_ptr()),
344 + static_cast<nv_bfloat16 *>(w.data_ptr()),
345 + indicies.data_ptr<int64_t>(), h_in, h_out,
346 + y_offset, full_y_size, B, num_layers,
347 + layer_idx, scale);
348 + break;
349 + default:
350 + break;
351 + }
352 + break;
353 + case at::ScalarType::BFloat16:
354 + switch (w.scalar_type()) {
355 + case at::ScalarType::Half:
356 + ok = launch_bgmv_kernel(static_cast<nv_bfloat16 *>(y.data_ptr()),
357 + static_cast<nv_half *>(x.data_ptr()),
358 + static_cast<nv_half *>(w.data_ptr()),
359 + indicies.data_ptr<int64_t>(), h_in, h_out,
360 + y_offset, full_y_size, B, num_layers,
361 + layer_idx, scale);
362 + break;
363 + case at::ScalarType::BFloat16:
364 + ok = launch_bgmv_kernel(static_cast<nv_bfloat16 *>(y.data_ptr()),
365 + static_cast<nv_half *>(x.data_ptr()),
366 + static_cast<nv_bfloat16 *>(w.data_ptr()),
367 + indicies.data_ptr<int64_t>(), h_in, h_out,
368 + y_offset, full_y_size, B, num_layers,
369 + layer_idx, scale);
370 + break;
371 + default:
372 + break;
373 + }
374 + break;
375 + case at::ScalarType::Float:
376 + switch (w.scalar_type()) {
377 + case at::ScalarType::Half:
378 + ok = launch_bgmv_kernel(static_cast<float *>(y.data_ptr()),
379 + static_cast<nv_half *>(x.data_ptr()),
380 + static_cast<nv_half *>(w.data_ptr()),
381 + indicies.data_ptr<int64_t>(), h_in, h_out,
382 + y_offset, full_y_size, B, num_layers,
383 + layer_idx, scale);
384 + break;
385 + case at::ScalarType::BFloat16:
386 + ok = launch_bgmv_kernel(static_cast<float *>(y.data_ptr()),
387 + static_cast<nv_half *>(x.data_ptr()),
388 + static_cast<nv_bfloat16 *>(w.data_ptr()),
389 + indicies.data_ptr<int64_t>(), h_in, h_out,
390 + y_offset, full_y_size, B, num_layers,
391 + layer_idx, scale);
392 + break;
393 + default:
394 + break;
395 + }
396 + break;
397 + default:
398 + break;
399 + }
400 + break;
401 + case at::ScalarType::BFloat16:
402 + switch (y.scalar_type()) {
403 + case at::ScalarType::Half:
404 + switch (w.scalar_type()) {
405 + case at::ScalarType::Half:
406 + ok = launch_bgmv_kernel(static_cast<nv_half *>(y.data_ptr()),
407 + static_cast<nv_bfloat16 *>(x.data_ptr()),
408 + static_cast<nv_half *>(w.data_ptr()),
409 + indicies.data_ptr<int64_t>(), h_in, h_out,
410 + y_offset, full_y_size, B, num_layers,
411 + layer_idx, scale);
412 + break;
413 + case at::ScalarType::BFloat16:
414 + ok = launch_bgmv_kernel(static_cast<nv_half *>(y.data_ptr()),
415 + static_cast<nv_bfloat16 *>(x.data_ptr()),
416 + static_cast<nv_bfloat16 *>(w.data_ptr()),
417 + indicies.data_ptr<int64_t>(), h_in, h_out,
418 + y_offset, full_y_size, B, num_layers,
419 + layer_idx, scale);
420 + break;
421 + default:
422 + break;
423 + }
424 + break;
425 + case at::ScalarType::BFloat16:
426 + switch (w.scalar_type()) {
427 + case at::ScalarType::Half:
428 + ok = launch_bgmv_kernel(static_cast<nv_bfloat16 *>(y.data_ptr()),
429 + static_cast<nv_bfloat16 *>(x.data_ptr()),
430 + static_cast<nv_half *>(w.data_ptr()),
431 + indicies.data_ptr<int64_t>(), h_in, h_out,
432 + y_offset, full_y_size, B, num_layers,
433 + layer_idx, scale);
434 + break;
435 + case at::ScalarType::BFloat16:
436 + ok = launch_bgmv_kernel(static_cast<nv_bfloat16 *>(y.data_ptr()),
437 + static_cast<nv_bfloat16 *>(x.data_ptr()),
438 + static_cast<nv_bfloat16 *>(w.data_ptr()),
439 + indicies.data_ptr<int64_t>(), h_in, h_out,
440 + y_offset, full_y_size, B, num_layers,
441 + layer_idx, scale);
442 + break;
443 + default:
444 + break;
445 + }
446 + break;
447 + case at::ScalarType::Float:
448 + switch (w.scalar_type()) {
449 + case at::ScalarType::Half:
450 + ok = launch_bgmv_kernel(static_cast<float *>(y.data_ptr()),
451 + static_cast<nv_bfloat16 *>(x.data_ptr()),
452 + static_cast<nv_half *>(w.data_ptr()),
453 + indicies.data_ptr<int64_t>(), h_in, h_out,
454 + y_offset, full_y_size, B, num_layers,
455 + layer_idx, scale);
456 + break;
457 + case at::ScalarType::BFloat16:
458 + ok = launch_bgmv_kernel(static_cast<float *>(y.data_ptr()),
459 + static_cast<nv_bfloat16 *>(x.data_ptr()),
460 + static_cast<nv_bfloat16 *>(w.data_ptr()),
461 + indicies.data_ptr<int64_t>(), h_in, h_out,
462 + y_offset, full_y_size, B, num_layers,
463 + layer_idx, scale);
464 + break;
465 + default:
466 + break;
467 + }
468 + break;
469 + default:
470 + break;
471 + }
472 + break;
473 + case at::ScalarType::Float:
474 + switch (y.scalar_type()) {
475 + case at::ScalarType::Half:
476 + switch (w.scalar_type()) {
477 + case at::ScalarType::Half:
478 + ok = launch_bgmv_kernel(static_cast<nv_half *>(y.data_ptr()),
479 + static_cast<float *>(x.data_ptr()),
480 + static_cast<nv_half *>(w.data_ptr()),
481 + indicies.data_ptr<int64_t>(), h_in, h_out,
482 + y_offset, full_y_size, B, num_layers,
483 + layer_idx, scale);
484 + break;
485 + case at::ScalarType::BFloat16:
486 + ok = launch_bgmv_kernel(static_cast<nv_half *>(y.data_ptr()),
487 + static_cast<float *>(x.data_ptr()),
488 + static_cast<nv_bfloat16 *>(w.data_ptr()),
489 + indicies.data_ptr<int64_t>(), h_in, h_out,
490 + y_offset, full_y_size, B, num_layers,
491 + layer_idx, scale);
492 + break;
493 + default:
494 + break;
495 + }
496 + break;
497 + case at::ScalarType::BFloat16:
498 + switch (w.scalar_type()) {
499 + case at::ScalarType::Half:
500 + ok = launch_bgmv_kernel(static_cast<nv_bfloat16 *>(y.data_ptr()),
501 + static_cast<float *>(x.data_ptr()),
502 + static_cast<nv_half *>(w.data_ptr()),
503 + indicies.data_ptr<int64_t>(), h_in, h_out,
504 + y_offset, full_y_size, B, num_layers,
505 + layer_idx, scale);
506 + break;
507 + case at::ScalarType::BFloat16:
508 + ok = launch_bgmv_kernel(static_cast<nv_bfloat16 *>(y.data_ptr()),
509 + static_cast<float *>(x.data_ptr()),
510 + static_cast<nv_bfloat16 *>(w.data_ptr()),
511 + indicies.data_ptr<int64_t>(), h_in, h_out,
512 + y_offset, full_y_size, B, num_layers,
513 + layer_idx, scale);
514 + break;
515 + default:
516 + break;
517 + }
518 + break;
519 + case at::ScalarType::Float:
520 + switch (w.scalar_type()) {
521 + case at::ScalarType::Half:
522 + ok = launch_bgmv_kernel(static_cast<float *>(y.data_ptr()),
523 + static_cast<float *>(x.data_ptr()),
524 + static_cast<nv_half *>(w.data_ptr()),
525 + indicies.data_ptr<int64_t>(), h_in, h_out,
526 + y_offset, full_y_size, B, num_layers,
527 + layer_idx, scale);
528 + break;
529 + case at::ScalarType::BFloat16:
530 + ok = launch_bgmv_kernel(static_cast<float *>(y.data_ptr()),
531 + static_cast<float *>(x.data_ptr()),
532 + static_cast<nv_bfloat16 *>(w.data_ptr()),
533 + indicies.data_ptr<int64_t>(), h_in, h_out,
534 + y_offset, full_y_size, B, num_layers,
535 + layer_idx, scale);
536 + break;
537 + default:
538 + break;
539 + }
540 + break;
541 + default:
542 + break;
543 + }
544 + break;
545 + default:
546 + break;
547 + }
548 + }
549 + TORCH_CHECK(ok, "No suitable kernel.", " h_in=", h_in, " h_out=", h_out,
550 + " dtype=", x.scalar_type(), " out_dtype=", y.scalar_type());
551 +}
552 +
553 +} // namespace
554 +
555 +//====== pybind ======
556 +
557 +#define DEFINE_pybind(name) m.def(#name, &name, #name);
558 +
559 +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
560 + m.def("dispatch_bgmv", &dispatch_bgmv, "dispatch_bgmv");
561 + m.def("dispatch_bgmv_low_level", &dispatch_bgmv_low_level,
562 + "dispatch_bgmv_low_level");
563 +}
examples/multilora_inference.py new
+117
@@ -0,0 +1,117 @@
1 +"""
2 +This example shows how to use the multi-LoRA functionality for offline inference.
3 +
4 +Requires HuggingFace credentials for access to Llama2.
5 +"""
6 +
7 +from typing import Optional, List, Tuple
8 +
9 +from huggingface_hub import snapshot_download
10 +
11 +from vllm import EngineArgs, LLMEngine, SamplingParams, RequestOutput
12 +from vllm.lora.request import LoRARequest
13 +
14 +
15 +def create_test_prompts(lora_path: str) -> List[Tuple[str, SamplingParams]]:
16 + """Create a list of test prompts with their sampling parameters.
17 +
18 + 2 requests for base model, 4 requests for the LoRA. We define 2
19 + different LoRA adapters (using the same model for demo purposes).
20 + Since we also set `max_loras=1`, the expectation is that the requests
21 + with the second LoRA adapter will be ran after all requests with the
22 + first adapter have finished.
23 + """
24 + return [
25 + ("A robot may not injure a human being",
26 + SamplingParams(temperature=0.0,
27 + logprobs=1,
28 + prompt_logprobs=1,
29 + max_tokens=128), None),
30 + ("To be or not to be,",
31 + SamplingParams(temperature=0.8,
32 + top_k=5,
33 + presence_penalty=0.2,
34 + max_tokens=128), None),
35 + ("[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_74 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]",
36 + SamplingParams(temperature=0.0,
37 + logprobs=1,
38 + prompt_logprobs=1,
39 + max_tokens=128,
40 + stop_token_ids=[32003]),
41 + LoRARequest("sql-lora", 1, lora_path)),
42 + ("[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_11 (nationality VARCHAR, elector VARCHAR)\n\n question: When Anchero Pantaleone was the elector what is under nationality? [/user] [assistant]",
43 + SamplingParams(n=3,
44 + best_of=3,
45 + use_beam_search=True,
46 + temperature=0,
47 + max_tokens=128,
48 + stop_token_ids=[32003]),
49 + LoRARequest("sql-lora", 1, lora_path)),
50 + ("[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_74 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]",
51 + SamplingParams(temperature=0.0,
52 + logprobs=1,
53 + prompt_logprobs=1,
54 + max_tokens=128,
55 + stop_token_ids=[32003]),
56 + LoRARequest("sql-lora2", 2, lora_path)),
57 + ("[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_11 (nationality VARCHAR, elector VARCHAR)\n\n question: When Anchero Pantaleone was the elector what is under nationality? [/user] [assistant]",
58 + SamplingParams(n=3,
59 + best_of=3,
60 + use_beam_search=True,
61 + temperature=0,
62 + max_tokens=128,
63 + stop_token_ids=[32003]),
64 + LoRARequest("sql-lora", 1, lora_path)),
65 + ]
66 +
67 +
68 +def process_requests(engine: LLMEngine,
69 + test_prompts: List[Tuple[str, SamplingParams,
70 + Optional[LoRARequest]]]):
71 + """Continuously process a list of prompts and handle the outputs."""
72 + request_id = 0
73 +
74 + while test_prompts or engine.has_unfinished_requests():
75 + if test_prompts:
76 + prompt, sampling_params, lora_request = test_prompts.pop(0)
77 + engine.add_request(str(request_id),
78 + prompt,
79 + sampling_params,
80 + lora_request=lora_request)
81 + request_id += 1
82 +
83 + request_outputs: List[RequestOutput] = engine.step()
84 +
85 + for request_output in request_outputs:
86 + if request_output.finished:
87 + print(request_output)
88 +
89 +
90 +def initialize_engine() -> LLMEngine:
91 + """Initialize the LLMEngine."""
92 + # max_loras: controls the number of LoRAs that can be used in the same
93 + # batch. Larger numbers will cause higher memory usage, as each LoRA
94 + # slot requires its own preallocated tensor.
95 + # max_lora_rank: controls the maximum supported rank of all LoRAs. Larger
96 + # numbers will cause higher memory usage. If you know that all LoRAs will
97 + # use the same rank, it is recommended to set this as low as possible.
98 + # max_cpu_loras: controls the size of the CPU LoRA cache.
99 + engine_args = EngineArgs(model="meta-llama/Llama-2-7b-hf",
100 + enable_lora=True,
101 + max_loras=1,
102 + max_lora_rank=8,
103 + max_cpu_loras=2,
104 + max_num_seqs=256)
105 + return LLMEngine.from_engine_args(engine_args)
106 +
107 +
108 +def main():
109 + """Main function that sets up and runs the prompt processing."""
110 + engine = initialize_engine()
111 + lora_path = snapshot_download(repo_id="yard1/llama-2-7b-sql-lora-test")
112 + test_prompts = create_test_prompts(lora_path)
113 + process_requests(engine, test_prompts)
114 +
115 +
116 +if __name__ == '__main__':
117 + main()
setup.py
+54 -5
@@ -1,13 +1,16 @@
1 +import contextlib
2 import io
3 import os
4 import re
5 import subprocess
5 -from typing import List, Set
6 import warnings
7 +from pathlib import Path
8 +from typing import List, Set
9
10 from packaging.version import parse, Version
11 import setuptools
12 import torch
13 +import torch.utils.cpp_extension as torch_cpp_ext
14 from torch.utils.cpp_extension import BuildExtension, CUDAExtension, CUDA_HOME, ROCM_HOME
15
16 ROOT_DIR = os.path.dirname(__file__)
@@ -28,7 +31,7 @@ def _is_neuron() -> bool:
31 torch_neuronx_installed = True
32 try:
33 subprocess.run(["neuron-ls"], capture_output=True, check=True)
31 - except FileNotFoundError as e:
34 + except FileNotFoundError:
35 torch_neuronx_installed = False
36 return torch_neuronx_installed
37
@@ -96,10 +99,16 @@ def get_hipcc_rocm_version():
99 return None
100
101
102 +def glob(pattern: str):
103 + root = Path(__name__).parent
104 + return [str(p) for p in root.glob(pattern)]
105 +
106 +
107 def get_neuronxcc_version():
108 import sysconfig
109 site_dir = sysconfig.get_paths()["purelib"]
102 - version_file = os.path.join(site_dir, "neuronxcc", "version", "__init__.py")
110 + version_file = os.path.join(site_dir, "neuronxcc", "version",
111 + "__init__.py")
112
113 # Check if the command was executed successfully
114 with open(version_file, "rt") as fp:
@@ -178,6 +187,8 @@ if _is_cuda() and not compute_capabilities:
187 "GPUs with compute capability below 7.0 are not supported.")
188 compute_capabilities.add(f"{major}.{minor}")
189
190 +ext_modules = []
191 +
192 if _is_cuda():
193 nvcc_cuda_version = get_nvcc_cuda_version(CUDA_HOME)
194 if not compute_capabilities:
@@ -215,6 +226,8 @@ if _is_cuda():
226 raise RuntimeError(
227 "CUDA 11.8 or higher is required for compute capability 9.0.")
228
229 + NVCC_FLAGS_PUNICA = NVCC_FLAGS.copy()
230 +
231 # Add target compute capabilities to NVCC flags.
232 for capability in compute_capabilities:
233 num = capability[0] + capability[2]
@@ -223,6 +236,14 @@ if _is_cuda():
236 NVCC_FLAGS += [
237 "-gencode", f"arch=compute_{num},code=compute_{num}"
238 ]
239 + if int(capability[0]) >= 8:
240 + NVCC_FLAGS_PUNICA += [
241 + "-gencode", f"arch=compute_{num},code=sm_{num}"
242 + ]
243 + if capability.endswith("+PTX"):
244 + NVCC_FLAGS_PUNICA += [
245 + "-gencode", f"arch=compute_{num},code=compute_{num}"
246 + ]
247
248 # Use NVCC threads to parallelize the build.
249 if nvcc_cuda_version >= Version("11.2"):
@@ -230,6 +251,36 @@ if _is_cuda():
251 num_threads = min(os.cpu_count(), nvcc_threads)
252 NVCC_FLAGS += ["--threads", str(num_threads)]
253
254 + # changes for punica kernels
255 + NVCC_FLAGS += torch_cpp_ext.COMMON_NVCC_FLAGS
256 + REMOVE_NVCC_FLAGS = [
257 + '-D__CUDA_NO_HALF_OPERATORS__',
258 + '-D__CUDA_NO_HALF_CONVERSIONS__',
259 + '-D__CUDA_NO_BFLOAT16_CONVERSIONS__',
260 + '-D__CUDA_NO_HALF2_OPERATORS__',
261 + ]
262 + for flag in REMOVE_NVCC_FLAGS:
263 + with contextlib.suppress(ValueError):
264 + torch_cpp_ext.COMMON_NVCC_FLAGS.remove(flag)
265 +
266 + install_punica = bool(int(os.getenv("VLLM_INSTALL_PUNICA_KERNELS", "1")))
267 + device_count = torch.cuda.device_count()
268 + for i in range(device_count):
269 + major, minor = torch.cuda.get_device_capability(i)
270 + if major < 8:
271 + install_punica = False
272 + break
273 + if install_punica:
274 + ext_modules.append(
275 + CUDAExtension(
276 + name="vllm._punica_C",
277 + sources=["csrc/punica/punica_ops.cc"] +
278 + glob("csrc/punica/bgmv/*.cu"),
279 + extra_compile_args={
280 + "cxx": CXX_FLAGS,
281 + "nvcc": NVCC_FLAGS_PUNICA,
282 + },
283 + ))
284 elif _is_hip():
285 amd_arch = get_amdgpu_offload_arch()
286 if amd_arch not in ROCM_SUPPORTED_ARCHS:
@@ -240,8 +291,6 @@ elif _is_hip():
291 elif _is_neuron():
292 neuronxcc_version = get_neuronxcc_version()
293
243 -ext_modules = []
244 -
294 vllm_extension_sources = [
295 "csrc/cache_kernels.cu",
296 "csrc/attention/attention_kernels.cu",
tests/async_engine/test_async_llm_engine.py
+11
@@ -25,6 +25,13 @@ class MockEngine:
25 return [RequestOutput(
26 request_id=self.request_id)] if self.request_id else []
27
28 + async def encode_request_async(
29 + self,
30 + *args,
31 + **kwargs,
32 + ):
33 + return [1]
34 +
35 def generate(self, request_id):
36 self.request_id = request_id
37
@@ -35,6 +42,10 @@ class MockEngine:
42 del kwargs # Unused
43 self.add_request_calls += 1
44
45 + async def add_request_async(self, **kwargs):
46 + del kwargs # Unused
47 + self.add_request_calls += 1
48 +
49 def abort_request(self, request_id):
50 del request_id # Unused
51 self.abort_request_calls += 1
tests/lora/__init__.py
tests/lora/conftest.py new
+143
@@ -0,0 +1,143 @@
1 +import contextlib
2 +import gc
3 +import tempfile
4 +from collections import OrderedDict
5 +from unittest.mock import patch, MagicMock
6 +
7 +import pytest
8 +import ray
9 +import torch
10 +import torch.nn as nn
11 +from huggingface_hub import snapshot_download
12 +
13 +import vllm
14 +from vllm.config import LoRAConfig
15 +from vllm.model_executor.layers.sampler import Sampler
16 +from vllm.model_executor.model_loader import get_model
17 +from vllm.model_executor.layers.linear import (ColumnParallelLinear,
18 + MergedColumnParallelLinear,
19 + RowParallelLinear)
20 +from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
21 +from vllm.model_executor.parallel_utils.parallel_state import (
22 + destroy_model_parallel, initialize_model_parallel)
23 +
24 +
25 +def cleanup():
26 + destroy_model_parallel()
27 + with contextlib.suppress(AssertionError):
28 + torch.distributed.destroy_process_group()
29 + gc.collect()
30 + torch.cuda.empty_cache()
31 + ray.shutdown()
32 +
33 +
34 +@pytest.fixture(autouse=True)
35 +def cleanup_fixture():
36 + yield
37 + cleanup()
38 +
39 +
40 +@pytest.fixture
41 +def dist_init():
42 + if not torch.distributed.is_initialized():
43 + temp_file = tempfile.mkstemp()[1]
44 + torch.distributed.init_process_group(
45 + backend="nccl",
46 + world_size=1,
47 + rank=0,
48 + init_method=f"file://{temp_file}",
49 + )
50 + torch.distributed.all_reduce(torch.zeros(1).cuda())
51 + initialize_model_parallel(1, 1)
52 + yield
53 + cleanup()
54 +
55 +
56 +@pytest.fixture
57 +def dist_init_torch_only():
58 + if torch.distributed.is_initialized():
59 + return
60 + temp_file = tempfile.mkstemp()[1]
61 + torch.distributed.init_process_group(
62 + backend="nccl",
63 + world_size=1,
64 + rank=0,
65 + init_method=f"file://{temp_file}",
66 + )
67 +
68 +
69 +@pytest.fixture
70 +def dummy_model() -> nn.Module:
71 + model = nn.Sequential(
72 + OrderedDict([
73 + ("dense1", ColumnParallelLinear(764, 100)),
74 + ("dense2", RowParallelLinear(100, 50)),
75 + (
76 + "layer1",
77 + nn.Sequential(
78 + OrderedDict([
79 + ("dense1", ColumnParallelLinear(100, 10)),
80 + ("dense2", RowParallelLinear(10, 50)),
81 + ])),
82 + ),
83 + ("act2", nn.ReLU()),
84 + ("output", ColumnParallelLinear(50, 10)),
85 + ("outact", nn.Sigmoid()),
86 + # Special handling for lm_head & sampler
87 + ("lm_head", ParallelLMHead(512, 10)),
88 + ("sampler", Sampler(512))
89 + ]))
90 + model.config = MagicMock()
91 + return model
92 +
93 +
94 +@pytest.fixture
95 +def dummy_model_gate_up() -> nn.Module:
96 + model = nn.Sequential(
97 + OrderedDict([
98 + ("dense1", ColumnParallelLinear(764, 100)),
99 + ("dense2", RowParallelLinear(100, 50)),
100 + (
101 + "layer1",
102 + nn.Sequential(
103 + OrderedDict([
104 + ("dense1", ColumnParallelLinear(100, 10)),
105 + ("dense2", RowParallelLinear(10, 50)),
106 + ])),
107 + ),
108 + ("act2", nn.ReLU()),
109 + ("gate_up_proj", MergedColumnParallelLinear(50, [5, 5])),
110 + ("outact", nn.Sigmoid()),
111 + # Special handling for lm_head & sampler
112 + ("lm_head", ParallelLMHead(512, 10)),
113 + ("sampler", Sampler(512))
114 + ]))
115 + model.config = MagicMock()
116 + return model
117 +
118 +
119 +@pytest.fixture(scope="session")
120 +def sql_lora_files():
121 + return snapshot_download(repo_id="yard1/llama-2-7b-sql-lora-test")
122 +
123 +
124 +@pytest.fixture
125 +def llama_2_7b_engine_extra_embeddings() -> nn.Module:
126 + cleanup()
127 + get_model_old = get_model
128 +
129 + def get_model_patched(model_config, lora_config=None):
130 + return get_model_old(model_config,
131 + LoRAConfig(max_loras=4, max_lora_rank=8))
132 +
133 + with patch("vllm.worker.model_runner.get_model", get_model_patched):
134 + engine = vllm.LLM("meta-llama/Llama-2-7b-hf", enable_lora=False)
135 + yield engine.llm_engine
136 + del engine
137 + cleanup()
138 +
139 +
140 +@pytest.fixture
141 +def llama_2_7b_model_extra_embeddings(
142 + llama_2_7b_engine_extra_embeddings) -> nn.Module:
143 + yield llama_2_7b_engine_extra_embeddings.driver_worker.model_runner.model
tests/lora/test_layers.py new
+709
@@ -0,0 +1,709 @@
1 +import pytest
2 +import random
3 +from copy import deepcopy
4 +from dataclasses import dataclass
5 +from typing import List, Optional, Dict, Tuple
6 +
7 +import torch
8 +import torch.nn.functional as F
9 +
10 +from vllm.lora.layers import (
11 + ColumnParallelLinearWithLoRA,
12 + MergedColumnParallelLinearWithLoRA,
13 + QKVParallelLinearWithLora,
14 + VocabParallelEmbeddingWithLoRA,
15 + RowParallelLinearWithLoRA,
16 + SamplerWithLoRA,
17 + LoRAMapping,
18 + BaseLayerWithLoRA,
19 +)
20 +from vllm.lora.models import LoRALayerWeights, convert_mapping, PackedLoRALayerWeights
21 +from vllm.config import LoRAConfig
22 +from vllm.model_executor.layers.sampler import Sampler
23 +from vllm.model_executor.layers.linear import (ColumnParallelLinear,
24 + MergedColumnParallelLinear,
25 + RowParallelLinear,
26 + QKVParallelLinear)
27 +from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding, ParallelLMHead
28 +from vllm.model_executor.utils import set_random_seed
29 +
30 +from .utils import DummyLoRAManager
31 +
32 +TOLERANCES = {
33 + torch.float16: (5e-3, 5e-3),
34 + torch.float32: (5e-3, 5e-3),
35 + torch.bfloat16: (3e-2, 2e-2),
36 +}
37 +
38 +
39 +def get_random_id_to_index(num_loras: int,
40 + num_slots: int,
41 + log: bool = True) -> List[Optional[int]]:
42 + """Creates a random lora_id_to_index mapping.
43 +
44 + Args:
45 + num_loras: The number of active loras in the mapping.
46 + num_slots: The number of slots in the mapping. Must be larger
47 + than num_loras.
48 + log: Whether to log the output.
49 + """
50 +
51 + if num_loras > num_slots:
52 + raise ValueError(
53 + f"num_loras is higher than num_slots: {num_loras} > {num_slots}. "
54 + "num_loras must be less than or equal to num_slots.")
55 +
56 + slots: List[Optional[int]] = [None] * num_slots
57 + random_slot_selections = (torch.randperm(num_slots)[:num_loras]).tolist()
58 + for lora_id, slot_idx in enumerate(random_slot_selections, start=1):
59 + slots[slot_idx] = lora_id
60 +
61 + if log:
62 + print(f"Created lora_id_to_index mapping: {slots}.")
63 +
64 + return slots
65 +
66 +
67 +def populate_loras(
68 + id_to_index: List[Optional[int]],
69 + layer: BaseLayerWithLoRA,
70 + layer_weights: torch.Tensor,
71 + generate_embeddings_tensor: int = 0,
72 + repeats: int = 1,
73 +) -> Tuple[Dict[int, LoRALayerWeights], Dict[int, List[LoRALayerWeights]]]:
74 + """This method populates the lora layers with lora weights.
75 +
76 + Args:
77 + id_to_index: a list of lora ids. The index of the lora id
78 + represents which memory slot the lora matrices are
79 + stored in. A None value indicates a free slot.
80 + layer: the LoRAlayer to populate.
81 + layer_weights: the PyTorch tensor containing the layer's
82 + weights.
83 + generate_embeddings_tensor: whether to generate an
84 + embeddings tensor for each LoRA.
85 + repeats: must only be set for column parallel packed
86 + layers. Indicates the number of loras to compose
87 + together to create a single lora layer.
88 + """
89 +
90 + # Dictionary that maps the lora ID to the
91 + # corresponding lora weights.
92 + lora_dict: Dict[int, LoRALayerWeights] = dict()
93 +
94 + # Dictionary that maps the lora ID to the
95 + # corresponding subloras. Only useful when
96 + # repeats > 1.
97 + sublora_dict: Dict[int, List[LoRALayerWeights]] = dict()
98 +
99 + for slot_idx, lora_id in enumerate(id_to_index):
100 + if lora_id is not None:
101 + subloras = []
102 + sublora_len = layer_weights.shape[0] // repeats
103 + for i in range(repeats):
104 + sublora = DummyLoRAManager().init_random_lora(
105 + module_name=f"fake_{i}",
106 + weight=layer_weights,
107 + generate_embeddings_tensor=generate_embeddings_tensor,
108 + )
109 + sublora.lora_b = sublora.lora_b[:, (sublora_len *
110 + i):(sublora_len * (i + 1))]
111 + sublora.optimize()
112 + subloras.append(sublora)
113 +
114 + lora = PackedLoRALayerWeights.pack(
115 + subloras) if repeats > 1 else subloras[0]
116 +
117 + layer.set_lora(
118 + slot_idx,
119 + lora_a=lora.lora_a,
120 + lora_b=lora.lora_b,
121 + embeddings_tensor=lora.embeddings_tensor,
122 + )
123 +
124 + lora_dict[lora_id] = lora
125 + sublora_dict[lora_id] = subloras
126 +
127 + return lora_dict, sublora_dict
128 +
129 +
130 +def create_random_inputs(
131 + active_lora_ids: List[int],
132 + num_inputs: int,
133 + input_size: Tuple[int, ...],
134 + input_range: Tuple[float, float],
135 + input_type: torch.dtype = torch.int,
136 +) -> Tuple[List[torch.Tensor], List[int], List[int]]:
137 + """Creates random inputs.
138 +
139 + Args:
140 + active_lora_ids: lora IDs of active lora weights.
141 + num_inputs: the number of inputs to create.
142 + input_size: the size of each individual input.
143 + input_range: the range of values to include in the input.
144 + input_range[0] <= possible input values < input_range[1]
145 + input_type: the type of values in the input.
146 + """
147 +
148 + low, high = input_range
149 +
150 + inputs, index_mapping, prompt_mapping = [], [], []
151 + for _ in range(num_inputs):
152 + if input_type == torch.int:
153 + inputs.append(
154 + torch.randint(low=int(low),
155 + high=int(high),
156 + size=input_size,
157 + device="cuda"))
158 + else:
159 + inputs.append(
160 + torch.rand(size=input_size, dtype=input_type, device="cuda") *
161 + high + low)
162 +
163 + lora_id = random.choice(active_lora_ids)
164 + index_mapping += [lora_id] * input_size[0]
165 + prompt_mapping += [lora_id]
166 +
167 + return inputs, index_mapping, prompt_mapping
168 +
169 +
170 +@torch.inference_mode()
171 +@pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
172 +def test_embeddings(dist_init, num_loras) -> None:
173 +
174 + max_loras = 8
175 + lora_config = LoRAConfig(max_loras=max_loras,
176 + max_lora_rank=8,
177 + lora_dtype=torch.float16)
178 +
179 + def create_random_embedding_layer():
180 + embedding = VocabParallelEmbedding(512, 256)
181 + embedding.weight.data = torch.rand_like(embedding.weight.data)
182 + embedding.weight.data[512:, :] = 0
183 + lora_embedding = VocabParallelEmbeddingWithLoRA(embedding)
184 + lora_embedding.create_lora_weights(max_loras, lora_config)
185 +
186 + return embedding, lora_embedding
187 +
188 + for i in range(10):
189 + set_random_seed(i)
190 +
191 + id_to_index = get_random_id_to_index(num_loras, max_loras)
192 + embedding, lora_embedding = create_random_embedding_layer()
193 +
194 + lora_dict, _ = populate_loras(
195 + id_to_index,
196 + layer=lora_embedding,
197 + layer_weights=embedding.weight.T,
198 + )
199 +
200 + inputs, index_mapping, prompt_mapping = create_random_inputs(
201 + active_lora_ids=list(lora_dict.keys()),
202 + num_inputs=num_loras * 3,
203 + input_size=(200, ),
204 + input_range=(1, 512),
205 + )
206 + lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
207 +
208 + mapping_info = convert_mapping(lora_mapping, id_to_index, max_loras,
209 + 512, lora_config.lora_extra_vocab_size)
210 + lora_embedding.set_mapping(*mapping_info)
211 +
212 + lora_result = lora_embedding(torch.cat(inputs))
213 +
214 + expected_results = []
215 + for input_, lora_id in zip(inputs, prompt_mapping):
216 + lora = lora_dict[lora_id]
217 + result = embedding(input_)
218 + after_a = F.embedding(
219 + input_,
220 + lora.lora_a,
221 + )
222 + result += (after_a @ lora.lora_b)
223 + expected_results.append(result)
224 + expected_result = torch.cat(expected_results)
225 +
226 + rtol, atol = TOLERANCES[lora_result.dtype]
227 + assert torch.allclose(lora_result,
228 + expected_result,
229 + rtol=rtol,
230 + atol=atol)
231 +
232 + # Check that resetting the lora weights succeeds
233 +
234 + for slot_idx in range(max_loras):
235 + lora_embedding.reset_lora(slot_idx)
236 +
237 + inputs, index_mapping, prompt_mapping = create_random_inputs(
238 + active_lora_ids=[0],
239 + num_inputs=num_loras * 3,
240 + input_size=(200, ),
241 + input_range=(1, 512),
242 + )
243 + lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
244 +
245 + mapping_info = convert_mapping(lora_mapping, id_to_index, max_loras,
246 + 512, lora_config.lora_extra_vocab_size)
247 + lora_embedding.set_mapping(*mapping_info, )
248 +
249 + lora_result = lora_embedding(torch.cat(inputs))
250 + expected_result = embedding(torch.cat(inputs))
251 +
252 + rtol, atol = TOLERANCES[lora_result.dtype]
253 + assert torch.allclose(lora_result,
254 + expected_result,
255 + rtol=rtol,
256 + atol=atol)
257 +
258 +
259 +@torch.inference_mode()
260 +# @pytest.mark.skip(reason="Fails when loras are in any slot other than the first.")
261 +@pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
262 +def test_embeddings_with_new_embeddings(dist_init, num_loras) -> None:
263 +
264 + max_loras = 8
265 + lora_config = LoRAConfig(max_loras=max_loras,
266 + max_lora_rank=8,
267 + lora_dtype=torch.float16)
268 +
269 + def create_random_embedding_layer():
270 + embedding = VocabParallelEmbedding(512, 256)
271 + embedding_data = torch.rand_like(embedding.weight.data)
272 + embedding.weight.data = embedding_data
273 + embedding.weight.data[512:, :] = 0
274 + expanded_embedding = VocabParallelEmbedding(
275 + 512 + lora_config.lora_extra_vocab_size * max_loras,
276 + 256,
277 + org_num_embeddings=512)
278 + expanded_embedding.weight.data[:512, :] = embedding_data
279 + # We need to deepcopy the embedding as it will be modifed
280 + # in place
281 + lora_embedding = VocabParallelEmbeddingWithLoRA(
282 + deepcopy(expanded_embedding))
283 + lora_embedding.create_lora_weights(max_loras, lora_config)
284 +
285 + return expanded_embedding, lora_embedding
286 +
287 + for i in range(10):
288 + set_random_seed(i)
289 +
290 + id_to_index = get_random_id_to_index(num_loras, max_loras)
291 + expanded_embedding, lora_embedding = create_random_embedding_layer()
292 + lora_dict, _ = populate_loras(
293 + id_to_index,
294 + layer=lora_embedding,
295 + layer_weights=torch.zeros(
296 + (256, 512 + lora_config.lora_extra_vocab_size)),
297 + generate_embeddings_tensor=256,
298 + )
299 +
300 + # All embeddings tensors have the same shape.
301 + embeddings_tensors = [
302 + lora_dict[id].embeddings_tensor for id in sorted(lora_dict.keys())
303 + ]
304 + embeddings_tensor_len = embeddings_tensors[0].shape[0]
305 +
306 + # Add empty embeddings_tensors for unoccupied lora slots.
307 + for _ in range(max_loras - len(embeddings_tensors)):
308 + embeddings_tensors.append(
309 + torch.zeros(embeddings_tensors[0].shape, device="cuda"))
310 +
311 + inputs, index_mapping, prompt_mapping = create_random_inputs(
312 + active_lora_ids=list(lora_dict.keys()),
313 + num_inputs=num_loras * 3,
314 + input_size=(200, ),
315 + input_range=(1, 512),
316 + )
317 + lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
318 +
319 + original_inputs = deepcopy(inputs)
320 +
321 + # Force some of the inputs to be in the extended embeddings range
322 + # to guarantee that their behavior is tested.
323 + for input_, original_input_, lora_id in zip(inputs, original_inputs,
324 + prompt_mapping):
325 + embedding_id = lora_id - 1
326 + input_[-1] = 512 + (embedding_id * embeddings_tensor_len)
327 + original_input_[-1] = 512
328 + input_[-2] = 512 + ((embedding_id + 1) * embeddings_tensor_len - 1)
329 + original_input_[-2] = 512 + embeddings_tensor_len - 1
330 +
331 + mapping_info = convert_mapping(lora_mapping, id_to_index, max_loras,
332 + 512, lora_config.lora_extra_vocab_size)
333 + lora_embedding.set_mapping(*mapping_info, )
334 +
335 + expanded_embedding.weight[512:512 +
336 + (embeddings_tensor_len *
337 + max_loras)] = torch.cat(embeddings_tensors)
338 +
339 + lora_result = lora_embedding(torch.cat(original_inputs))
340 +
341 + expected_results = []
342 + for input_, original_input_, lora_id in zip(inputs, original_inputs,
343 + prompt_mapping):
344 + lora = lora_dict[lora_id]
345 + result = expanded_embedding(input_)
346 + after_a = F.embedding(
347 + original_input_,
348 + lora.lora_a,
349 + )
350 + result += (after_a @ lora.lora_b)
351 + expected_results.append(result)
352 + expected_result = torch.cat(expected_results)
353 +
354 + rtol, atol = TOLERANCES[lora_result.dtype]
355 + assert torch.allclose(lora_result,
356 + expected_result,
357 + rtol=rtol,
358 + atol=atol)
359 +
360 + # Check that resetting the lora weights succeeds
361 +
362 + for slot_idx in range(max_loras):
363 + lora_embedding.reset_lora(slot_idx)
364 +
365 + inputs, index_mapping, prompt_mapping = create_random_inputs(
366 + active_lora_ids=[0],
367 + num_inputs=num_loras * 3,
368 + input_size=(200, ),
369 + input_range=(1, 512),
370 + )
371 + lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
372 +
373 + original_inputs = deepcopy(inputs)
374 +
375 + mapping_info = convert_mapping(lora_mapping, id_to_index, max_loras,
376 + 512, lora_config.lora_extra_vocab_size)
377 + lora_embedding.set_mapping(*mapping_info, )
378 +
379 + lora_result = lora_embedding(torch.cat(original_inputs))
380 + expected_result = expanded_embedding(torch.cat(inputs))
381 +
382 + rtol, atol = TOLERANCES[lora_result.dtype]
383 + assert torch.allclose(lora_result,
384 + expected_result,
385 + rtol=rtol,
386 + atol=atol)
387 +
388 +
389 +@torch.inference_mode()
390 +@pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
391 +def test_lm_head_sampler(dist_init, num_loras) -> None:
392 +
393 + max_loras = 8
394 + lora_config = LoRAConfig(max_loras=max_loras,
395 + max_lora_rank=8,
396 + lora_dtype=torch.float16)
397 +
398 + def create_random_sampler_layer():
399 + linear = ParallelLMHead(32000 + lora_config.lora_extra_vocab_size,
400 + 1024, 32000)
401 + linear.weight.data = torch.rand_like(linear.weight.data)
402 + linear.weight.data[:, 32000:] = 0
403 + sampler = Sampler(32000 + lora_config.lora_extra_vocab_size, 32000)
404 + lora_sampler = SamplerWithLoRA(sampler, 1024, linear.weight.dtype,
405 + linear.weight.device)
406 + lora_sampler.create_lora_weights(max_loras, lora_config)
407 +
408 + return linear, sampler, lora_sampler
409 +
410 + for i in range(10):
411 + set_random_seed(i)
412 +
413 + id_to_index = get_random_id_to_index(num_loras, max_loras)
414 + linear, sampler, lora_sampler = create_random_sampler_layer()
415 +
416 + # NOTE: all the generated loras share the same embeddings tensor.
417 + lora_dict, _ = populate_loras(
418 + id_to_index,
419 + layer=lora_sampler,
420 + layer_weights=linear.weight,
421 + generate_embeddings_tensor=1024,
422 + )
423 + embeddings_tensor = list(lora_dict.values())[0].embeddings_tensor
424 + embeddings_tensor_len = embeddings_tensor.shape[0]
425 +
426 + inputs, index_mapping, prompt_mapping = create_random_inputs(
427 + active_lora_ids=list(lora_dict.keys()),
428 + num_inputs=8 * num_loras, # * 3,
429 + input_size=(1, 1024),
430 + input_range=(0, 1),
431 + input_type=torch.float32,
432 + )
433 + lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
434 +
435 + input_ = torch.rand(20, 1024, device="cuda")
436 + mapping_info = convert_mapping(
437 + lora_mapping,
438 + id_to_index,
439 + max_loras,
440 + 32000,
441 + lora_config.lora_extra_vocab_size,
442 + )
443 + lora_sampler.set_mapping(*mapping_info, )
444 +
445 + lora_result = lora_sampler._get_logits(hidden_states=torch.cat(inputs),
446 + embedding=linear.weight,
447 + embedding_bias=None)
448 +
449 + original_weight = linear.weight.clone()
450 +
451 + linear.weight[sampler.org_vocab_size:sampler.org_vocab_size +
452 + embeddings_tensor_len] = embeddings_tensor
453 +
454 + sampler.org_vocab_size = 32000 + lora_config.lora_extra_vocab_size
455 + expected_results = []
456 + for input_, lora_id in zip(inputs, prompt_mapping):
457 + lora = lora_dict[lora_id]
458 + result = sampler._get_logits(hidden_states=input_,
459 + embedding=linear.weight,
460 + embedding_bias=None)
461 + result[:, 32000 + embeddings_tensor_len:] = float("-inf")
462 + result += input_ @ lora.lora_a @ lora.lora_b * lora.scaling
463 + expected_results.append(result)
464 + expected_result = torch.cat(expected_results)
465 + sampler.org_vocab_size = 32000
466 +
467 + # Check that resetting the lora weights succeeds
468 +
469 + for slot_idx in range(max_loras):
470 + lora_sampler.reset_lora(slot_idx)
471 +
472 + inputs, index_mapping, prompt_mapping = create_random_inputs(
473 + active_lora_ids=[0],
474 + num_inputs=8 * num_loras * 3,
475 + input_size=(1, 1024),
476 + input_range=(0, 1),
477 + input_type=torch.float32,
478 + )
479 + lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
480 +
481 + mapping_info = convert_mapping(lora_mapping, id_to_index, max_loras,
482 + 32000,
483 + lora_config.lora_extra_vocab_size)
484 + lora_sampler.set_mapping(*mapping_info, )
485 +
486 + lora_result = lora_sampler._get_logits(hidden_states=torch.cat(inputs),
487 + embedding=original_weight,
488 + embedding_bias=None)[:, :32000]
489 + expected_result = sampler._get_logits(hidden_states=torch.cat(inputs),
490 + embedding=original_weight,
491 + embedding_bias=None)
492 +
493 + rtol, atol = TOLERANCES[lora_result.dtype]
494 + assert torch.allclose(lora_result,
495 + expected_result,
496 + rtol=rtol,
497 + atol=atol)
498 +
499 +
500 +@torch.inference_mode()
501 +@pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
502 +@pytest.mark.parametrize("orientation", ["row", "column"])
503 +def test_linear_parallel(dist_init, num_loras, orientation) -> None:
504 +
505 + max_loras = 8
506 + lora_config = LoRAConfig(max_loras=max_loras,
507 + max_lora_rank=8,
508 + lora_dtype=torch.float16)
509 +
510 + def create_random_linear_parallel_layer():
511 + if orientation == "row":
512 + linear = RowParallelLinear(4096, 4096, bias=False)
513 + linear.weight.data = torch.rand_like(linear.weight.data)
514 + lora_linear = RowParallelLinearWithLoRA(linear)
515 + else:
516 + linear = ColumnParallelLinear(4096, 4096, bias=False)
517 + linear.weight.data = torch.rand_like(linear.weight.data)
518 + lora_linear = ColumnParallelLinearWithLoRA(linear)
519 + lora_linear.create_lora_weights(max_loras, lora_config)
520 +
521 + return linear, lora_linear
522 +
523 + for i in range(10):
524 + set_random_seed(i)
525 +
526 + id_to_index = get_random_id_to_index(num_loras, max_loras)
527 + linear, lora_linear = create_random_linear_parallel_layer()
528 +
529 + lora_dict, _ = populate_loras(
530 + id_to_index,
531 + layer=lora_linear,
532 + layer_weights=linear.weight,
533 + )
534 +
535 + inputs, index_mapping, prompt_mapping = create_random_inputs(
536 + active_lora_ids=list(lora_dict.keys()),
537 + num_inputs=32 * num_loras,
538 + input_size=(1, 4096),
539 + input_range=(0, 1),
540 + input_type=torch.float32,
541 + )
542 + lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
543 +
544 + mapping_info = convert_mapping(
545 + lora_mapping,
546 + id_to_index,
547 + max_loras,
548 + 512,
549 + lora_config.lora_extra_vocab_size,
550 + )
551 + lora_linear.set_mapping(*mapping_info, )
552 +
553 + lora_result = lora_linear(torch.cat(inputs))[0]
554 +
555 + expected_results = []
556 + for input_, lora_id in zip(inputs, prompt_mapping):
557 + lora = lora_dict[lora_id]
558 + result = linear(input_)[0]
559 + result += input_ @ lora.lora_a @ lora.lora_b * lora.scaling
560 + expected_results.append(result)
561 + expected_result = torch.cat(expected_results)
562 +
563 + rtol, atol = TOLERANCES[lora_result.dtype]
564 + assert torch.allclose(lora_result,
565 + expected_result,
566 + rtol=rtol,
567 + atol=atol)
568 +
569 + # Check that resetting the lora weights succeeds
570 +
571 + for slot_idx in range(max_loras):
572 + lora_linear.reset_lora(slot_idx)
573 +
574 + inputs, index_mapping, prompt_mapping = create_random_inputs(
575 + active_lora_ids=[0],
576 + num_inputs=32 * num_loras,
577 + input_size=(1, 4096),
578 + input_range=(0, 1),
579 + input_type=torch.float32,
580 + )
581 + lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
582 +
583 + mapping_info = convert_mapping(lora_mapping, id_to_index, max_loras,
584 + 512, lora_config.lora_extra_vocab_size)
585 + lora_linear.set_mapping(*mapping_info, )
586 +
587 + lora_result = lora_linear(torch.cat(inputs))[0]
588 + expected_result = linear(torch.cat(inputs))[0]
589 +
590 + rtol, atol = TOLERANCES[lora_result.dtype]
591 + assert torch.allclose(lora_result,
592 + expected_result,
593 + rtol=rtol,
594 + atol=atol)
595 +
596 +
597 +@torch.inference_mode()
598 +@pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
599 +@pytest.mark.parametrize("repeats", [2, 3])
600 +def test_column_parallel_packed(dist_init, num_loras, repeats) -> None:
601 +
602 + max_loras = 8
603 + lora_config = LoRAConfig(max_loras=max_loras,
604 + max_lora_rank=8,
605 + lora_dtype=torch.float16)
606 +
607 + def create_column_parallel_packed_layer():
608 + if repeats == 2:
609 + linear = MergedColumnParallelLinear(4096, [4096] * repeats,
610 + bias=False)
611 + linear.weight.data = torch.rand_like(linear.weight.data)
612 + lora_linear = MergedColumnParallelLinearWithLoRA(linear)
613 + else:
614 + linear = QKVParallelLinear(4096, 64, 32, bias=False)
615 + linear.weight.data = torch.rand_like(linear.weight.data)
616 + lora_linear = QKVParallelLinearWithLora(linear)
617 +
618 + @dataclass
619 + class FakeConfig:
620 + hidden_size = 4096
621 + num_key_value_heads = 32
622 + num_attention_heads = 32
623 +
624 + lora_linear.create_lora_weights(max_loras,
625 + lora_config,
626 + model_config=FakeConfig())
627 +
628 + return linear, lora_linear
629 +
630 + for i in range(10):
631 + set_random_seed(i)
632 +
633 + id_to_index = get_random_id_to_index(num_loras, max_loras)
634 +
635 + linear, lora_linear = create_column_parallel_packed_layer()
636 +
637 + lora_dict, sublora_dict = populate_loras(
638 + id_to_index,
639 + layer=lora_linear,
640 + layer_weights=linear.weight,
641 + repeats=repeats,
642 + )
643 +
644 + inputs, index_mapping, prompt_mapping = create_random_inputs(
645 + active_lora_ids=list(lora_dict.keys()),
646 + num_inputs=32 * num_loras,
647 + input_size=(1, 4096),
648 + input_range=(0, 1),
649 + input_type=torch.float32,
650 + )
651 + lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
652 +
653 + mapping_info = convert_mapping(
654 + lora_mapping,
655 + id_to_index,
656 + max_loras,
657 + 512,
658 + lora_config.lora_extra_vocab_size,
659 + )
660 + lora_linear.set_mapping(*mapping_info)
661 +
662 + lora_result = lora_linear(torch.cat(inputs))[0]
663 +
664 + expected_results = []
665 + for input_, lora_id in zip(inputs, prompt_mapping):
666 + result = linear(input_)[0]
667 + subloras = sublora_dict[lora_id]
668 + for i, sublora in enumerate(subloras):
669 + result[:, sublora.lora_b.shape[1] * i:sublora.lora_b.shape[1] * (
670 + i + 1
671 + )] += input_ @ sublora.lora_a @ sublora.lora_b * sublora.scaling
672 + expected_results.append(result)
673 + expected_result = torch.cat(expected_results)
674 +
675 + rtol, atol = TOLERANCES[lora_result.dtype]
676 + assert torch.allclose(lora_result,
677 + expected_result,
678 + rtol=rtol,
679 + atol=atol)
680 +
681 + for slot_idx in range(max_loras):
682 + lora_linear.reset_lora(slot_idx)
683 +
684 + inputs, index_mapping, prompt_mapping = create_random_inputs(
685 + active_lora_ids=[0],
686 + num_inputs=32 * num_loras,
687 + input_size=(1, 4096),
688 + input_range=(0, 1),
689 + input_type=torch.float32,
690 + )
691 + lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
692 +
693 + mapping_info = convert_mapping(
694 + lora_mapping,
695 + id_to_index,
696 + max_loras,
697 + 512,
698 + lora_config.lora_extra_vocab_size,
699 + )
700 + lora_linear.set_mapping(*mapping_info)
701 +
702 + lora_result = lora_linear(torch.cat(inputs))[0]
703 + expected_result = linear(torch.cat(inputs))[0]
704 +
705 + rtol, atol = TOLERANCES[lora_result.dtype]
706 + assert torch.allclose(lora_result,
707 + expected_result,
708 + rtol=rtol,
709 + atol=atol)
tests/lora/test_llama.py new
+144
@@ -0,0 +1,144 @@
1 +import pytest
2 +import ray
3 +
4 +import vllm
5 +from vllm.lora.request import LoRARequest
6 +from .conftest import cleanup
7 +
8 +MODEL_PATH = "meta-llama/Llama-2-7b-hf"
9 +
10 +
11 +def do_sample(llm, lora_path: str, lora_id: int):
12 + prompts = [
13 + "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_74 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]",
14 + "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_11 (nationality VARCHAR, elector VARCHAR)\n\n question: When Anchero Pantaleone was the elector what is under nationality? [/user] [assistant]",
15 + "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_95 (one_mora VARCHAR, gloss VARCHAR, accented_mora VARCHAR)\n\n question: What is the one mora for a low tone mora with a gloss of /˩okiru/ [òkìɽɯ́]? [/user] [assistant]",
16 + "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE candidate (people_id VARCHAR, unsure_rate INTEGER); CREATE TABLE people (sex VARCHAR, people_id VARCHAR)\n\n question: which gender got the highest average uncertain ratio. [/user] [assistant]",
17 + "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_60 (pick INTEGER, former_wnba_team VARCHAR)\n\n question: What pick was a player that previously played for the Minnesota Lynx? [/user] [assistant]",
18 + "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_28138035_4 (womens_doubles VARCHAR, mens_singles VARCHAR)\n\n question: Name the womens doubles for werner schlager [/user] [assistant]"
19 + ]
20 + sampling_params = vllm.SamplingParams(temperature=0,
21 + max_tokens=256,
22 + stop=["[/assistant]"])
23 + outputs = llm.generate(
24 + prompts,
25 + sampling_params,
26 + lora_request=LoRARequest(str(lora_id), lora_id, lora_path)
27 + if lora_id else None)
28 + # Print the outputs.
29 + generated_texts = []
30 + for output in outputs:
31 + prompt = output.prompt
32 + generated_text = output.outputs[0].text
33 + generated_texts.append(generated_text)
34 + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
35 + return generated_texts
36 +
37 +
38 +@pytest.mark.parametrize("tp_size", [1])
39 +def test_llama_lora(sql_lora_files, tp_size):
40 + # Cannot use as it will initialize torch.cuda too early...
41 + # if torch.cuda.device_count() < tp_size:
42 + # pytest.skip(f"Not enough GPUs for tensor parallelism {tp_size}")
43 +
44 + llm = vllm.LLM(MODEL_PATH,
45 + enable_lora=True,
46 + max_num_seqs=16,
47 + max_loras=4,
48 + tensor_parallel_size=tp_size)
49 +
50 + expected_no_lora_output = [
51 + "\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_75 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_76 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_77 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_78 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user]",
52 + " Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_11 (nationality VARCHAR, elector VARCHAR)\n\n question: When Anchero Pantaleone was the elector what is under nationality? ",
53 + "\n\n answer: 1\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_96 (one_mora VARCHAR, gloss VARCHAR, accented_mora VARCHAR)\n\n question: What is the one mora for a high tone mora with a gloss of /˧kot/ [kòt]? [/user] [assistant]\n\n answer: 2\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_97 (one_mora VARCHAR, gloss VARCHAR, accented_mora VARCHAR)\n\n question: What is the one mora for a high tone mora with a gloss of /˧kot/ [kòt]? [/user] [assistant]\n\n answer: 2\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_98 (one_mora VARCHAR, gloss VARCHAR, accented_mora VARCHAR)\n\n question: What is the one m",
54 + " Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE candidate (people_id VARCHAR, unsure_rate INTEGER); CREATE TABLE people (sex VARCHAR, people_id VARCHAR)\n\n question: which gender got the highest average uncertain ratio. ",
55 + " Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_60 (pick INTEGER, former_wnba_team VARCHAR)\n\n question: What pick was a player that previously played for the Minnesota Lynx? ",
56 + "\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_28138035_4 (womens_doubles VARCHAR, mens_singles VARCHAR)\n\n question: Name the womens doubles for werner schlager [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_28138035_4 (womens_doubles VARCHAR, mens_singles VARCHAR)\n\n question: Name the womens doubles for werner schlager [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_28138035_4 (womens_doubles VARCHAR, mens_singles VARCHAR)\n\n question: Name the womens doubles for werner schlager [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE",
57 + ]
58 + expected_lora_output = [
59 + " SELECT icao FROM table_name_74 WHERE airport = 'lilongwe international airport' ",
60 + " SELECT nationality FROM table_name_11 WHERE elector = 'anchero pantaleone' ",
61 + " SELECT one_mora FROM table_name_95 WHERE gloss = 'low tone mora with a gloss of /˩okiru/' [òkìɽɯ́] AND accented_mora = 'low tone mora with a gloss of /˩okiru/' [òkìɽɯ́] ",
62 + " SELECT sex FROM people WHERE people_id IN (SELECT people_id FROM candidate GROUP BY sex ORDER BY COUNT(people_id) DESC LIMIT 1) ",
63 + " SELECT pick FROM table_name_60 WHERE former_wnba_team = 'Minnesota Lynx' ",
64 + " SELECT womens_doubles FROM table_28138035_4 WHERE mens_singles = 'Werner Schlager' "
65 + ]
66 +
67 + print("lora adapter created")
68 + assert do_sample(llm, sql_lora_files, lora_id=0) == expected_no_lora_output
69 +
70 + print("lora 1")
71 + assert do_sample(llm, sql_lora_files, lora_id=1) == expected_lora_output
72 +
73 + print("no lora")
74 + assert do_sample(llm, sql_lora_files, lora_id=0) == expected_no_lora_output
75 +
76 + print("lora 2")
77 + assert do_sample(llm, sql_lora_files, lora_id=2) == expected_lora_output
78 +
79 + print("removing lora")
80 +
81 +
82 +@pytest.mark.skip("Requires multiple GPUs")
83 +def test_llama_tensor_parallel_equality(sql_lora_files):
84 + # Cannot use as it will initialize torch.cuda too early...
85 + # if torch.cuda.device_count() < 4:
86 + # pytest.skip(f"Not enough GPUs for tensor parallelism {4}")
87 +
88 + llm_tp1 = vllm.LLM(MODEL_PATH,
89 + enable_lora=True,
90 + max_num_seqs=16,
91 + max_loras=4,
92 + tensor_parallel_size=1)
93 + output_tp1 = do_sample(llm_tp1, sql_lora_files, lora_id=1)
94 +
95 + del llm_tp1
96 + cleanup()
97 +
98 + llm_tp2 = vllm.LLM(MODEL_PATH,
99 + enable_lora=True,
100 + max_num_seqs=16,
101 + max_loras=4,
102 + tensor_parallel_size=2)
103 + output_tp2 = do_sample(llm_tp2, sql_lora_files, lora_id=1)
104 +
105 + del llm_tp2
106 + cleanup()
107 +
108 + assert output_tp1 == output_tp2
109 +
110 + llm_tp4 = vllm.LLM(MODEL_PATH,
111 + enable_lora=True,
112 + max_num_seqs=16,
113 + max_loras=4,
114 + tensor_parallel_size=4)
115 + output_tp4 = do_sample(llm_tp4, sql_lora_files, lora_id=1)
116 +
117 + del llm_tp4
118 + cleanup()
119 +
120 + assert output_tp1 == output_tp4
121 +
122 +
123 +def test_llama_lora_warmup(sql_lora_files):
124 + """Test that the LLM initialization works with a warmup LORA path and is more conservative"""
125 +
126 + @ray.remote(num_gpus=1)
127 + def get_num_gpu_blocks_lora():
128 + llm = vllm.LLM(MODEL_PATH, enable_lora=True, max_num_seqs=16)
129 + num_gpu_blocks_lora_warmup = llm.llm_engine.cache_config.num_gpu_blocks
130 + return num_gpu_blocks_lora_warmup
131 +
132 + @ray.remote(num_gpus=1)
133 + def get_num_gpu_blocks_no_lora():
134 + llm = vllm.LLM(MODEL_PATH, max_num_seqs=16)
135 + num_gpu_blocks_no_lora_warmup = llm.llm_engine.cache_config.num_gpu_blocks
136 + return num_gpu_blocks_no_lora_warmup
137 +
138 + num_gpu_blocks_lora_warmup = ray.get(get_num_gpu_blocks_lora.remote())
139 + num_gpu_blocks_no_lora_warmup = ray.get(
140 + get_num_gpu_blocks_no_lora.remote())
141 + assert num_gpu_blocks_lora_warmup < num_gpu_blocks_no_lora_warmup, (
142 + "The warmup with lora should be more"
143 + " conservative than without lora, therefore the number of memory blocks for the KV cache should be "
144 + "less when using lora than when not using lora")
tests/lora/test_lora.py new
+224
@@ -0,0 +1,224 @@
1 +import pytest
2 +import torch
3 +
4 +from vllm.lora.layers import _apply_lora, _apply_lora_packed_nslice
5 +
6 +from .utils import DummyLoRAManager
7 +
8 +TENSOR_SIZES = [128, 1024, 2048, 4096, 8192, 11008, 11008 // 2, 11008 // 4]
9 +QKV_TENSOR_SIZES = [
10 + (8192, 1024, 1024),
11 + (8192 // 8, 1024 // 8, 1024 // 8),
12 + (4096, 4096, 4096),
13 + (4096 // 2, 4096 // 2, 4096 // 2),
14 +]
15 +BATCH_SIZES = [8, 32, 256]
16 +RANKS = [8]
17 +DTYPES = [torch.float16]
18 +TOLERANCES = {
19 + torch.float16: (5e-3, 5e-3),
20 + torch.bfloat16: (3e-2, 2e-2),
21 +}
22 +
23 +
24 +@pytest.mark.parametrize("m", TENSOR_SIZES)
25 +@pytest.mark.parametrize("n", TENSOR_SIZES)
26 +@pytest.mark.parametrize("k", BATCH_SIZES)
27 +@pytest.mark.parametrize("rank", RANKS)
28 +@pytest.mark.parametrize("dtype", DTYPES)
29 +def test_apply_lora(m, n, k, rank, dtype) -> None:
30 + manager = DummyLoRAManager()
31 +
32 + module_name = "module"
33 + weight = torch.rand([m, n], device="cuda", dtype=dtype)
34 +
35 + manager.init_random_lora(module_name, weight, rank=rank)
36 + lora = manager.get_module_lora(module_name)
37 +
38 + input = torch.rand(k, n, device="cuda", dtype=dtype)
39 + expected = input @ lora.lora_a @ lora.lora_b * lora.scaling
40 +
41 + lora_a_stack = torch.zeros(8,
42 + 1,
43 + lora.lora_a.shape[1],
44 + lora.lora_a.shape[0],
45 + device="cuda",
46 + dtype=dtype)
47 + lora_b_stack = torch.zeros(8,
48 + 1,
49 + lora.lora_b.shape[1],
50 + lora.lora_b.shape[0],
51 + device="cuda",
52 + dtype=dtype)
53 + for i in range(lora_a_stack.shape[0]):
54 + lora_a_stack[i][0] = lora.lora_a.T
55 + lora_b_stack[i][0] = (lora.lora_b * lora.scaling).T
56 +
57 + output = torch.zeros(k, m, device="cuda", dtype=dtype)
58 + _apply_lora(
59 + input, lora_a_stack, lora_b_stack,
60 + torch.randint(0, lora_a_stack.shape[0], (len(input), ), device="cuda"),
61 + output)
62 +
63 + rtol, atol = TOLERANCES[dtype]
64 + assert torch.allclose(expected, output, rtol=rtol, atol=atol)
65 +
66 + output[:] = 0
67 + _apply_lora(input, lora_a_stack, lora_b_stack,
68 + torch.full((len(input), ), -1, device="cuda"), output)
69 + assert torch.allclose(torch.zeros_like(output), output)
70 +
71 + manager.reset_lora()
72 +
73 +
74 +@pytest.mark.parametrize("m", TENSOR_SIZES)
75 +@pytest.mark.parametrize("n", TENSOR_SIZES)
76 +@pytest.mark.parametrize("k", BATCH_SIZES)
77 +@pytest.mark.parametrize("rank", RANKS)
78 +@pytest.mark.parametrize("dtype", DTYPES)
79 +def test_apply_lora_packed_2slice(m, n, k, rank, dtype) -> None:
80 + if m % 2 != 0:
81 + pytest.skip("m must be divisible by 2")
82 + if m // 2 not in TENSOR_SIZES:
83 + pytest.skip("m//2 must be in TENSOR_SIZES")
84 +
85 + manager = DummyLoRAManager()
86 +
87 + module_name = "module"
88 + weight = torch.rand([m // 2, n], device="cuda", dtype=dtype)
89 +
90 + manager.init_random_lora(module_name + "1", weight, rank=rank)
91 + lora_1 = manager.get_module_lora(module_name + "1")
92 + manager.init_random_lora(module_name + "2", weight, rank=rank)
93 + lora_2 = manager.get_module_lora(module_name + "2")
94 +
95 + input = torch.rand(k, n, device="cuda", dtype=dtype)
96 + expected = torch.cat([
97 + input @ lora_1.lora_a @ lora_1.lora_b * lora_1.scaling,
98 + input @ lora_2.lora_a @ lora_2.lora_b * lora_2.scaling
99 + ],
100 + dim=1)
101 +
102 + lora_a_stacks = [
103 + torch.zeros(8,
104 + 1,
105 + lora_1.lora_a.shape[1],
106 + lora_1.lora_a.shape[0],
107 + device="cuda",
108 + dtype=dtype) for i in range(2)
109 + ]
110 + lora_b_stacks = [
111 + torch.zeros(8,
112 + 1,
113 + lora_1.lora_b.shape[1],
114 + lora_1.lora_b.shape[0],
115 + device="cuda",
116 + dtype=dtype) for i in range(2)
117 + ]
118 + for i in range(lora_a_stacks[0].shape[0]):
119 + lora_a_stacks[0][i][0] = lora_1.lora_a.T
120 + lora_b_stacks[0][i][0] = (lora_1.lora_b * lora_1.scaling).T
121 + lora_a_stacks[1][i][0] = lora_2.lora_a.T
122 + lora_b_stacks[1][i][0] = (lora_2.lora_b * lora_2.scaling).T
123 +
124 + output = torch.zeros(k, m, device="cuda", dtype=dtype)
125 + _apply_lora_packed_nslice(
126 + input, lora_a_stacks, lora_b_stacks,
127 + torch.randint(0,
128 + lora_a_stacks[0].shape[0], (len(input), ),
129 + device="cuda"), output, (m // 2, m // 2))
130 +
131 + rtol, atol = TOLERANCES[dtype]
132 + assert torch.allclose(expected, output, rtol=rtol, atol=atol)
133 +
134 + output[:] = 0
135 + _apply_lora_packed_nslice(input, lora_a_stacks, lora_b_stacks,
136 + torch.full((len(input), ), -1, device="cuda"),
137 + output, (m // 2, m // 2))
138 + assert torch.allclose(torch.zeros_like(output), output)
139 +
140 + manager.reset_lora()
141 +
142 +
143 +@pytest.mark.parametrize("qkv", QKV_TENSOR_SIZES)
144 +@pytest.mark.parametrize("n", TENSOR_SIZES)
145 +@pytest.mark.parametrize("k", BATCH_SIZES)
146 +@pytest.mark.parametrize("rank", RANKS)
147 +@pytest.mark.parametrize("dtype", DTYPES)
148 +def test_apply_lora_packed_3slice(qkv, n, k, rank, dtype) -> None:
149 + manager = DummyLoRAManager()
150 +
151 + module_name = "module"
152 + weight_q = torch.empty(qkv[0], n, device="cuda", dtype=dtype)
153 + weight_kv = torch.empty(qkv[1], n, device="cuda", dtype=dtype)
154 +
155 + manager.init_random_lora(module_name + "q", weight_q, rank=rank)
156 + lora_q = manager.get_module_lora(module_name + "q")
157 + manager.init_random_lora(module_name + "k", weight_kv, rank=rank)
158 + lora_k = manager.get_module_lora(module_name + "k")
159 + manager.init_random_lora(module_name + "v", weight_kv, rank=rank)
160 + lora_v = manager.get_module_lora(module_name + "v")
161 +
162 + input = torch.rand(k, n, device="cuda", dtype=dtype)
163 + expected = torch.cat([
164 + input @ lora_q.lora_a @ lora_q.lora_b * lora_q.scaling,
165 + input @ lora_k.lora_a @ lora_k.lora_b * lora_k.scaling,
166 + input @ lora_v.lora_a @ lora_v.lora_b * lora_v.scaling
167 + ],
168 + dim=1)
169 +
170 + lora_a_stacks = [
171 + torch.zeros(8,
172 + 1,
173 + lora_q.lora_a.shape[1],
174 + lora_q.lora_a.shape[0],
175 + device="cuda",
176 + dtype=dtype)
177 + ] + [
178 + torch.zeros(8,
179 + 1,
180 + lora_k.lora_a.shape[1],
181 + lora_k.lora_a.shape[0],
182 + device="cuda",
183 + dtype=dtype) for i in range(2)
184 + ]
185 + lora_b_stacks = [
186 + torch.zeros(8,
187 + 1,
188 + lora_q.lora_b.shape[1],
189 + lora_q.lora_b.shape[0],
190 + device="cuda",
191 + dtype=dtype)
192 + ] + [
193 + torch.zeros(8,
194 + 1,
195 + lora_k.lora_b.shape[1],
196 + lora_k.lora_b.shape[0],
197 + device="cuda",
198 + dtype=dtype) for i in range(2)
199 + ]
200 + for i in range(lora_a_stacks[0].shape[0]):
201 + lora_a_stacks[0][i][0] = lora_q.lora_a.T
202 + lora_b_stacks[0][i][0] = (lora_q.lora_b * lora_q.scaling).T
203 + lora_a_stacks[1][i][0] = lora_k.lora_a.T
204 + lora_b_stacks[1][i][0] = (lora_k.lora_b * lora_k.scaling).T
205 + lora_a_stacks[2][i][0] = lora_v.lora_a.T
206 + lora_b_stacks[2][i][0] = (lora_v.lora_b * lora_v.scaling).T
207 +
208 + output = torch.zeros(k, sum(qkv), device="cuda", dtype=dtype)
209 + _apply_lora_packed_nslice(
210 + input, lora_a_stacks, lora_b_stacks,
211 + torch.randint(0,
212 + lora_a_stacks[0].shape[0], (len(input), ),
213 + device="cuda"), output, (qkv[0], qkv[1], qkv[2]))
214 +
215 + rtol, atol = TOLERANCES[dtype]
216 + assert torch.allclose(expected, output, rtol=rtol, atol=atol)
217 +
218 + output[:] = 0
219 + _apply_lora_packed_nslice(input, lora_a_stacks, lora_b_stacks,
220 + torch.full((len(input), ), -1, device="cuda"),
221 + output, (qkv[0], qkv[1], qkv[2]))
222 + assert torch.allclose(torch.zeros_like(output), output)
223 +
224 + manager.reset_lora()
tests/lora/test_lora_manager.py new
+475
@@ -0,0 +1,475 @@
1 +import os
2 +from typing import List
3 +
4 +import pytest
5 +import torch
6 +from safetensors.torch import load_file
7 +from torch import nn
8 +
9 +from vllm.config import LoRAConfig
10 +from vllm.lora.layers import (ColumnParallelLinearWithLoRA,
11 + RowParallelLinearWithLoRA,
12 + MergedColumnParallelLinearWithLoRA)
13 +from vllm.lora.lora import LoRALayerWeights, PackedLoRALayerWeights
14 +from vllm.lora.models import (EMBEDDING_MODULES, LoRAModel, LoRAModelManager,
15 + LRUCacheLoRAModelManager, LoRAMapping)
16 +from vllm.lora.request import LoRARequest
17 +from vllm.lora.worker_manager import (LRUCacheWorkerLoRAManager,
18 + WorkerLoRAManager)
19 +from vllm.model_executor.layers.linear import RowParallelLinear
20 +
21 +
22 +def test_from_lora_tensors(sql_lora_files):
23 + tensors = load_file(
24 + os.path.join(sql_lora_files, "adapter_model.safetensors"))
25 + new_embeddings = load_file(
26 + os.path.join(sql_lora_files, "new_embeddings.safetensors"))
27 + lora_model = LoRAModel.from_lora_tensors(1,
28 + 8,
29 + 16,
30 + tensors,
31 + "cuda",
32 + embeddings=new_embeddings)
33 + for module_name, lora in lora_model.loras.items():
34 + assert lora.module_name == module_name
35 + assert lora.rank == 8
36 + assert lora.lora_alpha == 16
37 + assert lora.lora_a is not None
38 + assert lora.lora_b is not None
39 + assert (lora.lora_a.shape[1] == lora.lora_b.shape[0]
40 + ), f"{lora.lora_a.shape=}, {lora.lora_b.shape=}"
41 + assert lora.lora_a.shape[1] == 8
42 + embeddings_module = next(
43 + (k for k in EMBEDDING_MODULES if k in module_name), None)
44 + if embeddings_module:
45 + assert torch.equal(
46 + lora.embeddings_tensor,
47 + new_embeddings[EMBEDDING_MODULES[embeddings_module]].to(
48 + device=lora.embeddings_tensor.device))
49 + else:
50 + assert lora.embeddings_tensor is None
51 +
52 +
53 +def create_lora(lora_id: int, model: nn.Module,
54 + sub_modules: List[str]) -> LoRAModel:
55 + loras = {}
56 + for name in sub_modules:
57 + w = model.get_submodule(name).weight
58 + loras[name] = LoRALayerWeights(
59 + name,
60 + 8,
61 + 16,
62 + torch.rand([w.shape[1], 8], device="cuda"),
63 + torch.rand([8, w.shape[0]], device="cuda"),
64 + )
65 + return LoRAModel(lora_id, 8, loras)
66 +
67 +
68 +def create_packed_lora(
69 + lora_id: int,
70 + model: nn.Module,
71 + module_name,
72 + replaced_module_names,
73 + empty_replaced_module_name=None,
74 +) -> LoRAModel:
75 + w = model.get_submodule(module_name).weight
76 + loras = {}
77 + for replaced_module_name in replaced_module_names:
78 + if replaced_module_name == empty_replaced_module_name:
79 + continue
80 + loras[replaced_module_name] = LoRALayerWeights(
81 + replaced_module_name,
82 + 8,
83 + 16,
84 + torch.rand([w.shape[1], 8], device="cuda"),
85 + torch.rand([8, w.shape[0] // len(replaced_module_names)],
86 + device="cuda"),
87 + )
88 + return LoRAModel(lora_id, 8, loras)
89 +
90 +
91 +def test_replace_submodules(dist_init, dummy_model):
92 + model = dummy_model
93 + manager = LoRAModelManager(model,
94 + 1,
95 + 1,
96 + 1,
97 + LoRAConfig(max_lora_rank=8,
98 + max_cpu_loras=8,
99 + max_loras=8),
100 + lora_target_modules=["dense1", "layer1.dense2"])
101 + model = manager.model
102 +
103 + assert isinstance(model.get_submodule("dense1"),
104 + ColumnParallelLinearWithLoRA)
105 + assert isinstance(model.get_submodule("layer1.dense1"),
106 + ColumnParallelLinearWithLoRA)
107 + assert isinstance(model.get_submodule("dense2"), RowParallelLinear)
108 + assert isinstance(model.get_submodule("layer1.dense2"),
109 + RowParallelLinearWithLoRA)
110 +
111 +
112 +def test_lora_model_manager(dist_init, dummy_model):
113 + model = dummy_model
114 + model_lora1 = create_lora(1, model, ["layer1.dense1", "dense2", "lm_head"])
115 + model_lora2 = create_lora(2, model, ["dense1", "dense2", "lm_head"])
116 + model_lora3 = create_lora(3, model, ["dense1", "dense2", "lm_head"])
117 + manager = LoRAModelManager(
118 + model,
119 + 2,
120 + 2,
121 + 2,
122 + LoRAConfig(max_lora_rank=8, max_cpu_loras=3, max_loras=2),
123 + lora_target_modules=["dense1", "dense2", "lm_head"])
124 + assert all(x is None for x in manager.lora_index_to_id)
125 + assert manager.add_lora(model_lora1)
126 + assert manager.activate_lora(1)
127 + assert manager.lora_index_to_id[0] == 1
128 + assert not manager.add_lora(model_lora1)
129 + assert not manager.activate_lora(1)
130 + assert manager.add_lora(model_lora2)
131 + assert manager.activate_lora(2)
132 + assert manager.lora_index_to_id[0] == 1
133 + assert manager.lora_index_to_id[1] == 2
134 + assert not manager.add_lora(model_lora2)
135 + assert not manager.activate_lora(2)
136 + assert manager.add_lora(model_lora3)
137 + assert manager.lora_index_to_id[0] == 1
138 + assert manager.lora_index_to_id[1] == 2
139 + with pytest.raises(ValueError):
140 + assert manager.activate_lora(3)
141 + assert manager.lora_index_to_id[0] == 1
142 + assert manager.lora_index_to_id[1] == 2
143 + assert manager.remove_lora(model_lora2.id)
144 + assert manager.lora_index_to_id[1] is None
145 + assert not manager.remove_lora(model_lora2.id)
146 + assert manager.remove_lora(model_lora1.id)
147 + assert not manager.remove_lora(model_lora1.id)
148 + assert manager.add_lora(model_lora1)
149 + assert manager.lora_index_to_id[0] is None
150 + assert manager.lora_index_to_id[1] is None
151 + assert manager.add_lora(model_lora2)
152 + assert manager.activate_lora(3)
153 + assert manager.lora_index_to_id[0] == 3
154 + assert manager.lora_index_to_id[1] is None
155 + assert manager.activate_lora(2)
156 + assert manager.lora_index_to_id[0] == 3
157 + assert manager.lora_index_to_id[1] == 2
158 +
159 +
160 +def test_lora_lru_cache_model_manager(dist_init, dummy_model):
161 + model = dummy_model
162 + model_lora1 = create_lora(1, model, ["layer1.dense1", "dense2", "lm_head"])
163 + model_lora2 = create_lora(2, model, ["dense1", "dense2", "lm_head"])
164 + model_lora3 = create_lora(3, model, ["dense1", "dense2", "lm_head"])
165 + manager = LRUCacheLoRAModelManager(
166 + model,
167 + 2,
168 + 2,
169 + 2,
170 + LoRAConfig(max_lora_rank=8, max_cpu_loras=3, max_loras=2),
171 + lora_target_modules=["dense1", "dense2", "lm_head"])
172 + assert all(x is None for x in manager.lora_index_to_id)
173 + assert manager.add_lora(model_lora1)
174 + assert manager.activate_lora(1)
175 + assert manager.lora_index_to_id[0] == 1
176 + assert not manager.add_lora(model_lora1)
177 + assert not manager.activate_lora(1)
178 + assert manager.add_lora(model_lora2)
179 + assert manager.activate_lora(2)
180 + assert manager.lora_index_to_id[0] == 1
181 + assert manager.lora_index_to_id[1] == 2
182 + assert not manager.add_lora(model_lora2)
183 + assert not manager.activate_lora(2)
184 + assert manager.add_lora(model_lora3)
185 + assert manager.lora_index_to_id[0] == 1
186 + assert manager.lora_index_to_id[1] == 2
187 + assert manager.activate_lora(3)
188 + assert manager.lora_index_to_id[0] == 3
189 + assert manager.lora_index_to_id[1] == 2
190 + assert manager.remove_lora(model_lora2.id)
191 + assert manager.lora_index_to_id[1] is None
192 + assert not manager.remove_lora(model_lora2.id)
193 + assert manager.remove_lora(model_lora1.id)
194 + assert not manager.remove_lora(model_lora1.id)
195 + assert manager.add_lora(model_lora1)
196 + assert manager.activate_lora(1)
197 + assert manager.lora_index_to_id[0] == 3
198 + assert manager.lora_index_to_id[1] == 1
199 + assert manager.add_lora(model_lora2)
200 + assert manager.deactivate_lora(3)
201 + assert manager.lora_index_to_id[0] is None
202 + assert manager.lora_index_to_id[1] == 1
203 + assert manager.activate_lora(2)
204 + assert manager.lora_index_to_id[0] == 2
205 + assert manager.lora_index_to_id[1] == 1
206 + assert manager.activate_lora(3)
207 + assert manager.lora_index_to_id[0] == 2
208 + assert manager.lora_index_to_id[1] == 3
209 +
210 +
211 +def test_lru_lora_model_manager(dist_init, dummy_model):
212 + # This tests just the LRU cache functionality, everything else is
213 + # tested in test_lora_model_manager
214 + model = dummy_model
215 + model_lora1 = create_lora(1, model, ["layer1.dense1", "dense2", "lm_head"])
216 + model_lora2 = create_lora(2, model, ["dense1", "dense2", "lm_head"])
217 + model_lora3 = create_lora(3, model, ["dense1", "dense2", "lm_head"])
218 + model_lora4 = create_lora(4, model, ["dense1", "dense2", "lm_head"])
219 + manager = LRUCacheLoRAModelManager(
220 + model, 2, 2, 2,
221 + LoRAConfig(max_lora_rank=8, max_cpu_loras=2, max_loras=2),
222 + ["dense1", "dense2", "lm_head"])
223 +
224 + assert all(x is None for x in manager.lora_index_to_id)
225 +
226 + # Add up to capacity
227 + assert manager.add_lora(model_lora1)
228 + assert manager.add_lora(model_lora2)
229 + assert manager.activate_lora(1)
230 + assert manager.activate_lora(2)
231 +
232 + assert set(manager.list_loras()) == {1, 2}
233 + assert manager.lora_index_to_id[0] == 1
234 + assert manager.lora_index_to_id[1] == 2
235 +
236 + # Add over capacity
237 + assert manager.add_lora(model_lora3)
238 + assert manager.add_lora(model_lora4)
239 + assert manager.activate_lora(3)
240 + assert manager.activate_lora(4)
241 +
242 + assert set(manager.list_loras()) == {3, 4}
243 + assert manager.lora_index_to_id[0] == 3
244 + assert manager.lora_index_to_id[1] == 4
245 +
246 + # Add 3 again to move it to the top and then add 2
247 + # should return false since it's in already
248 + assert not manager.add_lora(model_lora3)
249 + assert not manager.activate_lora(3)
250 + assert manager.add_lora(model_lora2)
251 + assert manager.activate_lora(2)
252 +
253 + assert set(manager.list_loras()) == {3, 2}
254 + assert manager.lora_index_to_id[0] == 3
255 + assert manager.lora_index_to_id[1] == 2
256 +
257 + # Remove manually
258 + assert manager.remove_lora(3)
259 + assert not manager.remove_lora(3)
260 +
261 + assert set(manager.list_loras()) == {2}
262 + assert manager.lora_index_to_id[0] is None
263 + assert manager.lora_index_to_id[1] == 2
264 +
265 + assert manager.add_lora(model_lora3)
266 + assert manager.activate_lora(3)
267 + assert manager.add_lora(model_lora4)
268 + assert manager.activate_lora(4)
269 +
270 + assert set(manager.list_loras()) == {3, 4}
271 + assert manager.lora_index_to_id[0] == 3
272 + assert manager.lora_index_to_id[1] == 4
273 +
274 + assert manager.remove_oldest_lora()
275 + assert set(manager.list_loras()) == {4}
276 + assert manager.lora_index_to_id[0] is None
277 + assert manager.lora_index_to_id[1] == 4
278 +
279 + assert manager.remove_oldest_lora()
280 + assert set(manager.list_loras()) == set()
281 + assert all(x is None for x in manager.lora_index_to_id)
282 +
283 + assert not manager.remove_oldest_lora()
284 + assert set(manager.list_loras()) == set()
285 + assert all(x is None for x in manager.lora_index_to_id)
286 +
287 +
288 +def test_lru_cache_worker_lora_manager(llama_2_7b_model_extra_embeddings,
289 + sql_lora_files):
290 + lora_config = LoRAConfig(max_lora_rank=8, max_cpu_loras=4, max_loras=4)
291 + worker_lora_manager = LRUCacheWorkerLoRAManager(
292 + 4, 2, llama_2_7b_model_extra_embeddings.config.vocab_size, lora_config,
293 + torch.device("cuda"))
294 + worker_lora_manager.create_lora_manager(llama_2_7b_model_extra_embeddings)
295 +
296 + mapping = LoRAMapping([], [])
297 + worker_lora_manager.set_active_loras([
298 + LoRARequest("1", 1, sql_lora_files),
299 + LoRARequest("2", 2, sql_lora_files)
300 + ], mapping)
301 + assert worker_lora_manager.list_loras() == {1, 2}
302 + assert worker_lora_manager._lora_manager.lora_index_to_id[0] == 1
303 + assert worker_lora_manager._lora_manager.lora_index_to_id[1] == 2
304 +
305 + worker_lora_manager.set_active_loras([
306 + LoRARequest("1", 1, sql_lora_files),
307 + LoRARequest("3", 3, sql_lora_files),
308 + LoRARequest("4", 4, sql_lora_files)
309 + ], mapping)
310 + assert worker_lora_manager.list_loras() == {1, 2, 3, 4}
311 + assert worker_lora_manager._lora_manager.lora_index_to_id[0] == 1
312 + assert worker_lora_manager._lora_manager.lora_index_to_id[1] == 2
313 + assert worker_lora_manager._lora_manager.lora_index_to_id[2] == 3
314 + assert worker_lora_manager._lora_manager.lora_index_to_id[3] == 4
315 +
316 + worker_lora_manager.set_active_loras([
317 + LoRARequest("1", 1, sql_lora_files),
318 + LoRARequest("2", 2, sql_lora_files),
319 + LoRARequest("5", 5, sql_lora_files)
320 + ], mapping)
321 + assert worker_lora_manager.list_loras() == {1, 2, 4, 5}
322 + assert worker_lora_manager._lora_manager.lora_index_to_id[0] == 1
323 + assert worker_lora_manager._lora_manager.lora_index_to_id[1] == 2
324 + assert worker_lora_manager._lora_manager.lora_index_to_id[2] == 5
325 + assert worker_lora_manager._lora_manager.lora_index_to_id[3] == 4
326 +
327 + worker_lora_manager.set_active_loras([
328 + LoRARequest("1", 1, sql_lora_files),
329 + LoRARequest("1", 1, sql_lora_files),
330 + LoRARequest("1", 1, sql_lora_files)
331 + ], mapping)
332 + assert worker_lora_manager.list_loras() == {1, 2, 4, 5}
333 + assert worker_lora_manager._lora_manager.lora_index_to_id[0] == 1
334 + assert worker_lora_manager._lora_manager.lora_index_to_id[1] == 2
335 + assert worker_lora_manager._lora_manager.lora_index_to_id[2] == 5
336 + assert worker_lora_manager._lora_manager.lora_index_to_id[3] == 4
337 +
338 + worker_lora_manager.set_active_loras([
339 + LoRARequest("6", 6, sql_lora_files),
340 + LoRARequest("7", 7, sql_lora_files),
341 + LoRARequest("8", 8, sql_lora_files)
342 + ], mapping)
343 + assert worker_lora_manager.list_loras() == {1, 6, 7, 8}
344 + assert worker_lora_manager._lora_manager.lora_index_to_id[0] == 1
345 + assert worker_lora_manager._lora_manager.lora_index_to_id[1] == 7
346 + assert worker_lora_manager._lora_manager.lora_index_to_id[2] == 8
347 + assert worker_lora_manager._lora_manager.lora_index_to_id[3] == 6
348 +
349 + # Over capacity
350 + with pytest.raises(RuntimeError):
351 + worker_lora_manager.set_active_loras([
352 + LoRARequest("10", 10, sql_lora_files),
353 + LoRARequest("11", 11, sql_lora_files),
354 + LoRARequest("12", 12, sql_lora_files),
355 + LoRARequest("13", 13, sql_lora_files),
356 + LoRARequest("14", 14, sql_lora_files)
357 + ], mapping)
358 +
359 +
360 +def test_worker_lora_manager(llama_2_7b_model_extra_embeddings,
361 + sql_lora_files):
362 + # Should remove every LoRA not specified in the request.
363 + lora_config = LoRAConfig(max_lora_rank=8, max_cpu_loras=4, max_loras=4)
364 + worker_lora_manager = WorkerLoRAManager(
365 + 4, 2, llama_2_7b_model_extra_embeddings.config.vocab_size, lora_config,
366 + torch.device("cuda"))
367 + worker_lora_manager.create_lora_manager(llama_2_7b_model_extra_embeddings)
368 +
369 + mapping = LoRAMapping([], [])
370 + worker_lora_manager.set_active_loras([
371 + LoRARequest("1", 1, sql_lora_files),
372 + LoRARequest("2", 2, sql_lora_files)
373 + ], mapping)
374 + assert worker_lora_manager.list_loras() == {1, 2}
375 + assert worker_lora_manager._lora_manager.lora_index_to_id[0] == 1
376 + assert worker_lora_manager._lora_manager.lora_index_to_id[1] == 2
377 +
378 + worker_lora_manager.set_active_loras([
379 + LoRARequest("1", 1, sql_lora_files),
380 + LoRARequest("3", 3, sql_lora_files),
381 + LoRARequest("4", 4, sql_lora_files)
382 + ], mapping)
383 + assert worker_lora_manager.list_loras() == {1, 3, 4}
384 + assert worker_lora_manager._lora_manager.lora_index_to_id[0] == 1
385 + assert worker_lora_manager._lora_manager.lora_index_to_id[1] == 3
386 + assert worker_lora_manager._lora_manager.lora_index_to_id[2] == 4
387 +
388 + worker_lora_manager.set_active_loras([
389 + LoRARequest("1", 1, sql_lora_files),
390 + LoRARequest("2", 2, sql_lora_files),
391 + LoRARequest("5", 5, sql_lora_files)
392 + ], mapping)
393 + assert worker_lora_manager.list_loras() == {1, 2, 5}
394 + assert worker_lora_manager._lora_manager.lora_index_to_id[0] == 1
395 + assert worker_lora_manager._lora_manager.lora_index_to_id[1] == 2
396 + assert worker_lora_manager._lora_manager.lora_index_to_id[2] == 5
397 +
398 + worker_lora_manager.set_active_loras([
399 + LoRARequest("1", 1, sql_lora_files),
400 + LoRARequest("1", 1, sql_lora_files),
401 + LoRARequest("1", 1, sql_lora_files)
402 + ], mapping)
403 + assert worker_lora_manager.list_loras() == {1}
404 + assert worker_lora_manager._lora_manager.lora_index_to_id[0] == 1
405 + assert worker_lora_manager._lora_manager.lora_index_to_id[1] is None
406 + assert worker_lora_manager._lora_manager.lora_index_to_id[2] is None
407 +
408 + worker_lora_manager.set_active_loras([
409 + LoRARequest("6", 6, sql_lora_files),
410 + LoRARequest("7", 7, sql_lora_files),
411 + LoRARequest("8", 8, sql_lora_files)
412 + ], mapping)
413 + assert worker_lora_manager.list_loras() == {6, 7, 8}
414 + assert worker_lora_manager._lora_manager.lora_index_to_id[0] == 8
415 + assert worker_lora_manager._lora_manager.lora_index_to_id[1] == 6
416 + assert worker_lora_manager._lora_manager.lora_index_to_id[2] == 7
417 +
418 + # Over capacity
419 + with pytest.raises(RuntimeError):
420 + worker_lora_manager.set_active_loras([
421 + LoRARequest("10", 10, sql_lora_files),
422 + LoRARequest("11", 11, sql_lora_files),
423 + LoRARequest("12", 12, sql_lora_files),
424 + LoRARequest("13", 13, sql_lora_files),
425 + LoRARequest("14", 14, sql_lora_files)
426 + ], mapping)
427 +
428 +
429 +def test_packed_loras(dist_init, dummy_model_gate_up):
430 + model = dummy_model_gate_up
431 + model_lora = create_packed_lora(
432 + 1,
433 + model,
434 + module_name="gate_up_proj",
435 + replaced_module_names=["gate_proj", "up_proj"])
436 + model_lora1 = create_packed_lora(
437 + 2,
438 + model,
439 + module_name="gate_up_proj",
440 + replaced_module_names=["gate_proj", "up_proj"],
441 + empty_replaced_module_name="gate_proj",
442 + )
443 +
444 + manager = LoRAModelManager(
445 + model, 2, 2, 2,
446 + LoRAConfig(max_lora_rank=8, max_cpu_loras=2, max_loras=2),
447 + ["gate_up_proj"])
448 + model = manager.model
449 +
450 + assert isinstance(model.get_submodule("gate_up_proj"),
451 + MergedColumnParallelLinearWithLoRA)
452 + assert manager.add_lora(model_lora)
453 + assert manager.add_lora(model_lora1)
454 +
455 + packed_lora = model_lora.get_lora("gate_up_proj")
456 + assert packed_lora and isinstance(packed_lora, PackedLoRALayerWeights)
457 +
458 + assert torch.allclose(packed_lora.lora_a[0],
459 + model_lora.get_lora("gate_proj").lora_a)
460 + assert torch.allclose(packed_lora.lora_b[0],
461 + model_lora.get_lora("gate_proj").lora_b)
462 + assert torch.allclose(packed_lora.lora_a[1],
463 + model_lora.get_lora("up_proj").lora_a)
464 + assert torch.allclose(packed_lora.lora_b[1],
465 + model_lora.get_lora("up_proj").lora_b)
466 +
467 + packed_lora1 = model_lora1.get_lora("gate_up_proj")
468 + assert packed_lora1 and isinstance(packed_lora1, PackedLoRALayerWeights)
469 +
470 + assert packed_lora1.lora_a[0] is None
471 + assert packed_lora1.lora_b[0] is None
472 + assert torch.allclose(packed_lora1.lora_a[1],
473 + model_lora1.get_lora("up_proj").lora_a)
474 + assert torch.allclose(packed_lora1.lora_b[1],
475 + model_lora1.get_lora("up_proj").lora_b)
tests/lora/test_punica.py new
+175
@@ -0,0 +1,175 @@
1 +# Based on code from https://github.com/punica-ai/punica
2 +
3 +import pytest
4 +import torch
5 +
6 +import vllm.lora.punica as punica
7 +
8 +
9 +def assert_close(a, b):
10 + rtol, atol = {
11 + torch.float16: (5e-3, 5e-3),
12 + torch.bfloat16: (3e-2, 2e-2),
13 + torch.float32: (None, None),
14 + }[a.dtype]
15 + torch.testing.assert_close(a, b, rtol=rtol, atol=atol)
16 +
17 +
18 +def _lora_ref_impl(
19 + y_final: torch.Tensor,
20 + x: torch.Tensor,
21 + wa_T_all: torch.Tensor,
22 + wb_T_all: torch.Tensor,
23 + indicies: torch.LongTensor,
24 + layer_idx: int,
25 + scale: float,
26 +):
27 + y_stage_1 = torch.empty(
28 + (x.size(0), wa_T_all.size(-2)),
29 + dtype=torch.float32,
30 + device=x.device,
31 + )
32 + bs = x.shape[0]
33 + s = torch.tensor(scale, dtype=torch.float32, device=x.device)
34 + for i, lora_idx in zip(range(bs), indicies.cpu().tolist()):
35 + xi = x[i].unsqueeze(0).to(torch.float32)
36 + wa = wa_T_all[lora_idx, layer_idx].transpose(-1, -2).to(torch.float32)
37 + wb = wb_T_all[lora_idx, layer_idx].transpose(-1, -2).to(torch.float32)
38 +
39 + tmp = xi @ wa
40 + y_stage_1[i] = tmp.squeeze(0)
41 + y_final[i] += (tmp @ wb).squeeze(0) * s
42 + return y_final, y_stage_1
43 +
44 +
45 +H1 = H2 = [
46 + 128, 256, 512, 1024, 1280, 2048, 2560, 2752, 3072, 3456, 3584, 4096, 5120,
47 + 5504, 5632, 6912, 7168, 8192, 9216, 10240, 11008, 13824, 14336, 32000,
48 + 32256, 32512, 32768, 33024
49 +]
50 +SEED = [0xabcdabcd987]
51 +
52 +
53 +@pytest.mark.parametrize("dtype_str", ["float16", "bfloat16"])
54 +@pytest.mark.parametrize("h1", H1)
55 +@pytest.mark.parametrize("h2", H2)
56 +@pytest.mark.parametrize("seed", SEED)
57 +@torch.inference_mode()
58 +def test_lora_correctness(dtype_str, h1, h2, seed):
59 + torch.manual_seed(seed)
60 + num_loras = 4
61 + num_layers = 1
62 + r = 8
63 + bs = 32
64 + scale = 0.123
65 + dtype = getattr(torch, dtype_str)
66 + device = torch.device("cuda")
67 +
68 + wa_T_all = torch.randn(num_loras,
69 + num_layers,
70 + r,
71 + h1,
72 + dtype=dtype,
73 + device=device)
74 + wb_T_all = torch.randn(num_loras,
75 + num_layers,
76 + h2,
77 + r,
78 + dtype=dtype,
79 + device=device)
80 + indices = torch.randint(num_loras, (bs, ), dtype=torch.long, device=device)
81 +
82 + for layer_idx in range(num_layers):
83 + x = torch.randn(bs, h1, dtype=dtype, device=device)
84 + y = torch.randn(bs, h2, dtype=dtype, device=device)
85 +
86 + y_ref = y.clone()
87 + _lora_ref_impl(y_ref, x, wa_T_all, wb_T_all, indices, layer_idx, scale)
88 +
89 + y_our = y.clone()
90 + punica.add_lora(y_our, x, wa_T_all, wb_T_all, indices, layer_idx,
91 + scale)
92 +
93 + assert_close(y_ref, y_our)
94 +
95 +
96 +@pytest.mark.parametrize("dtype_str", ["float16", "bfloat16"])
97 +@pytest.mark.parametrize("h1", H1)
98 +@pytest.mark.parametrize("h2", H2)
99 +@pytest.mark.parametrize("seed", SEED)
100 +@torch.inference_mode()
101 +def test_lora_correctness_slice(dtype_str, h1, h2, seed):
102 + if h2 % 3 != 0 or h2 // 3 not in H1:
103 + pytest.skip("h2 must be divisible by 3 and in supported shapes")
104 + torch.manual_seed(seed)
105 + num_loras = 4
106 + num_layers = 1
107 + r = 8
108 + bs = 32
109 + scale = 0.123
110 + dtype = getattr(torch, dtype_str)
111 + device = torch.device("cuda")
112 +
113 + wa_T_all_0 = torch.randn(num_loras,
114 + num_layers,
115 + r,
116 + h1,
117 + dtype=dtype,
118 + device=device)
119 + wa_T_all_1 = torch.randn(num_loras,
120 + num_layers,
121 + r,
122 + h1,
123 + dtype=dtype,
124 + device=device)
125 + wa_T_all_2 = torch.randn(num_loras,
126 + num_layers,
127 + r,
128 + h1,
129 + dtype=dtype,
130 + device=device)
131 + wb_T_all_0 = torch.randn(num_loras,
132 + num_layers,
133 + h2 // 3,
134 + r,
135 + dtype=dtype,
136 + device=device)
137 + wb_T_all_1 = torch.randn(num_loras,
138 + num_layers,
139 + h2 // 3,
140 + r,
141 + dtype=dtype,
142 + device=device)
143 + wb_T_all_2 = torch.randn(num_loras,
144 + num_layers,
145 + h2 // 3,
146 + r,
147 + dtype=dtype,
148 + device=device)
149 +
150 + indices = torch.randint(num_loras, (bs, ), dtype=torch.long, device=device)
151 +
152 + for layer_idx in range(num_layers):
153 + x = torch.randn(bs, h1, dtype=dtype, device=device)
154 + y = torch.randn(bs, h2, dtype=dtype, device=device)
155 + s = h2 // 3
156 +
157 + y_ref = y.clone()
158 + _lora_ref_impl(y_ref[:, :s], x, wa_T_all_0, wb_T_all_0, indices,
159 + layer_idx, scale)
160 + _lora_ref_impl(y_ref[:, s:s * 2], x, wa_T_all_1, wb_T_all_1, indices,
161 + layer_idx, scale)
162 + _lora_ref_impl(y_ref[:, s * 2:], x, wa_T_all_2, wb_T_all_2, indices,
163 + layer_idx, scale)
164 +
165 + y_our = y.clone()
166 + punica.add_lora_slice(y_our, x, wa_T_all_0, wb_T_all_0, indices,
167 + layer_idx, scale, 0, s)
168 + punica.add_lora_slice(y_our, x, wa_T_all_1, wb_T_all_1, indices,
169 + layer_idx, scale, s, s)
170 + punica.add_lora_slice(y_our, x, wa_T_all_2, wb_T_all_2, indices,
171 + layer_idx, scale, s * 2, s)
172 +
173 + assert_close(y_ref[:, :s], y_our[:, :s])
174 + assert_close(y_ref[:, s:s * 2], y_our[:, s:s * 2])
175 + assert_close(y_ref[:, s * 2:], y_our[:, s * 2:])
tests/lora/test_tokenizer.py new
+69
@@ -0,0 +1,69 @@
1 +import pytest
2 +from transformers import AutoTokenizer, PreTrainedTokenizerBase
3 +
4 +from vllm.lora.request import LoRARequest
5 +from vllm.transformers_utils.tokenizer import TokenizerGroup, get_lora_tokenizer
6 +
7 +
8 +@pytest.mark.asyncio
9 +async def test_transformers_tokenizer():
10 + reference_tokenizer = AutoTokenizer.from_pretrained("gpt2")
11 + tokenizer = TokenizerGroup(
12 + tokenizer_id="gpt2",
13 + enable_lora=False,
14 + max_num_seqs=1,
15 + max_input_length=None,
16 + )
17 + assert reference_tokenizer.encode("prompt") == tokenizer.encode(
18 + request_id="request_id", prompt="prompt", lora_request=None)
19 + assert reference_tokenizer.encode(
20 + "prompt") == await tokenizer.encode_async(request_id="request_id",
21 + prompt="prompt",
22 + lora_request=None)
23 + assert isinstance(tokenizer.get_lora_tokenizer(None),
24 + PreTrainedTokenizerBase)
25 + assert tokenizer.get_lora_tokenizer(
26 + None) == await tokenizer.get_lora_tokenizer_async(None)
27 +
28 +
29 +@pytest.mark.asyncio
30 +async def test_transformers_tokenizer_lora(sql_lora_files):
31 + reference_tokenizer = AutoTokenizer.from_pretrained(sql_lora_files)
32 + tokenizer = TokenizerGroup(
33 + tokenizer_id="gpt2",
34 + enable_lora=True,
35 + max_num_seqs=1,
36 + max_input_length=None,
37 + )
38 + lora_request = LoRARequest("1", 1, sql_lora_files)
39 + assert reference_tokenizer.encode("prompt") == tokenizer.encode(
40 + request_id="request_id", prompt="prompt", lora_request=lora_request)
41 + assert reference_tokenizer.encode(
42 + "prompt") == await tokenizer.encode_async(request_id="request_id",
43 + prompt="prompt",
44 + lora_request=lora_request)
45 + assert isinstance(tokenizer.get_lora_tokenizer(None),
46 + PreTrainedTokenizerBase)
47 + assert tokenizer.get_lora_tokenizer(
48 + None) == await tokenizer.get_lora_tokenizer_async(None)
49 +
50 + assert isinstance(tokenizer.get_lora_tokenizer(lora_request),
51 + PreTrainedTokenizerBase)
52 + assert tokenizer.get_lora_tokenizer(
53 + lora_request) != tokenizer.get_lora_tokenizer(None)
54 + assert tokenizer.get_lora_tokenizer(
55 + lora_request) == await tokenizer.get_lora_tokenizer_async(lora_request)
56 +
57 +
58 +def test_get_lora_tokenizer(sql_lora_files, tmpdir):
59 + lora_request = None
60 + tokenizer = get_lora_tokenizer(lora_request)
61 + assert not tokenizer
62 +
63 + lora_request = LoRARequest("1", 1, sql_lora_files)
64 + tokenizer = get_lora_tokenizer(lora_request)
65 + assert tokenizer.get_added_vocab()
66 +
67 + lora_request = LoRARequest("1", 1, str(tmpdir))
68 + tokenizer = get_lora_tokenizer(lora_request)
69 + assert not tokenizer
tests/lora/test_utils.py new
+172
@@ -0,0 +1,172 @@
1 +from collections import OrderedDict
2 +
3 +from torch import nn
4 +
5 +from vllm.utils import LRUCache
6 +from vllm.lora.utils import (parse_fine_tuned_lora_name, replace_submodule)
7 +
8 +
9 +def test_parse_fine_tuned_lora_name():
10 + fixture = {
11 + ("base_model.model.lm_head.lora_A.weight", "lm_head", True),
12 + ("base_model.model.lm_head.lora_B.weight", "lm_head", False),
13 + (
14 + "base_model.model.model.embed_tokens.lora_embedding_A",
15 + "model.embed_tokens",
16 + True,
17 + ),
18 + (
19 + "base_model.model.model.embed_tokens.lora_embedding_B",
20 + "model.embed_tokens",
21 + False,
22 + ),
23 + (
24 + "base_model.model.model.layers.9.mlp.down_proj.lora_A.weight",
25 + "model.layers.9.mlp.down_proj",
26 + True,
27 + ),
28 + (
29 + "base_model.model.model.layers.9.mlp.down_proj.lora_B.weight",
30 + "model.layers.9.mlp.down_proj",
31 + False,
32 + ),
33 + }
34 + for name, module_name, is_lora_a in fixture:
35 + assert (module_name, is_lora_a) == parse_fine_tuned_lora_name(name)
36 +
37 +
38 +def test_replace_submodule():
39 + model = nn.Sequential(
40 + OrderedDict([
41 + ("dense1", nn.Linear(764, 100)),
42 + ("act1", nn.ReLU()),
43 + ("dense2", nn.Linear(100, 50)),
44 + (
45 + "seq1",
46 + nn.Sequential(
47 + OrderedDict([
48 + ("dense1", nn.Linear(100, 10)),
49 + ("dense2", nn.Linear(10, 50)),
50 + ])),
51 + ),
52 + ("act2", nn.ReLU()),
53 + ("output", nn.Linear(50, 10)),
54 + ("outact", nn.Sigmoid()),
55 + ]))
56 +
57 + sigmoid = nn.Sigmoid()
58 +
59 + replace_submodule(model, "act1", sigmoid)
60 + assert dict(model.named_modules())["act1"] == sigmoid
61 +
62 + dense2 = nn.Linear(1, 5)
63 + replace_submodule(model, "seq1.dense2", dense2)
64 + assert dict(model.named_modules())["seq1.dense2"] == dense2
65 +
66 +
67 +class TestLRUCache(LRUCache):
68 +
69 + def _on_remove(self, key, value):
70 + if not hasattr(self, "_remove_counter"):
71 + self._remove_counter = 0
72 + self._remove_counter += 1
73 +
74 +
75 +def test_lru_cache():
76 + cache = TestLRUCache(3)
77 +
78 + cache.put(1, 1)
79 + assert len(cache) == 1
80 +
81 + cache.put(1, 1)
82 + assert len(cache) == 1
83 +
84 + cache.put(2, 2)
85 + assert len(cache) == 2
86 +
87 + cache.put(3, 3)
88 + assert len(cache) == 3
89 + assert set(cache.cache) == {1, 2, 3}
90 +
91 + cache.put(4, 4)
92 + assert len(cache) == 3
93 + assert set(cache.cache) == {2, 3, 4}
94 + assert cache._remove_counter == 1
95 + assert cache.get(2) == 2
96 +
97 + cache.put(5, 5)
98 + assert set(cache.cache) == {2, 4, 5}
99 + assert cache._remove_counter == 2
100 +
101 + assert cache.pop(5) == 5
102 + assert len(cache) == 2
103 + assert set(cache.cache) == {2, 4}
104 + assert cache._remove_counter == 3
105 +
106 + cache.pop(10)
107 + assert len(cache) == 2
108 + assert set(cache.cache) == {2, 4}
109 + assert cache._remove_counter == 3
110 +
111 + cache.get(10)
112 + assert len(cache) == 2
113 + assert set(cache.cache) == {2, 4}
114 + assert cache._remove_counter == 3
115 +
116 + cache.put(6, 6)
117 + assert len(cache) == 3
118 + assert set(cache.cache) == {2, 4, 6}
119 + assert 2 in cache
120 + assert 4 in cache
121 + assert 6 in cache
122 +
123 + cache.remove_oldest()
124 + assert len(cache) == 2
125 + assert set(cache.cache) == {2, 6}
126 + assert cache._remove_counter == 4
127 +
128 + cache.clear()
129 + assert len(cache) == 0
130 + assert cache._remove_counter == 6
131 +
132 + cache._remove_counter = 0
133 +
134 + cache[1] = 1
135 + assert len(cache) == 1
136 +
137 + cache[1] = 1
138 + assert len(cache) == 1
139 +
140 + cache[2] = 2
141 + assert len(cache) == 2
142 +
143 + cache[3] = 3
144 + assert len(cache) == 3
145 + assert set(cache.cache) == {1, 2, 3}
146 +
147 + cache[4] = 4
148 + assert len(cache) == 3
149 + assert set(cache.cache) == {2, 3, 4}
150 + assert cache._remove_counter == 1
151 + assert cache[2] == 2
152 +
153 + cache[5] = 5
154 + assert set(cache.cache) == {2, 4, 5}
155 + assert cache._remove_counter == 2
156 +
157 + del cache[5]
158 + assert len(cache) == 2
159 + assert set(cache.cache) == {2, 4}
160 + assert cache._remove_counter == 3
161 +
162 + cache.pop(10)
163 + assert len(cache) == 2
164 + assert set(cache.cache) == {2, 4}
165 + assert cache._remove_counter == 3
166 +
167 + cache[6] = 6
168 + assert len(cache) == 3
169 + assert set(cache.cache) == {2, 4, 6}
170 + assert 2 in cache
171 + assert 4 in cache
172 + assert 6 in cache
tests/lora/test_worker.py new
+61
@@ -0,0 +1,61 @@
1 +import os
2 +import random
3 +import tempfile
4 +from unittest.mock import patch
5 +
6 +from vllm.lora.models import LoRAMapping
7 +from vllm.lora.request import LoRARequest
8 +from vllm.config import ModelConfig, ParallelConfig, SchedulerConfig, LoRAConfig
9 +from vllm.worker.worker import Worker
10 +
11 +
12 +@patch.dict(os.environ, {"RANK": "0"})
13 +def test_worker_apply_lora(sql_lora_files):
14 + worker = Worker(
15 + model_config=ModelConfig(
16 + "meta-llama/Llama-2-7b-hf",
17 + "meta-llama/Llama-2-7b-hf",
18 + tokenizer_mode="auto",
19 + trust_remote_code=False,
20 + download_dir=None,
21 + load_format="dummy",
22 + seed=0,
23 + dtype="float16",
24 + revision=None,
25 + ),
26 + parallel_config=ParallelConfig(1, 1, False),
27 + scheduler_config=SchedulerConfig(32, 32, 32, 256),
28 + local_rank=0,
29 + rank=0,
30 + lora_config=LoRAConfig(max_lora_rank=8, max_cpu_loras=32,
31 + max_loras=32),
32 + distributed_init_method=f"file://{tempfile.mkstemp()[1]}",
33 + )
34 + worker.init_model()
35 + worker.load_model()
36 +
37 + worker.model_runner.set_active_loras([], LoRAMapping([], []))
38 + assert worker.list_loras() == set()
39 +
40 + n_loras = 32
41 + lora_requests = [
42 + LoRARequest(str(i + 1), i + 1, sql_lora_files) for i in range(n_loras)
43 + ]
44 +
45 + worker.model_runner.set_active_loras(lora_requests, LoRAMapping([], []))
46 + assert worker.list_loras() == {
47 + lora_request.lora_int_id
48 + for lora_request in lora_requests
49 + }
50 +
51 + for i in range(32):
52 + random.seed(i)
53 + iter_lora_requests = random.choices(lora_requests,
54 + k=random.randint(1, n_loras))
55 + random.shuffle(iter_lora_requests)
56 + iter_lora_requests = iter_lora_requests[:-random.randint(0, n_loras)]
57 + worker.model_runner.set_active_loras(iter_lora_requests,
58 + LoRAMapping([], []))
59 + assert worker.list_loras().issuperset(
60 + {lora_request.lora_int_id
61 + for lora_request in iter_lora_requests})
tests/lora/utils.py new
+88
@@ -0,0 +1,88 @@
1 +from typing import List, Optional
2 +
3 +import torch
4 +
5 +from vllm.lora.lora import LoRALayerWeights, PackedLoRALayerWeights
6 +
7 +
8 +class DummyLoRAManager:
9 +
10 + def __init__(self):
11 + super().__init__()
12 + self._loras = {}
13 +
14 + def set_module_lora(self, module_name: str, lora: LoRALayerWeights):
15 + self._loras[module_name] = lora
16 +
17 + def get_module_lora(self, module_name: str) -> Optional[LoRALayerWeights]:
18 + return self._loras.get(module_name, None)
19 +
20 + def init_random_lora(self,
21 + module_name: str,
22 + weight: torch.Tensor,
23 + rank: int = 8,
24 + generate_embeddings_tensor: int = 0):
25 + lora = LoRALayerWeights(
26 + module_name,
27 + rank=rank,
28 + lora_alpha=1,
29 + lora_a=torch.rand([weight.shape[1], rank],
30 + dtype=weight.dtype,
31 + device="cuda"),
32 + lora_b=torch.rand([rank, weight.shape[0]],
33 + dtype=weight.dtype,
34 + device="cuda"),
35 + )
36 + if generate_embeddings_tensor:
37 + lora.embeddings_tensor = torch.rand(5,
38 + generate_embeddings_tensor,
39 + dtype=weight.dtype,
40 + device="cuda")
41 + self.set_module_lora(module_name, lora)
42 +
43 + return lora
44 +
45 + def init_lora(self,
46 + module_name: str,
47 + input_dim: int,
48 + output_dim: int,
49 + rank=8,
50 + noop=False,
51 + embeddings_tensor=None):
52 + lora = LoRALayerWeights(
53 + module_name,
54 + rank=rank,
55 + lora_alpha=1,
56 + lora_a=torch.rand([input_dim, rank], device="cuda"),
57 + lora_b=torch.rand([rank, output_dim], device="cuda"),
58 + embeddings_tensor=embeddings_tensor,
59 + )
60 + self.set_module_lora(module_name, lora)
61 + return lora
62 +
63 + def reset_lora(self):
64 + self._loras = {}
65 +
66 + def init_packed_lora(
67 + self,
68 + module_name: str,
69 + input_dim: int,
70 + output_dims: List[int],
71 + noop_lora_index: List[int] = None,
72 + rank=8,
73 + ):
74 + base_loras = []
75 + noop_lora_index = set(noop_lora_index or [])
76 +
77 + for i, out_dim in enumerate(output_dims):
78 + base_lora = self.init_lora(
79 + module_name + "_000_" + str(i),
80 + input_dim,
81 + out_dim,
82 + rank=rank,
83 + noop=i in noop_lora_index,
84 + )
85 + base_loras.append(base_lora)
86 + packed_lora = PackedLoRALayerWeights.pack(base_loras)
87 + self.set_module_lora(module_name, packed_lora)
88 + return packed_lora
tests/samplers/test_sampler.py
+7 -6
@@ -19,10 +19,11 @@ class MockLogitsSampler(Sampler):
19 self.fake_logits = fake_logits
20
21 def forward(self, *args, **kwargs):
22 - with patch("vllm.model_executor.layers.sampler._prune_hidden_states",
23 - lambda x, y: x), patch(
24 - "vllm.model_executor.layers.sampler._get_logits",
25 - lambda *args, **kwargs: self.fake_logits):
22 + with patch(
23 + "vllm.model_executor.layers.sampler._prune_hidden_states",
24 + lambda x, y: x), patch(
25 + "vllm.model_executor.layers.sampler.Sampler._get_logits",
26 + lambda *args, **kwargs: self.fake_logits):
27 return super().forward(*args, **kwargs)
28
29
@@ -38,7 +39,7 @@ def _prepare_test(
39 device=input_tensor.device,
40 dtype=input_tensor.dtype)
41 sampler = MockLogitsSampler(32000, fake_logits)
41 - model_runner = ModelRunner(None, None, None)
42 + model_runner = ModelRunner(None, None, None, None)
43 return input_tensor, fake_logits, sampler, model_runner
44
45
@@ -266,7 +267,7 @@ def test_sampler_top_k_top_p(seed: int):
267 device=input_tensor.device,
268 dtype=input_tensor.dtype)
269 sampler = MockLogitsSampler(32000, fake_logits)
269 - model_runner = ModelRunner(None, None, None)
270 + model_runner = ModelRunner(None, None, None, None)
271
272 generation_model = GenerationMixin()
273 generation_config = GenerationConfig(top_k=top_k,
tests/worker/spec_decode/utils.py
+2 -2
@@ -83,8 +83,8 @@ def create_worker(cls: type,
83 enforce_eager=enforce_eager,
84 )
85
86 - (model_config, cache_config, parallel_config,
87 - scheduler_config) = engine_args.create_engine_configs()
86 + (model_config, cache_config, parallel_config, scheduler_config,
87 + _) = engine_args.create_engine_configs()
88
89 distributed_init_method = get_distributed_init_method(
90 get_ip(), get_open_port())
tests/worker/test_model_runner.py
+2 -2
@@ -6,7 +6,7 @@ from vllm.worker.model_runner import ModelRunner
6
7
8 def test_prepare_prompt():
9 - model_runner = ModelRunner(None, None, None)
9 + model_runner = ModelRunner(None, None, None, None)
10 model_runner.set_block_size(16)
11
12 batch_size = random.randint(1, 256)
@@ -33,7 +33,7 @@ def test_prepare_prompt():
33 expected_selected_token_indices.append(selected_token_start_idx +
34 prompt_len - 1)
35 selected_token_start_idx += max_seq_len
36 - input_tokens, input_positions, _, return_prompt_lens, _ = (
36 + input_tokens, input_positions, _, return_prompt_lens, _, _, _, _ = (
37 model_runner._prepare_prompt(seq_group_metadata_list))
38 assert return_prompt_lens == prompt_lens
39 sampling_metadata = model_runner._prepare_sample(seq_group_metadata_list,
vllm/config.py
+50 -1
@@ -1,4 +1,5 @@
1 -from typing import Optional, Union
1 +from typing import Optional, Union, ClassVar
2 +from dataclasses import dataclass
3 import os
4
5 import torch
@@ -397,6 +398,54 @@ class SchedulerConfig:
398 f"({self.max_num_seqs}).")
399
400
401 +@dataclass
402 +class LoRAConfig:
403 + max_lora_rank: int
404 + max_loras: int
405 + max_cpu_loras: Optional[int] = None
406 + lora_dtype: Optional[torch.dtype] = None
407 + lora_extra_vocab_size: int = 256
408 + # This is a constant.
409 + lora_vocab_padding_size: ClassVar[int] = 256
410 +
411 + def __post_init__(self):
412 + # Keep this in sync with csrc/punica/bgmv/bgmv_config.h
413 + possible_max_ranks = (8, 16, 32, 64)
414 + possible_lora_extra_vocab_size = (0, 256, 512)
415 + if self.max_lora_rank not in possible_max_ranks:
416 + raise ValueError(
417 + f"max_lora_rank ({self.max_lora_rank}) must be one of "
418 + f"{possible_max_ranks}.")
419 + if self.lora_extra_vocab_size not in possible_lora_extra_vocab_size:
420 + raise ValueError(
421 + f"lora_extra_vocab_size ({self.lora_extra_vocab_size}) "
422 + f"must be one of {possible_lora_extra_vocab_size}.")
423 + if self.max_loras < 1:
424 + raise ValueError(f"max_loras ({self.max_loras}) must be >= 1.")
425 + if self.max_cpu_loras is None:
426 + self.max_cpu_loras = self.max_loras
427 + elif self.max_cpu_loras < self.max_loras:
428 + raise ValueError(
429 + f"max_cpu_loras ({self.max_cpu_loras}) must be >= "
430 + f"max_num_seqs ({self.max_loras})")
431 +
432 + def verify_with_model_config(self, model_config: ModelConfig):
433 + if self.lora_dtype in (None, "auto"):
434 + self.lora_dtype = model_config.dtype
435 + elif isinstance(self.lora_dtype, str):
436 + self.lora_dtype = getattr(torch, self.lora_dtype)
437 + if model_config.quantization is not None:
438 + raise ValueError(
439 + "LoRA is not supported with quantized models yet.")
440 +
441 + def verify_with_scheduler_config(self, scheduler_config: SchedulerConfig):
442 + if scheduler_config.max_num_batched_tokens > 65528:
443 + raise ValueError(
444 + "Due to limitations of the custom LoRA CUDA kernel, "
445 + "max_num_batched_tokens must be <= 65528 when "
446 + "LoRA is enabled.")
447 +
448 +
449 _STR_DTYPE_TO_TORCH_DTYPE = {
450 "half": torch.float16,
451 "float16": torch.float16,
vllm/core/scheduler.py
+68 -5
@@ -1,11 +1,12 @@
1 from collections import deque
2 import enum
3 import time
4 -from typing import Deque, Dict, Iterable, List, Optional, Tuple, Union
4 +from typing import Deque, Dict, Iterable, List, Optional, Tuple, Union, Set
5
6 -from vllm.config import CacheConfig, SchedulerConfig
6 +from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig
7 from vllm.core.block_manager import AllocStatus, BlockSpaceManager
8 from vllm.core.policy import PolicyFactory
9 +from vllm.lora.request import LoRARequest
10 from vllm.logger import init_logger
11 from vllm.sequence import (Sequence, SequenceData, SequenceGroup,
12 SequenceGroupMetadata, SequenceStatus)
@@ -49,11 +50,25 @@ class SchedulerOutputs:
50 assert not (blocks_to_swap_in and blocks_to_swap_out)
51 self.ignored_seq_groups = ignored_seq_groups
52
53 + self.num_loras = len(self.lora_requests)
54 + if self.num_loras > 0:
55 + self._sort_by_lora_ids()
56 +
57 def is_empty(self) -> bool:
58 # NOTE: We do not consider the ignored sequence groups.
59 return (not self.scheduled_seq_groups and not self.blocks_to_swap_in
60 and not self.blocks_to_swap_out and not self.blocks_to_copy)
61
62 + def _sort_by_lora_ids(self) -> bool:
63 + self.scheduled_seq_groups = sorted(
64 + self.scheduled_seq_groups,
65 + key=lambda g: (g.lora_request.lora_int_id
66 + if g.lora_request else 0, g.request_id))
67 +
68 + @property
69 + def lora_requests(self) -> Set[LoRARequest]:
70 + return {g.lora_request for g in self.scheduled_seq_groups}
71 +
72
73 class Scheduler:
74
@@ -61,9 +76,14 @@ class Scheduler:
76 self,
77 scheduler_config: SchedulerConfig,
78 cache_config: CacheConfig,
79 + lora_config: Optional[LoRAConfig],
80 ) -> None:
81 self.scheduler_config = scheduler_config
82 self.cache_config = cache_config
83 + # Note for LoRA scheduling: the current policy is extremely
84 + # simple and NOT fair. It can lead to starvation of some
85 + # LoRAs. This should be improved in the future.
86 + self.lora_config = lora_config
87
88 self.prompt_limit = min(self.scheduler_config.max_model_len,
89 self.scheduler_config.max_num_batched_tokens)
@@ -87,6 +107,10 @@ class Scheduler:
107 # Sequence groups in the SWAPPED state.
108 self.swapped: Deque[SequenceGroup] = deque()
109
110 + @property
111 + def lora_enabled(self) -> bool:
112 + return bool(self.lora_config)
113 +
114 def add_seq_group(self, seq_group: SequenceGroup) -> None:
115 # Add sequence groups to the waiting queue.
116 self.waiting.append(seq_group)
@@ -150,14 +174,17 @@ class Scheduler:
174 # requests in the generation phase.
175 num_curr_seqs = sum(seq_group.get_max_num_running_seqs()
176 for seq_group in self.running)
177 + curr_loras = set(
178 + seq_group.lora_int_id
179 + for seq_group in self.running) if self.lora_enabled else None
180 seq_lens: List[int] = []
181
182 # Optimization: We do not sort the waiting queue since the preempted
183 # sequence groups are added to the front and the new sequence groups
184 # are added to the back.
185 + leftover_waiting_sequences = deque()
186 while self.waiting:
187 seq_group = self.waiting[0]
160 -
188 waiting_seqs = seq_group.get_seqs(
189 status=SequenceStatus.WAITING)
190 assert len(waiting_seqs) == 1, (
@@ -188,6 +215,17 @@ class Scheduler:
215 self.waiting.popleft()
216 continue
217
218 + lora_int_id = 0
219 + if self.lora_enabled:
220 + lora_int_id = seq_group.lora_int_id
221 + if lora_int_id > 0 and lora_int_id not in curr_loras and len(
222 + curr_loras) >= self.lora_config.max_loras:
223 + # We don't have a space for another LoRA, so
224 + # we ignore this request for now.
225 + leftover_waiting_sequences.appendleft(seq_group)
226 + self.waiting.popleft()
227 + continue
228 +
229 # If the number of batched tokens exceeds the limit, stop.
230 new_seq_lens = seq_lens + [num_prompt_tokens]
231 num_batched_tokens = len(new_seq_lens) * max(new_seq_lens)
@@ -207,12 +245,16 @@ class Scheduler:
245 break
246 seq_lens = new_seq_lens
247
210 - seq_group = self.waiting.popleft()
248 + if lora_int_id > 0:
249 + curr_loras.add(lora_int_id)
250 + self.waiting.popleft()
251 self._allocate(seq_group)
252 self.running.append(seq_group)
253 num_curr_seqs += num_new_seqs
254 scheduled.append(seq_group)
255
256 + self.waiting.extendleft(leftover_waiting_sequences)
257 +
258 if scheduled or ignored_seq_groups:
259 scheduler_outputs = SchedulerOutputs(
260 scheduled_seq_groups=scheduled,
@@ -260,9 +302,25 @@ class Scheduler:
302 if not preempted:
303 num_curr_seqs = sum(seq_group.get_max_num_running_seqs()
304 for seq_group in self.running)
305 + curr_loras = set(
306 + seq_group.lora_int_id
307 + for seq_group in self.running) if self.lora_enabled else None
308 +
309 + leftover_swapped = deque()
310
311 while self.swapped:
312 seq_group = self.swapped[0]
313 + lora_int_id = 0
314 + if self.lora_enabled:
315 + lora_int_id = seq_group.lora_int_id
316 + if lora_int_id > 0 and lora_int_id not in curr_loras and len(
317 + curr_loras) >= self.lora_config.max_loras:
318 + # We don't have a space for another LoRA, so
319 + # we ignore this request for now.
320 + leftover_swapped.appendleft(seq_group)
321 + self.swapped.popleft()
322 + continue
323 +
324 # If the sequence group cannot be swapped in, stop.
325 if not self.block_manager.can_swap_in(seq_group):
326 break
@@ -274,12 +332,16 @@ class Scheduler:
332 self.scheduler_config.max_num_seqs):
333 break
334
277 - seq_group = self.swapped.popleft()
335 + if lora_int_id > 0:
336 + curr_loras.add(lora_int_id)
337 + self.swapped.popleft()
338 self._swap_in(seq_group, blocks_to_swap_in)
339 self._append_slot(seq_group, blocks_to_copy)
340 num_curr_seqs += num_new_seqs
341 self.running.append(seq_group)
342
343 + self.swapped.extendleft(leftover_swapped)
344 +
345 # Each sequence in the generation phase only takes one token slot.
346 # Therefore, the number of batched tokens is equal to the number of
347 # sequences in the RUNNING state.
@@ -320,6 +382,7 @@ class Scheduler:
382 seq_data=seq_data,
383 sampling_params=seq_group.sampling_params,
384 block_tables=block_tables,
385 + lora_request=seq_group.lora_request,
386 prefix=seq_group.prefix,
387 )
388 seq_group_metadata_list.append(seq_group_metadata)
vllm/engine/arg_utils.py
+50 -3
@@ -4,7 +4,7 @@ from dataclasses import dataclass
4 from typing import Optional, Tuple
5
6 from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
7 - SchedulerConfig)
7 + SchedulerConfig, LoRAConfig)
8
9
10 @dataclass
@@ -35,6 +35,12 @@ class EngineArgs:
35 quantization: Optional[str] = None
36 enforce_eager: bool = False
37 max_context_len_to_capture: int = 8192
38 + enable_lora: bool = False
39 + max_loras: int = 1
40 + max_lora_rank: int = 16
41 + lora_extra_vocab_size: int = 256
42 + lora_dtype = 'auto'
43 + max_cpu_loras: Optional[int] = None
44
45 def __post_init__(self):
46 if self.tokenizer is None:
@@ -202,6 +208,39 @@ class EngineArgs:
208 help='maximum context length covered by CUDA '
209 'graphs. When a sequence has context length '
210 'larger than this, we fall back to eager mode.')
211 + # LoRA related configs
212 + parser.add_argument('--enable-lora',
213 + action='store_true',
214 + help='If True, enable handling of LoRA adapters.')
215 + parser.add_argument('--max-loras',
216 + type=int,
217 + default=EngineArgs.max_loras,
218 + help='Max number of LoRAs in a single batch.')
219 + parser.add_argument('--max-lora-rank',
220 + type=int,
221 + default=EngineArgs.max_lora_rank,
222 + help='Max LoRA rank.')
223 + parser.add_argument(
224 + '--lora-extra-vocab-size',
225 + type=int,
226 + default=EngineArgs.lora_extra_vocab_size,
227 + help=('Maximum size of extra vocabulary that can be '
228 + 'present in a LoRA adapter (added to the base '
229 + 'model vocabulary).'))
230 + parser.add_argument(
231 + '--lora-dtype',
232 + type=str,
233 + default=EngineArgs.lora_dtype,
234 + choices=['auto', 'float16', 'bfloat16', 'float32'],
235 + help=('Data type for LoRA. If auto, will default to '
236 + 'base model dtype.'))
237 + parser.add_argument(
238 + '--max-cpu-loras',
239 + type=int,
240 + default=EngineArgs.max_cpu_loras,
241 + help=('Maximum number of LoRAs to store in CPU memory. '
242 + 'Must be >= than max_num_seqs. '
243 + 'Defaults to max_num_seqs.'))
244 return parser
245
246 @classmethod
@@ -214,7 +253,8 @@ class EngineArgs:
253
254 def create_engine_configs(
255 self,
217 - ) -> Tuple[ModelConfig, CacheConfig, ParallelConfig, SchedulerConfig]:
256 + ) -> Tuple[ModelConfig, CacheConfig, ParallelConfig, SchedulerConfig,
257 + Optional[LoRAConfig]]:
258 model_config = ModelConfig(self.model, self.tokenizer,
259 self.tokenizer_mode, self.trust_remote_code,
260 self.download_dir, self.load_format,
@@ -234,7 +274,14 @@ class EngineArgs:
274 self.max_num_seqs,
275 model_config.max_model_len,
276 self.max_paddings)
237 - return model_config, cache_config, parallel_config, scheduler_config
277 + lora_config = LoRAConfig(
278 + max_lora_rank=self.max_lora_rank,
279 + max_loras=self.max_loras,
280 + lora_extra_vocab_size=self.lora_extra_vocab_size,
281 + lora_dtype=self.lora_dtype,
282 + max_cpu_loras=self.max_cpu_loras if self.max_cpu_loras
283 + and self.max_cpu_loras > 0 else None) if self.enable_lora else None
284 + return model_config, cache_config, parallel_config, scheduler_config, lora_config
285
286
287 @dataclass
vllm/engine/async_llm_engine.py
+71 -8
@@ -4,6 +4,7 @@ from functools import partial
4 from typing import (Any, Dict, Iterable, List, Optional, Set, Tuple, Type,
5 Union, AsyncIterator)
6
7 +from vllm.lora.request import LoRARequest
8 from vllm.config import ModelConfig
9 from vllm.engine.arg_utils import AsyncEngineArgs
10 from vllm.engine.llm_engine import LLMEngine
@@ -203,6 +204,52 @@ class _AsyncLLMEngine(LLMEngine):
204
205 return self._process_model_outputs(output, scheduler_outputs)
206
207 + async def encode_request_async(
208 + self,
209 + request_id: str, # pylint: disable=unused-argument
210 + prompt: Optional[str],
211 + prompt_token_ids: Optional[List[int]] = None,
212 + lora_request: Optional[LoRARequest] = None,
213 + ):
214 + if prompt_token_ids is None:
215 + assert prompt is not None
216 + prompt_token_ids = await self.tokenizer.encode_async(
217 + request_id=request_id,
218 + prompt=prompt,
219 + lora_request=lora_request)
220 + return prompt_token_ids
221 +
222 + async def add_request_async(
223 + self,
224 + request_id: str,
225 + prompt: Optional[str],
226 + sampling_params: SamplingParams,
227 + prompt_token_ids: Optional[List[int]] = None,
228 + arrival_time: Optional[float] = None,
229 + lora_request: Optional[LoRARequest] = None,
230 + prefix_pos: Optional[int] = None,
231 + ) -> None:
232 + if lora_request is not None and not self.lora_config:
233 + raise ValueError(f"Got lora_request {lora_request} but LoRA is "
234 + "not enabled!")
235 + if arrival_time is None:
236 + arrival_time = time.time()
237 + prompt_token_ids = await self.encode_request_async(
238 + request_id=request_id,
239 + prompt=prompt,
240 + prompt_token_ids=prompt_token_ids,
241 + lora_request=lora_request)
242 +
243 + return self.add_request(
244 + request_id,
245 + prompt=prompt,
246 + prompt_token_ids=prompt_token_ids,
247 + sampling_params=sampling_params,
248 + arrival_time=arrival_time,
249 + lora_request=lora_request,
250 + prefix_pos=prefix_pos,
251 + )
252 +
253 async def _run_workers_async(
254 self,
255 method: str,
@@ -332,7 +379,7 @@ class AsyncLLMEngine:
379 if self.engine_use_ray:
380 await self.engine.add_request.remote(**new_request)
381 else:
335 - self.engine.add_request(**new_request)
382 + await self.engine.add_request_async(**new_request)
383
384 if finished_requests:
385 await self._engine_abort(finished_requests)
@@ -371,6 +418,7 @@ class AsyncLLMEngine:
418 sampling_params: SamplingParams,
419 prompt_token_ids: Optional[List[int]] = None,
420 arrival_time: Optional[float] = None,
421 + lora_request: Optional[LoRARequest] = None,
422 prefix_pos: Optional[int] = None,
423 ) -> AsyncStream:
424 if self.log_requests:
@@ -386,7 +434,8 @@ class AsyncLLMEngine:
434 f"prompt: {shortened_prompt!r}, "
435 f"prefix_pos: {prefix_pos},"
436 f"sampling params: {sampling_params}, "
389 - f"prompt token ids: {shortened_token_ids}.")
437 + f"prompt token ids: {shortened_token_ids}, "
438 + f"lora_request: {lora_request}.")
439
440 if not self.is_running:
441 if self.start_engine_loop:
@@ -398,12 +447,21 @@ class AsyncLLMEngine:
447 "error that caused the background loop to stop "
448 "(AsyncEngineDeadError).")
449
450 + if arrival_time is None:
451 + arrival_time = time.time()
452 + prompt_token_ids = await self.engine.encode_request_async(
453 + request_id=request_id,
454 + prompt=prompt,
455 + prompt_token_ids=prompt_token_ids,
456 + lora_request=lora_request)
457 +
458 stream = self._request_tracker.add_request(
459 request_id,
460 prompt=prompt,
461 sampling_params=sampling_params,
462 prompt_token_ids=prompt_token_ids,
463 arrival_time=arrival_time,
464 + lora_request=lora_request,
465 prefix_pos=prefix_pos)
466
467 return stream
@@ -414,6 +472,7 @@ class AsyncLLMEngine:
472 sampling_params: SamplingParams,
473 request_id: str,
474 prompt_token_ids: Optional[List[int]] = None,
475 + lora_request: Optional[LoRARequest] = None,
476 prefix_pos: Optional[int] = None,
477 ) -> AsyncIterator[RequestOutput]:
478 """Generate outputs for a request.
@@ -429,6 +488,7 @@ class AsyncLLMEngine:
488 request_id: The unique id of the request.
489 prompt_token_ids: The token IDs of the prompt. If None, we
490 use the tokenizer to convert the prompts to token IDs.
491 + lora_request: LoRA request to use for generation, if any.
492 prefix_pos: If not None, we use the given position as the prefix
493 position for each prompt. We will cache the prefix's KV
494 cache and reuse it for the next request with the same prefix.
@@ -487,12 +547,15 @@ class AsyncLLMEngine:
547 arrival_time = time.monotonic()
548
549 try:
490 - stream = await self.add_request(request_id,
491 - prompt,
492 - sampling_params,
493 - prompt_token_ids=prompt_token_ids,
494 - arrival_time=arrival_time,
495 - prefix_pos=prefix_pos)
550 + stream = await self.add_request(
551 + request_id,
552 + prompt,
553 + sampling_params,
554 + prompt_token_ids=prompt_token_ids,
555 + arrival_time=arrival_time,
556 + lora_request=lora_request,
557 + prefix_pos=prefix_pos,
558 + )
559
560 async for request_output in stream:
561 yield request_output
vllm/engine/llm_engine.py
+87 -24
@@ -5,8 +5,9 @@ import time
5 from typing import (TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Tuple,
6 Union)
7
8 +from vllm.lora.request import LoRARequest
9 from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
9 - SchedulerConfig)
10 + SchedulerConfig, LoRAConfig)
11 from vllm.core.scheduler import Scheduler, SchedulerOutputs
12 from vllm.engine.arg_utils import EngineArgs
13 from vllm.engine.metrics import record_metrics
@@ -17,7 +18,7 @@ from vllm.sampling_params import SamplingParams
18 from vllm.sequence import (SamplerOutput, Sequence, SequenceGroup,
19 SequenceGroupOutput, SequenceOutput, SequenceStatus)
20 from vllm.transformers_utils.tokenizer import (detokenize_incrementally,
20 - get_tokenizer)
21 + TokenizerGroup)
22 from vllm.utils import Counter, set_cuda_visible_devices, get_ip, get_open_port, get_distributed_init_method
23
24 if ray:
@@ -64,6 +65,7 @@ class LLMEngine:
65 cache_config: CacheConfig,
66 parallel_config: ParallelConfig,
67 scheduler_config: SchedulerConfig,
68 + lora_config: Optional[LoRAConfig],
69 placement_group: Optional["PlacementGroup"],
70 log_stats: bool,
71 ) -> None:
@@ -87,17 +89,13 @@ class LLMEngine:
89
90 self.model_config = model_config
91 self.cache_config = cache_config
92 + self.lora_config = lora_config
93 self.parallel_config = parallel_config
94 self.scheduler_config = scheduler_config
95 self.log_stats = log_stats
96 self._verify_args()
97
95 - self.tokenizer = get_tokenizer(
96 - model_config.tokenizer,
97 - tokenizer_mode=model_config.tokenizer_mode,
98 - trust_remote_code=model_config.trust_remote_code,
99 - tokenizer_revision=model_config.tokenizer_revision,
100 - revision=model_config.revision)
98 + self._init_tokenizer()
99 self.seq_counter = Counter()
100
101 # Create the parallel GPU workers.
@@ -114,7 +112,7 @@ class LLMEngine:
112 self._init_cache()
113
114 # Create the scheduler.
117 - self.scheduler = Scheduler(scheduler_config, cache_config)
115 + self.scheduler = Scheduler(scheduler_config, cache_config, lora_config)
116
117 # Logging.
118 self.last_logging_time = 0.0
@@ -123,6 +121,9 @@ class LLMEngine:
121 # List of (timestamp, num_tokens)
122 self.num_generation_tokens: List[Tuple[float, int]] = []
123
124 + def get_tokenizer_for_seq(self, sequence: Sequence):
125 + return self.tokenizer.get_lora_tokenizer(sequence.lora_request)
126 +
127 def _init_workers(self):
128 # Lazy import the Worker to avoid importing torch.cuda/xformers
129 # before CUDA_VISIBLE_DEVICES is set in the Worker
@@ -141,11 +142,24 @@ class LLMEngine:
142 local_rank=0,
143 rank=0,
144 distributed_init_method=distributed_init_method,
145 + lora_config=self.lora_config,
146 is_driver_worker=True,
147 )
148 self._run_workers("init_model")
149 self._run_workers("load_model")
150
151 + def _init_tokenizer(self, **tokenizer_init_kwargs):
152 + init_kwargs = dict(
153 + enable_lora=bool(self.lora_config),
154 + max_num_seqs=self.scheduler_config.max_num_seqs,
155 + max_input_length=None,
156 + tokenizer_mode=self.model_config.tokenizer_mode,
157 + trust_remote_code=self.model_config.trust_remote_code,
158 + revision=self.model_config.tokenizer_revision)
159 + init_kwargs.update(tokenizer_init_kwargs)
160 + self.tokenizer: TokenizerGroup = TokenizerGroup(
161 + self.model_config.tokenizer, **init_kwargs)
162 +
163 def _init_workers_ray(self, placement_group: "PlacementGroup",
164 **ray_remote_kwargs):
165 if self.parallel_config.tensor_parallel_size == 1:
@@ -233,6 +247,7 @@ class LLMEngine:
247 local_rank,
248 rank,
249 distributed_init_method,
250 + lora_config=self.lora_config,
251 ))
252
253 driver_rank = 0
@@ -244,6 +259,7 @@ class LLMEngine:
259 driver_local_rank,
260 driver_rank,
261 distributed_init_method,
262 + lora_config=self.lora_config,
263 is_driver_worker=True,
264 )
265
@@ -257,6 +273,10 @@ class LLMEngine:
273 def _verify_args(self) -> None:
274 self.model_config.verify_with_parallel_config(self.parallel_config)
275 self.cache_config.verify_with_parallel_config(self.parallel_config)
276 + if self.lora_config:
277 + self.lora_config.verify_with_model_config(self.model_config)
278 + self.lora_config.verify_with_scheduler_config(
279 + self.scheduler_config)
280
281 def _init_cache(self) -> None:
282 """Profiles the memory usage and initializes the KV cache.
@@ -332,6 +352,20 @@ class LLMEngine:
352 log_stats=not engine_args.disable_log_stats)
353 return engine
354
355 + def encode_request(
356 + self,
357 + request_id: str, # pylint: disable=unused-argument
358 + prompt: Optional[str],
359 + prompt_token_ids: Optional[List[int]] = None,
360 + lora_request: Optional[LoRARequest] = None,
361 + ):
362 + if prompt_token_ids is None:
363 + assert prompt is not None
364 + prompt_token_ids = self.tokenizer.encode(request_id=request_id,
365 + prompt=prompt,
366 + lora_request=lora_request)
367 + return prompt_token_ids
368 +
369 def add_request(
370 self,
371 request_id: str,
@@ -339,6 +373,7 @@ class LLMEngine:
373 sampling_params: SamplingParams,
374 prompt_token_ids: Optional[List[int]] = None,
375 arrival_time: Optional[float] = None,
376 + lora_request: Optional[LoRARequest] = None,
377 prefix_pos: Optional[int] = None,
378 ) -> None:
379 """Add a request to the engine's request pool.
@@ -386,24 +421,31 @@ class LLMEngine:
421 >>> # continue the request processing
422 >>> ...
423 """
424 + if lora_request is not None and not self.lora_config:
425 + raise ValueError(f"Got lora_request {lora_request} but LoRA is "
426 + "not enabled!")
427 if arrival_time is None:
428 arrival_time = time.monotonic()
391 - if prompt_token_ids is None:
392 - assert prompt is not None
393 - prompt_token_ids = self.tokenizer.encode(prompt)
429 + prompt_token_ids = self.encode_request(
430 + request_id=request_id,
431 + prompt=prompt,
432 + prompt_token_ids=prompt_token_ids,
433 + lora_request=lora_request)
434
435 # Create the sequences.
436 block_size = self.cache_config.block_size
437 seq_id = next(self.seq_counter)
398 - seq = Sequence(seq_id, prompt, prompt_token_ids, block_size)
438 + seq = Sequence(seq_id, prompt, prompt_token_ids, block_size,
439 + lora_request)
440
441 # Check whether the input specifies prefix
442 prefix = self.scheduler.prefix_pool.add_or_get_prefix(
402 - prompt_token_ids[:prefix_pos]) if prefix_pos is not None else None
443 + prompt_token_ids[:prefix_pos], lora_request.lora_int_id
444 + if lora_request else 0) if prefix_pos is not None else None
445
446 # Create the sequence group.
447 seq_group = SequenceGroup(request_id, [seq], sampling_params,
406 - arrival_time, prefix)
448 + arrival_time, lora_request, prefix)
449
450 # Add the sequence group to the scheduler.
451 self.scheduler.add_seq_group(seq_group)
@@ -453,11 +495,13 @@ class LLMEngine:
495
496 current_worst_score = (current_worst_seq.get_beam_search_score(
497 length_penalty=length_penalty,
456 - eos_token_id=self.tokenizer.eos_token_id))
498 + eos_token_id=self.get_tokenizer_for_seq(
499 + current_worst_seq).eos_token_id))
500 if early_stopping is False:
501 highest_attainable_score = (best_running_seq.get_beam_search_score(
502 length_penalty=length_penalty,
460 - eos_token_id=self.tokenizer.eos_token_id))
503 + eos_token_id=self.get_tokenizer_for_seq(
504 + best_running_seq).eos_token_id))
505 else:
506 assert early_stopping == "never"
507 if length_penalty > 0.0:
@@ -471,7 +515,8 @@ class LLMEngine:
515 highest_attainable_score = (
516 best_running_seq.get_beam_search_score(
517 length_penalty=length_penalty,
474 - eos_token_id=self.tokenizer.eos_token_id,
518 + eos_token_id=self.get_tokenizer_for_seq(
519 + best_running_seq).eos_token_id,
520 seq_len=max_possible_length))
521 else:
522 # Otherwise, beam search will prefer shorter sequences. The
@@ -480,7 +525,8 @@ class LLMEngine:
525 highest_attainable_score = (
526 best_running_seq.get_beam_search_score(
527 length_penalty=length_penalty,
483 - eos_token_id=self.tokenizer.eos_token_id))
528 + eos_token_id=self.get_tokenizer_for_seq(
529 + best_running_seq).eos_token_id))
530 return current_worst_score >= highest_attainable_score
531
532 def _process_sequence_group_outputs(self, seq_group: SequenceGroup,
@@ -571,7 +617,7 @@ class LLMEngine:
617 # Sort the finished sequences by their scores.
618 all_finished_seqs.sort(key=lambda x: x[0].get_beam_search_score(
619 length_penalty=length_penalty,
574 - eos_token_id=self.tokenizer.eos_token_id),
620 + eos_token_id=self.get_tokenizer_for_seq(x[0]).eos_token_id),
621 reverse=True)
622 for seq, parent, is_new in all_finished_seqs[:beam_width]:
623 if is_new:
@@ -599,7 +645,7 @@ class LLMEngine:
645 # Sort the running sequences by their scores.
646 running_child_seqs.sort(key=lambda x: x[0].get_beam_search_score(
647 length_penalty=length_penalty,
602 - eos_token_id=self.tokenizer.eos_token_id),
648 + eos_token_id=self.get_tokenizer_for_seq(x[0]).eos_token_id),
649 reverse=True)
650
651 # Check if we can stop the beam search.
@@ -837,7 +883,7 @@ class LLMEngine:
883 """Decodes the new token for a sequence."""
884 (new_tokens, new_output_text, prefix_offset,
885 read_offset) = detokenize_incrementally(
840 - self.tokenizer,
886 + self.get_tokenizer_for_seq(seq),
887 all_input_ids=seq.get_token_ids(),
888 prev_tokens=seq.tokens,
889 prefix_offset=seq.prefix_offset,
@@ -879,11 +925,28 @@ class LLMEngine:
925 return
926
927 # Check if the sequence has generated the EOS token.
882 - if ((not sampling_params.ignore_eos)
883 - and seq.get_last_token_id() == self.tokenizer.eos_token_id):
928 + if ((not sampling_params.ignore_eos) and seq.get_last_token_id()
929 + == self.get_tokenizer_for_seq(seq).eos_token_id):
930 seq.status = SequenceStatus.FINISHED_STOPPED
931 return
932
933 + def add_lora(self, lora_request: LoRARequest) -> bool:
934 + assert lora_request.lora_int_id > 0, "lora_id must be greater than 0."
935 + return self._run_workers(
936 + "add_lora",
937 + lora_request=lora_request,
938 + )
939 +
940 + def remove_lora(self, lora_id: int) -> bool:
941 + assert lora_id > 0, "lora_id must be greater than 0."
942 + return self._run_workers(
943 + "remove_lora",
944 + lora_id=lora_id,
945 + )
946 +
947 + def list_loras(self) -> List[int]:
948 + return self._run_workers("list_loras")
949 +
950 def _run_workers(
951 self,
952 method: str,
vllm/entrypoints/llm.py
+10 -1
@@ -3,6 +3,7 @@ from typing import List, Optional, Union
3 from tqdm import tqdm
4 from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
5
6 +from vllm.lora.request import LoRARequest
7 from vllm.engine.arg_utils import EngineArgs
8 from vllm.engine.llm_engine import LLMEngine
9 from vllm.outputs import RequestOutput
@@ -122,6 +123,7 @@ class LLM:
123 prompt_token_ids: Optional[List[List[int]]] = None,
124 prefix_pos: Optional[Union[int, List[int]]] = None,
125 use_tqdm: bool = True,
126 + lora_request: Optional[LoRARequest] = None,
127 ) -> List[RequestOutput]:
128 """Generates the completions for the input prompts.
129
@@ -141,6 +143,7 @@ class LLM:
143 This is an experimental feature, and may be replaced with
144 automatic prefix caching in the future.
145 use_tqdm: Whether to use tqdm to display the progress bar.
146 + lora_request: LoRA request to use for generation, if any.
147
148 Returns:
149 A list of `RequestOutput` objects containing the generated
@@ -168,7 +171,11 @@ class LLM:
171 prefix_pos_i = prefix_pos[i] if prefix_pos is not None else None
172 token_ids = None if prompt_token_ids is None else prompt_token_ids[
173 i]
171 - self._add_request(prompt, sampling_params, token_ids, prefix_pos_i)
174 + self._add_request(prompt,
175 + sampling_params,
176 + token_ids,
177 + lora_request=lora_request,
178 + prefix_pos=prefix_pos_i)
179 return self._run_engine(use_tqdm)
180
181 def _add_request(
@@ -176,6 +183,7 @@ class LLM:
183 prompt: Optional[str],
184 sampling_params: SamplingParams,
185 prompt_token_ids: Optional[List[int]],
186 + lora_request: Optional[LoRARequest] = None,
187 prefix_pos: Optional[int] = None,
188 ) -> None:
189 request_id = str(next(self.request_counter))
@@ -183,6 +191,7 @@ class LLM:
191 prompt,
192 sampling_params,
193 prompt_token_ids,
194 + lora_request=lora_request,
195 prefix_pos=prefix_pos)
196
197 def _run_engine(self, use_tqdm: bool) -> List[RequestOutput]:
vllm/lora/__init__.py
vllm/lora/layers.py new
+975
@@ -0,0 +1,975 @@
1 +# pylint: disable=unused-argument
2 +import math
3 +from dataclasses import dataclass
4 +from typing import TYPE_CHECKING, List, Optional, Tuple
5 +
6 +import torch
7 +import torch.nn as nn
8 +import torch.nn.functional as F
9 +from transformers import PretrainedConfig
10 +
11 +from vllm.config import LoRAConfig
12 +from vllm.lora.punica import add_lora, add_lora_slice, bgmv
13 +from vllm.model_executor.layers.sampler import Sampler
14 +from vllm.model_executor.parallel_utils.communication_op import (
15 + tensor_model_parallel_all_gather,
16 + tensor_model_parallel_all_reduce,
17 + tensor_model_parallel_gather,
18 +)
19 +from vllm.model_executor.layers.linear import (ColumnParallelLinear,
20 + RowParallelLinear,
21 + QKVParallelLinear,
22 + MergedColumnParallelLinear)
23 +from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding, ParallelLMHead
24 +from vllm.model_executor.parallel_utils.parallel_state import (
25 + get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
26 +from vllm.model_executor.parallel_utils.utils import split_tensor_along_last_dim
27 +
28 +if TYPE_CHECKING:
29 + pass
30 +
31 +
32 +def _apply_lora(
33 + x: torch.Tensor,
34 + lora_a_stacked: torch.Tensor,
35 + lora_b_stacked: torch.Tensor,
36 + indices: torch.Tensor,
37 + output: torch.Tensor,
38 +):
39 + """Applies lora to each input.
40 +
41 + This method applies all loras to each input. It uses the
42 + indices vector to determine which lora yields the
43 + correct output. An index of -1 means no lora should be
44 + applied. This method adds the final lora results to the
45 + output.
46 +
47 + Input shapes:
48 + x: (batch_size, hidden_dim)
49 + lora_a_stacked: (num_loras, lora_rank, hidden_dim)
50 + lora_b_stacked: (num_loras, output_dim, lora_rank)
51 + indices: (batch_size)
52 + output: (batch_size, output_dim)
53 + """
54 + org_output = output
55 + x = x.view(-1, x.shape[-1])
56 + output = output.view(-1, output.shape[-1])
57 + indices = indices.view(-1)
58 + add_lora(output, x, lora_a_stacked, lora_b_stacked, indices, 0, 1.0)
59 + return output.view_as(org_output)
60 +
61 +
62 +def _apply_lora_packed_nslice(
63 + x: torch.Tensor,
64 + lora_a_stacked: Tuple[torch.Tensor, torch.Tensor, torch.Tensor],
65 + lora_b_stacked: Tuple[torch.Tensor, torch.Tensor, torch.Tensor],
66 + indices: torch.Tensor,
67 + output: torch.Tensor,
68 + output_slices: Tuple[int, ...],
69 +):
70 + """Applies lora to each input.
71 +
72 + This method applies all loras to each input. It uses the
73 + indices vector to determine which lora yields the
74 + correct output. An index of -1 means no lora should be
75 + applied. This method adds the final lora results to the
76 + output.
77 +
78 + This method is used for layers that are composed of multiple sublayers
79 + (slices) packed together.
80 +
81 + Input shapes:
82 + x: (batch_size, hidden_dim)
83 + lora_a_stacked: 3 element tuple of (num_loras, lora_rank, hidden_dim)
84 + lora_b_stacked: 3 element tuple of (num_loras, output_dim, lora_rank)
85 + indices: (batch_size)
86 + output: (batch_size, q_slice_size + 2*kv_slice_size)
87 + output_slices: n-1 element tuple of (slice_size...), where n is number of slices
88 + """
89 + org_output = output
90 + x = x.view(-1, x.shape[-1])
91 + output = output.view(-1, output.shape[-1])
92 + indices = indices.view(-1)
93 + offset_left = 0
94 + for slice_idx in range(len(output_slices)):
95 + add_lora_slice(output, x, lora_a_stacked[slice_idx],
96 + lora_b_stacked[slice_idx], indices, 0, 1.0, offset_left,
97 + output_slices[slice_idx])
98 + offset_left += output_slices[slice_idx]
99 + return output.view_as(org_output)
100 +
101 +
102 +@dataclass
103 +class LoRAMapping:
104 + # Per every token in input_ids:
105 + index_mapping: Tuple[int, ...]
106 + # Per sampled token:
107 + prompt_mapping: Tuple[int, ...]
108 +
109 + def __post_init__(self):
110 + self.index_mapping = tuple(self.index_mapping)
111 + self.prompt_mapping = tuple(self.prompt_mapping)
112 +
113 +
114 +class BaseLayerWithLoRA(nn.Module):
115 +
116 + def create_lora_weights(self, max_loras: int, lora_config: LoRAConfig,
117 + model_config: PretrainedConfig) -> None:
118 + """Initializes lora matrices."""
119 + ...
120 +
121 + def reset_lora(self, index: int):
122 + """Resets the lora weights at index back to 0."""
123 + ...
124 +
125 + def set_lora(
126 + self,
127 + index: int,
128 + lora_a: torch.Tensor,
129 + lora_b: torch.Tensor,
130 + embeddings_tensor: Optional[torch.Tensor],
131 + ):
132 + """Overwrites lora tensors at index."""
133 + ...
134 +
135 + def set_mapping(
136 + self,
137 + base_indices: torch.Tensor,
138 + sampler_indices: torch.Tensor,
139 + sampler_indices_padded: torch.Tensor,
140 + embeddings_indices: torch.Tensor,
141 + indices_len: List[int],
142 + ):
143 + """Sets the mapping indices."""
144 + ...
145 +
146 +
147 +class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA):
148 +
149 + def __init__(self, base_layer: VocabParallelEmbedding) -> None:
150 + super().__init__()
151 + self.base_layer = base_layer
152 +
153 + def create_lora_weights(
154 + self,
155 + max_loras: int,
156 + lora_config: LoRAConfig,
157 + model_config: Optional[PretrainedConfig] = None) -> None:
158 +
159 + lora_vocab_start_idx = self.base_layer.org_vocab_size
160 + weights_idx = None
161 + if self.base_layer.vocab_end_index > lora_vocab_start_idx:
162 + # We can start adding lora weights
163 + weights_idx = max(
164 + lora_vocab_start_idx - self.base_layer.vocab_start_index, 0)
165 + self.embeddings_slice = (self.base_layer.vocab_start_index -
166 + self.base_layer.org_vocab_size +
167 + weights_idx,
168 + self.base_layer.vocab_end_index -
169 + self.base_layer.org_vocab_size)
170 + self.embeddings_weights = self.base_layer.weight.data[weights_idx:]
171 + self.embeddings_weights.fill_(0)
172 + else:
173 + self.embeddings_slice = None
174 + self.embeddings_weights = None
175 +
176 + self.embeddings_tensors = torch.zeros(
177 + (
178 + max_loras,
179 + lora_config.lora_extra_vocab_size,
180 + self.base_layer.embedding_dim,
181 + ),
182 + dtype=self.base_layer.weight.dtype,
183 + device=self.base_layer.weight.device,
184 + )
185 + self.lora_a_stacked = torch.zeros(
186 + (
187 + max_loras,
188 + self.base_layer.org_vocab_size +
189 + lora_config.lora_extra_vocab_size,
190 + lora_config.max_lora_rank,
191 + ),
192 + dtype=lora_config.lora_dtype,
193 + device=self.base_layer.weight.device,
194 + )
195 + self.lora_b_stacked = torch.zeros(
196 + (
197 + max_loras,
198 + 1,
199 + self.base_layer.embedding_dim,
200 + lora_config.max_lora_rank,
201 + ),
202 + dtype=lora_config.lora_dtype,
203 + device=self.base_layer.weight.device,
204 + )
205 + self.lora_a_stacked_2d = self.lora_a_stacked.view(
206 + self.lora_a_stacked.shape[0] * self.lora_a_stacked.shape[1],
207 + self.lora_a_stacked.shape[2],
208 + )
209 + self.indices: Optional[torch.Tensor] = None
210 + self.indices_len: Optional[List[int]] = None
211 + self.embeddings_indices = None
212 +
213 + def reset_lora(self, index: int):
214 + self.lora_a_stacked[index] = 0
215 + self.lora_b_stacked[index] = 0
216 + self.embeddings_tensors[index] = 0
217 +
218 + def set_lora(
219 + self,
220 + index: int,
221 + lora_a: torch.Tensor,
222 + lora_b: torch.Tensor,
223 + embeddings_tensor: Optional[torch.Tensor],
224 + ):
225 + self.reset_lora(index)
226 + self.lora_a_stacked[index, :lora_a.shape[0], :lora_a.shape[1]].copy_(
227 + lora_a, non_blocking=True)
228 + self.lora_b_stacked[index,
229 + 0, :lora_b.shape[1], :lora_b.shape[0]].copy_(
230 + lora_b.T, non_blocking=True)
231 + if embeddings_tensor is not None:
232 + self.embeddings_tensors[
233 + index, :embeddings_tensor.shape[0], :embeddings_tensor.
234 + shape[1]].copy_(embeddings_tensor, non_blocking=True)
235 + if self.embeddings_slice is not None:
236 + # TODO(yard1): Optimize this copy, we don't need to copy
237 + # everything, just the modified part
238 + embeddings = self.embeddings_tensors.view(
239 + self.embeddings_tensors.shape[0] *
240 + self.embeddings_tensors.shape[1],
241 + self.embeddings_tensors.shape[2]
242 + )[self.embeddings_slice[0]:self.embeddings_slice[1]]
243 + self.embeddings_weights[:embeddings.shape[0]].copy_(embeddings)
244 +
245 + def set_mapping(
246 + self,
247 + base_indices: torch.Tensor,
248 + sampler_indices: torch.Tensor,
249 + sampler_indices_padded: torch.Tensor,
250 + embeddings_indices: torch.Tensor,
251 + indices_len: List[int],
252 + ):
253 + self.indices = base_indices
254 + self.embeddings_indices = embeddings_indices
255 + self.indices_len = indices_len
256 +
257 + def forward(self, x: torch.Tensor) -> torch.Tensor:
258 + added_tokens_mask = x > self.base_layer.org_vocab_size - 1
259 + indices = self.embeddings_indices[1][:self.indices_len[3]].view_as(x)
260 + full_lora_a_embeddings = F.embedding(
261 + x + indices,
262 + self.lora_a_stacked_2d,
263 + )
264 + indices = self.embeddings_indices[0][:self.indices_len[3]].view_as(x)
265 + full_output = self.base_layer.forward(
266 + x.add_(indices * added_tokens_mask))
267 +
268 + full_output_org = full_output
269 + if full_output.ndim == 3:
270 + full_output = full_output.view(
271 + full_output.shape[0] * full_output.shape[1], -1)
272 + if full_lora_a_embeddings.ndim == 3:
273 + full_lora_a_embeddings = full_lora_a_embeddings.view(
274 + full_lora_a_embeddings.shape[0] *
275 + full_lora_a_embeddings.shape[1], -1)
276 + bgmv(full_output, full_lora_a_embeddings, self.lora_b_stacked,
277 + self.indices[:self.indices_len[0]], 0, 1.0)
278 + return full_output.view_as(full_output_org)
279 +
280 +
281 +class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
282 +
283 + def __init__(self, base_layer: ColumnParallelLinear) -> None:
284 + super().__init__()
285 + self.base_layer = base_layer
286 +
287 + def create_lora_weights(
288 + self,
289 + max_loras: int,
290 + lora_config: LoRAConfig,
291 + model_config: Optional[PretrainedConfig] = None) -> None:
292 + self.lora_a_stacked = torch.zeros(
293 + max_loras,
294 + 1,
295 + lora_config.max_lora_rank,
296 + self.base_layer.weight.shape[1],
297 + dtype=lora_config.lora_dtype,
298 + device=self.base_layer.weight.device,
299 + )
300 + self.lora_b_stacked = torch.zeros(
301 + max_loras,
302 + 1,
303 + self.base_layer.weight.shape[0],
304 + lora_config.max_lora_rank,
305 + dtype=lora_config.lora_dtype,
306 + device=self.base_layer.weight.device,
307 + )
308 +
309 + self.indices: Optional[torch.Tensor] = None
310 + self.indices_len: Optional[List[int]] = None
311 + self.output_dim = self.lora_b_stacked.shape[1]
312 +
313 + def reset_lora(self, index: int):
314 + self.lora_a_stacked[index] = 0
315 + self.lora_b_stacked[index] = 0
316 +
317 + def set_lora(
318 + self,
319 + index: int,
320 + lora_a: torch.Tensor,
321 + lora_b: torch.Tensor,
322 + embeddings_tensor: Optional[torch.Tensor],
323 + ):
324 + self.reset_lora(index)
325 +
326 + self.lora_a_stacked[index,
327 + 0, :lora_a.shape[1], :lora_a.shape[0]].copy_(
328 + lora_a.T, non_blocking=True)
329 + self.lora_b_stacked[index,
330 + 0, :lora_b.shape[1], :lora_b.shape[0]].copy_(
331 + lora_b.T, non_blocking=True)
332 +
333 + def set_mapping(
334 + self,
335 + base_indices: torch.Tensor,
336 + sampler_indices: torch.Tensor,
337 + sampler_indices_padded: torch.Tensor,
338 + embeddings_indices: torch.Tensor,
339 + indices_len: List[int],
340 + ):
341 + self.indices = base_indices
342 + self.indices_len = indices_len
343 +
344 + def apply_weights(self, x: torch.Tensor,
345 + bias: Optional[torch.Tensor]) -> torch.Tensor:
346 + output = self.base_layer.linear_method.apply_weights(
347 + self.base_layer.linear_weights, x, bias)
348 + _apply_lora(
349 + x,
350 + self.lora_a_stacked,
351 + self.lora_b_stacked,
352 + self.indices[:self.indices_len[0]],
353 + output,
354 + )
355 + return output
356 +
357 + def forward(self, input_):
358 + """Forward of ColumnParallelLinear
359 +
360 + Args:
361 + input_: Tensor whose last dimension is `input_size`.
362 +
363 + Returns:
364 + - output
365 + - bias
366 + """
367 + bias = (self.base_layer.bias
368 + if not self.base_layer.skip_bias_add else None)
369 +
370 + # Matrix multiply.
371 + output_parallel = self.apply_weights(input_, bias)
372 + if self.base_layer.gather_output:
373 + # All-gather across the partitions.
374 + output = tensor_model_parallel_all_gather(output_parallel)
375 + else:
376 + output = output_parallel
377 + output_bias = (self.base_layer.bias
378 + if self.base_layer.skip_bias_add else None)
379 + return output, output_bias
380 +
381 + @property
382 + def linear_weights(self):
383 + return self.base_layer.linear_weights
384 +
385 +
386 +class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
387 + """ColumnParallelLinear layer that is composed of 2 sublayers (slices)
388 + packed together (eg. gate_proj + up_proj -> gate_up_proj).
389 +
390 + This means we have 2 LoRAs, each applied to one half of the layer.
391 +
392 + Both slices must have the same size.
393 + """
394 +
395 + def __init__(self, base_layer: MergedColumnParallelLinear) -> None:
396 + super().__init__(base_layer)
397 +
398 + def create_lora_weights(
399 + self,
400 + max_loras: int,
401 + lora_config: LoRAConfig,
402 + model_config: Optional[PretrainedConfig] = None) -> None:
403 + n_slices = 2
404 + if not (len(self.base_layer.output_sizes) == n_slices
405 + and self.base_layer.output_sizes[0]
406 + == self.base_layer.output_sizes[1]):
407 + raise ValueError(
408 + "LoRAColumnParallelLinear2Slice requires 2 slices with "
409 + "the same size.")
410 + self.tp_size = get_tensor_model_parallel_world_size()
411 +
412 + self.lora_a_stacked = tuple(
413 + torch.zeros(
414 + max_loras,
415 + 1,
416 + lora_config.max_lora_rank,
417 + self.base_layer.weight.shape[1],
418 + dtype=lora_config.lora_dtype,
419 + device=self.base_layer.weight.device,
420 + ) for _ in range(n_slices))
421 + self.lora_b_stacked = tuple(
422 + torch.zeros(
423 + max_loras,
424 + 1,
425 + self.base_layer.weight.shape[0] // 2,
426 + lora_config.max_lora_rank,
427 + dtype=lora_config.lora_dtype,
428 + device=self.base_layer.weight.device,
429 + ) for _ in range(n_slices))
430 +
431 + self.indices: Optional[torch.Tensor] = None
432 + self.output_dim = self.lora_b_stacked[0].shape[2]
433 +
434 + def reset_lora(self, index: int):
435 + self.lora_a_stacked[0][index] = 0
436 + self.lora_a_stacked[1][index] = 0
437 + self.lora_b_stacked[0][index] = 0
438 + self.lora_b_stacked[1][index] = 0
439 +
440 + def set_lora(
441 + self,
442 + index: int,
443 + lora_a: torch.Tensor,
444 + lora_b: torch.Tensor,
445 + embeddings_tensor: Optional[torch.Tensor],
446 + ):
447 + self.reset_lora(index)
448 +
449 + if self.tp_size > 1:
450 + tensor_model_parallel_rank = get_tensor_model_parallel_rank()
451 + shard_size = self.output_dim
452 + start_idx = tensor_model_parallel_rank * shard_size
453 + end_idx = (tensor_model_parallel_rank + 1) * shard_size
454 + lora_b = lora_b[0][:,
455 + start_idx:end_idx], lora_b[1][:,
456 + start_idx:end_idx]
457 +
458 + if lora_a[0] is not None:
459 + self.lora_a_stacked[0][
460 + index, 0, :lora_a[0].shape[1], :lora_a[0].shape[0]].copy_(
461 + lora_a[0].T, non_blocking=True)
462 + self.lora_b_stacked[0][
463 + index, 0, :lora_b[0].shape[1], :lora_b[0].shape[0]].copy_(
464 + lora_b[0].T, non_blocking=True)
465 + if lora_a[1] is not None:
466 + self.lora_a_stacked[1][
467 + index, 0, :lora_a[1].shape[1], :lora_a[1].shape[0]].copy_(
468 + lora_a[1].T, non_blocking=True)
469 + self.lora_b_stacked[1][
470 + index, 0, :lora_b[1].shape[1], :lora_b[1].shape[0]].copy_(
471 + lora_b[1].T, non_blocking=True)
472 +
473 + def apply_weights(self, x: torch.Tensor,
474 + bias: Optional[torch.Tensor]) -> torch.Tensor:
475 + output = self.base_layer.linear_method.apply_weights(
476 + self.base_layer.linear_weights, x, bias)
477 + _apply_lora_packed_nslice(
478 + x,
479 + self.lora_a_stacked,
480 + self.lora_b_stacked,
481 + self.indices[:self.indices_len[0]],
482 + output,
483 + (self.output_dim, self.output_dim),
484 + )
485 + return output
486 +
487 +
488 +class QKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
489 + """ColumnParallelLinear layer that is composed of 3 sublayers (slices)
490 + packed together in qkv proj fashion
491 + (q_proj + k_proj + v_proj -> qkv_proj).
492 +
493 + This means we have 3 LoRAs, each applied to one slice of the layer.
494 +
495 + Q slice may have different shape than K and V slices (which both have
496 + the same shape).
497 + """
498 +
499 + def __init__(self, base_layer: QKVParallelLinear) -> None:
500 + super().__init__(base_layer)
501 +
502 + def create_lora_weights(
503 + self,
504 + max_loras: int,
505 + lora_config: LoRAConfig,
506 + model_config: Optional[PretrainedConfig] = None) -> None:
507 + self.tp_size = get_tensor_model_parallel_world_size()
508 + tp_rank = get_tensor_model_parallel_rank()
509 + self.q_proj_shard_size = (self.base_layer.num_heads *
510 + self.base_layer.head_size)
511 + self.kv_proj_shard_size = (self.base_layer.num_kv_heads *
512 + self.base_layer.head_size)
513 + self.q_shard_id = tp_rank
514 + self.kv_shard_id = tp_rank // self.base_layer.num_kv_head_replicas
515 +
516 + # q, k, v
517 + self.lora_a_stacked = (
518 + torch.zeros(
519 + max_loras,
520 + 1,
521 + lora_config.max_lora_rank,
522 + self.base_layer.weight.shape[1],
523 + dtype=lora_config.lora_dtype,
524 + device=self.base_layer.weight.device,
525 + ),
526 + torch.zeros(
527 + max_loras,
528 + 1,
529 + lora_config.max_lora_rank,
530 + self.base_layer.weight.shape[1],
531 + dtype=lora_config.lora_dtype,
532 + device=self.base_layer.weight.device,
533 + ),
534 + torch.zeros(
535 + max_loras,
536 + 1,
537 + lora_config.max_lora_rank,
538 + self.base_layer.weight.shape[1],
539 + dtype=lora_config.lora_dtype,
540 + device=self.base_layer.weight.device,
541 + ),
542 + )
543 + self.lora_b_stacked = (
544 + torch.zeros(
545 + max_loras,
546 + 1,
547 + self.q_proj_shard_size,
548 + lora_config.max_lora_rank,
549 + dtype=lora_config.lora_dtype,
550 + device=self.base_layer.weight.device,
551 + ),
552 + torch.zeros(
553 + max_loras,
554 + 1,
555 + self.kv_proj_shard_size,
556 + lora_config.max_lora_rank,
557 + dtype=lora_config.lora_dtype,
558 + device=self.base_layer.weight.device,
559 + ),
560 + torch.zeros(
561 + max_loras,
562 + 1,
563 + self.kv_proj_shard_size,
564 + lora_config.max_lora_rank,
565 + dtype=lora_config.lora_dtype,
566 + device=self.base_layer.weight.device,
567 + ),
568 + )
569 +
570 + self.output_slices = (self.q_proj_shard_size, self.kv_proj_shard_size,
571 + self.kv_proj_shard_size)
572 + self.packed_indices: Optional[torch.Tensor] = None
573 + self.standard_indices: Optional[torch.Tensor] = None
574 + self.indices_len: Optional[List[int]] = None
575 +
576 + def reset_lora(self, index: int):
577 + self.lora_a_stacked[0][index] = 0
578 + self.lora_b_stacked[0][index] = 0
579 + self.lora_a_stacked[1][index] = 0
580 + self.lora_b_stacked[1][index] = 0
581 + self.lora_a_stacked[2][index] = 0
582 + self.lora_b_stacked[2][index] = 0
583 +
584 + def set_lora(
585 + self,
586 + index: int,
587 + lora_a: torch.Tensor,
588 + lora_b: torch.Tensor,
589 + embeddings_tensor: Optional[torch.Tensor],
590 + ):
591 + self.reset_lora(index)
592 +
593 + if self.tp_size > 1:
594 + if lora_b[0] is not None:
595 + lora_b_q = lora_b[0][:, self.q_proj_shard_size *
596 + self.q_shard_id:self.q_proj_shard_size *
597 + (self.q_shard_id + 1)]
598 + self.lora_b_stacked[0][
599 + index, 0, :lora_b_q.shape[1], :lora_b_q.shape[0]].copy_(
600 + lora_b_q.T, non_blocking=True)
601 + if lora_b[1] is not None:
602 + lora_b_k = lora_b[1][:, self.kv_proj_shard_size *
603 + self.kv_shard_id:self.kv_proj_shard_size *
604 + (self.kv_shard_id + 1)]
605 + self.lora_b_stacked[1][
606 + index, 0, :lora_b_k.shape[1], :lora_b_k.shape[0]].copy_(
607 + lora_b_k.T, non_blocking=True)
608 + if lora_b[2] is not None:
609 + lora_b_v = lora_b[2][:, self.kv_proj_shard_size *
610 + self.kv_shard_id:self.kv_proj_shard_size *
611 + (self.kv_shard_id + 1)]
612 + self.lora_b_stacked[2][
613 + index, 0, :lora_b_v.shape[1], :lora_b_v.shape[0]].copy_(
614 + lora_b_v.T, non_blocking=True)
615 + else:
616 + if lora_b[0] is not None:
617 + self.lora_b_stacked[0][
618 + index, 0, :lora_b[0].shape[1], :lora_b[0].shape[0]].copy_(
619 + lora_b[0].T, non_blocking=True)
620 + if lora_b[1] is not None:
621 + self.lora_b_stacked[1][
622 + index, 0, :lora_b[1].shape[1], :lora_b[1].shape[0]].copy_(
623 + lora_b[1].T, non_blocking=True)
624 + if lora_b[2] is not None:
625 + self.lora_b_stacked[2][
626 + index, 0, :lora_b[2].shape[1], :lora_b[2].shape[0]].copy_(
627 + lora_b[2].T, non_blocking=True)
628 +
629 + if lora_a[0] is not None:
630 + self.lora_a_stacked[0][
631 + index, 0, :lora_a[0].shape[1], :lora_a[0].shape[0]].copy_(
632 + lora_a[0].T, non_blocking=True)
633 + if lora_a[1] is not None:
634 + self.lora_a_stacked[1][
635 + index, 0, :lora_a[1].shape[1], :lora_a[1].shape[0]].copy_(
636 + lora_a[1].T, non_blocking=True)
637 + if lora_a[2] is not None:
638 + self.lora_a_stacked[2][
639 + index, 0, :lora_a[2].shape[1], :lora_a[2].shape[0]].copy_(
640 + lora_a[2].T, non_blocking=True)
641 +
642 + def apply_weights(self, x: torch.Tensor,
643 + bias: Optional[torch.Tensor]) -> torch.Tensor:
644 + output = self.base_layer.linear_method.apply_weights(
645 + self.base_layer.linear_weights, x, bias)
646 + _apply_lora_packed_nslice(
647 + x,
648 + self.lora_a_stacked,
649 + self.lora_b_stacked,
650 + self.indices[:self.indices_len[0]],
651 + output,
652 + self.output_slices,
653 + )
654 + return output
655 +
656 +
657 +class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
658 +
659 + def __init__(self, base_layer: RowParallelLinear) -> None:
660 + super().__init__()
661 + self.base_layer = base_layer
662 +
663 + def create_lora_weights(
664 + self,
665 + max_loras: int,
666 + lora_config: LoRAConfig,
667 + model_config: Optional[PretrainedConfig] = None) -> None:
668 + self.lora_a_stacked = torch.zeros(
669 + (
670 + max_loras,
671 + 1,
672 + lora_config.max_lora_rank,
673 + self.base_layer.weight.shape[1],
674 + ),
675 + dtype=lora_config.lora_dtype,
676 + device=self.base_layer.weight.device,
677 + )
678 + self.lora_b_stacked = torch.zeros(
679 + (
680 + max_loras,
681 + 1,
682 + self.base_layer.weight.shape[0],
683 + lora_config.max_lora_rank,
684 + ),
685 + dtype=lora_config.lora_dtype,
686 + device=self.base_layer.weight.device,
687 + )
688 + self.indices: Optional[torch.Tensor] = None
689 + self.indices_len: Optional[List[int]] = None
690 +
691 + def reset_lora(self, index: int):
692 + self.lora_a_stacked[index] = 0
693 + self.lora_b_stacked[index] = 0
694 +
695 + def set_lora(
696 + self,
697 + index: int,
698 + lora_a: torch.Tensor,
699 + lora_b: torch.Tensor,
700 + embeddings_tensor: Optional[torch.Tensor],
701 + ):
702 + self.reset_lora(index)
703 + if self.base_layer.tp_size > 1:
704 + tensor_model_parallel_rank = get_tensor_model_parallel_rank()
705 + shard_size = self.base_layer.weight.shape[1]
706 + start_idx = tensor_model_parallel_rank * shard_size
707 + end_idx = (tensor_model_parallel_rank + 1) * shard_size
708 + lora_a = lora_a[start_idx:end_idx, :]
709 +
710 + self.lora_a_stacked[index,
711 + 0, :lora_a.shape[1], :lora_a.shape[0]].copy_(
712 + lora_a.T, non_blocking=True)
713 + self.lora_b_stacked[index,
714 + 0, :lora_b.shape[1], :lora_b.shape[0]].copy_(
715 + lora_b.T, non_blocking=True)
716 +
717 + def set_mapping(
718 + self,
719 + base_indices: torch.Tensor,
720 + sampler_indices: torch.Tensor,
721 + sampler_indices_padded: torch.Tensor,
722 + embeddings_indices: torch.Tensor,
723 + indices_len: List[int],
724 + ):
725 + self.indices = base_indices
726 + self.indices_len = indices_len
727 +
728 + def apply_weights(self, x: torch.Tensor) -> torch.Tensor:
729 + output = self.base_layer.linear_method.apply_weights(
730 + self.base_layer.linear_weights, x)
731 + _apply_lora(
732 + x,
733 + self.lora_a_stacked,
734 + self.lora_b_stacked,
735 + self.indices[:self.indices_len[0]],
736 + output,
737 + )
738 + return output
739 +
740 + def forward(self, input_):
741 + """Forward of RowParallelLinear
742 +
743 + Args:
744 + input_: tensor whose last dimension is `input_size`. If
745 + `input_is_parallel` is set, then the last dimension
746 + is `input_size // tp_size`.
747 +
748 + Returns:
749 + - output
750 + - bias
751 + """
752 + # Set up backprop all-reduce.
753 + if self.base_layer.input_is_parallel:
754 + input_parallel = input_
755 + else:
756 + # TODO: simplify code below
757 + tp_rank = get_tensor_model_parallel_rank()
758 + splitted_input = split_tensor_along_last_dim(
759 + input_, num_partitions=self.base_layer.tp_size)
760 + input_parallel = splitted_input[tp_rank].contiguous()
761 +
762 + # Matrix multiply.
763 + output_parallel = self.apply_weights(input_parallel)
764 + if self.base_layer.reduce_results and self.base_layer.tp_size > 1:
765 + output_ = tensor_model_parallel_all_reduce(output_parallel)
766 + else:
767 + output_ = output_parallel
768 +
769 + if not self.base_layer.skip_bias_add:
770 + output = (output_ + self.base_layer.bias
771 + if self.base_layer.bias is not None else output_)
772 + output_bias = None
773 + else:
774 + output = output_
775 + output_bias = self.base_layer.bias
776 + return output, output_bias
777 +
778 + @property
779 + def weight(self):
780 + return self.base_layer.weight
781 +
782 +
783 +class SamplerWithLoRA(BaseLayerWithLoRA):
784 +
785 + def __init__(
786 + self,
787 + base_layer: Sampler,
788 + hidden_size: int,
789 + dtype: torch.dtype,
790 + device: torch.device,
791 + ) -> None:
792 + super().__init__()
793 + self.base_layer = base_layer
794 + self.hidden_size = hidden_size
795 + self.dtype = dtype
796 + self.device = device
797 +
798 + @property
799 + def vocab_size(self):
800 + return self.base_layer.vocab_size
801 +
802 + @property
803 + def org_vocab_size(self):
804 + return self.base_layer.org_vocab_size
805 +
806 + @property
807 + def include_gpu_probs_tensor(self):
808 + return self.base_layer.include_gpu_probs_tensor
809 +
810 + def create_lora_weights(
811 + self,
812 + max_loras: int,
813 + lora_config: LoRAConfig,
814 + model_config: Optional[PretrainedConfig] = None,
815 + ) -> None:
816 + # Keep this in sync with csrc/punica/bgmv/bgmv_config.h
817 + if 32000 < self.base_layer.vocab_size > 33024:
818 + raise ValueError(
819 + "When using LoRA, vocab size must be 32000 >= vocab_size <= 33024"
820 + )
821 + self.lora_a_stacked = torch.zeros(
822 + (
823 + max_loras,
824 + 1,
825 + lora_config.max_lora_rank,
826 + self.hidden_size,
827 + ),
828 + dtype=lora_config.lora_dtype,
829 + device=self.device,
830 + )
831 + self.lora_b_stacked = torch.zeros(
832 + (
833 + max_loras,
834 + 1,
835 + # Pad for kernel compatibility
836 + math.ceil(self.base_layer.vocab_size /
837 + lora_config.lora_vocab_padding_size) *
838 + lora_config.lora_vocab_padding_size,
839 + lora_config.max_lora_rank,
840 + ),
841 + dtype=lora_config.lora_dtype,
842 + device=self.device,
843 + )
844 + self.embeddings_tensors = torch.full(
845 + (max_loras, lora_config.lora_extra_vocab_size, self.hidden_size),
846 + fill_value=float("-inf"),
847 + dtype=self.dtype,
848 + device=self.device,
849 + )
850 + self.indices = None
851 + self.indices_padded = None
852 + self.indices_len = None
853 +
854 + def reset_lora(self, index: int):
855 + self.lora_a_stacked[index] = 0
856 + self.lora_b_stacked[index] = 0
857 + self.embeddings_tensors[index] = float("-inf")
858 +
859 + def set_lora(
860 + self,
861 + index: int,
862 + lora_a: torch.Tensor,
863 + lora_b: torch.Tensor,
864 + embeddings_tensor: Optional[torch.Tensor],
865 + ):
866 + self.reset_lora(index)
867 + self.lora_a_stacked[index,
868 + 0, :lora_a.shape[1], :lora_a.shape[0]].copy_(
869 + lora_a.T, non_blocking=True)
870 + self.lora_b_stacked[index,
871 + 0, :lora_b.shape[1], :lora_b.shape[0]].copy_(
872 + lora_b.T, non_blocking=True)
873 + if embeddings_tensor is not None:
874 + self.embeddings_tensors[
875 + index, :embeddings_tensor.shape[0], :embeddings_tensor.
876 + shape[1], ] = embeddings_tensor
877 +
878 + def set_mapping(
879 + self,
880 + base_indices: torch.Tensor,
881 + sampler_indices: torch.Tensor,
882 + sampler_indices_padded: torch.Tensor,
883 + embeddings_indices: torch.Tensor,
884 + indices_len: List[int],
885 + ):
886 + self.indices = sampler_indices
887 + self.indices_padded = sampler_indices_padded
888 + self.indices_len = indices_len
889 +
890 + def _get_logits(
891 + self,
892 + hidden_states: torch.Tensor,
893 + embedding: torch.Tensor,
894 + embedding_bias: Optional[torch.Tensor] = None,
895 + ) -> torch.Tensor:
896 + # Get the logits for the next tokens.
897 + logits = torch.matmul(hidden_states, embedding.t())
898 + if embedding_bias is not None:
899 + logits += embedding_bias
900 + logits = tensor_model_parallel_gather(logits)
901 + if logits is None:
902 + return None
903 +
904 + lora_logits = torch.empty(
905 + self.embeddings_tensors.shape[0] + 1,
906 + self.embeddings_tensors.shape[1],
907 + hidden_states.shape[0],
908 + dtype=self.embeddings_tensors.dtype,
909 + device=self.embeddings_tensors.device,
910 + )
911 + torch.matmul(self.embeddings_tensors,
912 + hidden_states.T,
913 + out=lora_logits[:-1])
914 + lora_logits[-1] = float("-inf")
915 + lora_logits = lora_logits.mT
916 + lora_logits = (lora_logits.reshape(
917 + lora_logits.shape[0] * lora_logits.shape[1],
918 + lora_logits.shape[2],
919 + ).index_select(0,
920 + self.indices_padded[:self.indices_len[2]]).nan_to_num_(
921 + nan=float("-inf"),
922 + posinf=float("inf"),
923 + neginf=float("-inf")))
924 + logits[:,
925 + self.base_layer.org_vocab_size:self.base_layer.org_vocab_size +
926 + lora_logits.shape[1]] = lora_logits
927 +
928 + _apply_lora(
929 + hidden_states,
930 + self.lora_a_stacked,
931 + self.lora_b_stacked,
932 + self.indices[:self.indices_len[1]],
933 + logits,
934 + )
935 +
936 + # Remove paddings in vocab (if any).
937 + logits = logits[:, :self.base_layer.vocab_size]
938 +
939 + return logits
940 +
941 + def forward(self, *args, **kwargs):
942 + return type(self.base_layer).forward(self, *args, **kwargs)
943 +
944 +
945 +def from_layer(
946 + layer: nn.Module,
947 + max_loras: int,
948 + lora_config: LoRAConfig,
949 + model_config: Optional[PretrainedConfig] = None) -> BaseLayerWithLoRA:
950 + supported_layer_types = {
951 + VocabParallelEmbedding: VocabParallelEmbeddingWithLoRA,
952 + ColumnParallelLinear: ColumnParallelLinearWithLoRA,
953 + QKVParallelLinear: QKVParallelLinearWithLora,
954 + MergedColumnParallelLinear: MergedColumnParallelLinearWithLoRA,
955 + RowParallelLinear: RowParallelLinearWithLoRA,
956 + }
957 + for src_layer_type, lora_layer_type in supported_layer_types.items():
958 + if type(layer) is src_layer_type: # pylint: disable=unidiomatic-typecheck
959 + ret = lora_layer_type(layer)
960 + ret.create_lora_weights(max_loras, lora_config, model_config)
961 + return ret
962 + return layer
963 +
964 +
965 +def from_layer_sampler(
966 + layer: Sampler,
967 + lm_head: ParallelLMHead,
968 + max_loras: int,
969 + lora_config: LoRAConfig,
970 + model_config: Optional[PretrainedConfig] = None,
971 +) -> SamplerWithLoRA:
972 + ret = SamplerWithLoRA(layer, lm_head.embedding_dim, lm_head.weight.dtype,
973 + lm_head.weight.device)
974 + ret.create_lora_weights(max_loras, lora_config, model_config)
975 + return ret
vllm/lora/lora.py new
+160
@@ -0,0 +1,160 @@
1 +from typing import List, Optional
2 +
3 +import torch
4 +from vllm.utils import in_wsl
5 +
6 +
7 +class LoRALayerWeights:
8 + """LoRA weights for a layer composed of two low rank matrixes."""
9 +
10 + def __init__(
11 + self,
12 + module_name: str,
13 + rank: int,
14 + lora_alpha: int,
15 + lora_a: torch.Tensor,
16 + lora_b: torch.Tensor,
17 + embeddings_tensor: Optional[torch.Tensor] = None,
18 + scaling: Optional[float] = None,
19 + ) -> None:
20 + self.module_name = module_name
21 + self.rank = rank
22 + self.lora_alpha = lora_alpha
23 + self.lora_a = lora_a
24 + self.lora_b = lora_b
25 + self.embeddings_tensor = embeddings_tensor
26 +
27 + if scaling is None:
28 + self.scaling = self.lora_alpha / self.rank
29 + else:
30 + self.scaling = scaling
31 +
32 + def optimize(self) -> "LoRALayerWeights":
33 + """Optimize the LoRA by merging the scaling into lora_b."""
34 + if self.scaling == 1:
35 + return
36 + self.lora_b *= self.scaling
37 + self.scaling = 1
38 + return self
39 +
40 + @property
41 + def input_dim(self) -> int:
42 + return self.lora_a.shape[0]
43 +
44 + @property
45 + def output_dim(self) -> int:
46 + return self.lora_b.shape[1]
47 +
48 + @property
49 + def is_packed(self) -> bool:
50 + return False
51 +
52 + @property
53 + def extra_vocab_size(self) -> int:
54 + return self.embeddings_tensor.shape[
55 + 0] if self.embeddings_tensor is not None else 0
56 +
57 + @classmethod
58 + def create_dummy_lora_weights(
59 + cls,
60 + module_name: str,
61 + input_dim: int,
62 + output_dim: int,
63 + rank: int,
64 + dtype: torch.dtype,
65 + device: torch.device,
66 + embeddings_tensor_dim: Optional[int] = None) -> "LoRALayerWeights":
67 + pin_memory = str(device) == "cpu" and not in_wsl()
68 + lora_a = torch.zeros([input_dim, rank],
69 + dtype=dtype,
70 + device=device,
71 + pin_memory=pin_memory)
72 + lora_b = torch.zeros([rank, output_dim],
73 + dtype=dtype,
74 + device=device,
75 + pin_memory=pin_memory)
76 + embeddings_tensor = torch.rand(
77 + 10,
78 + embeddings_tensor_dim,
79 + dtype=dtype,
80 + device=device,
81 + pin_memory=pin_memory) if embeddings_tensor_dim else None
82 + return cls(
83 + module_name,
84 + rank=rank,
85 + lora_alpha=1,
86 + lora_a=lora_a,
87 + lora_b=lora_b,
88 + embeddings_tensor=embeddings_tensor,
89 + )
90 +
91 +
92 +class PackedLoRALayerWeights(LoRALayerWeights):
93 + """LoRA used for packed layers (eg. qkv_proj)."""
94 +
95 + def __init__(
96 + self,
97 + module_name: str,
98 + rank: int,
99 + lora_alphas: List[int],
100 + lora_a: List[torch.Tensor],
101 + lora_b: List[torch.Tensor],
102 + scaling: Optional[List[float]] = None,
103 + ) -> None:
104 + super().__init__(
105 + module_name=module_name,
106 + rank=rank,
107 + lora_alpha=0,
108 + lora_a=lora_a,
109 + lora_b=lora_b,
110 + scaling=scaling,
111 + embeddings_tensor=None,
112 + )
113 + self.lora_alphas = lora_alphas
114 + if scaling is None:
115 + self.scaling = [
116 + lora_alpha / self.rank for lora_alpha in self.lora_alphas
117 + ]
118 +
119 + @classmethod
120 + def pack(cls, loras: List["LoRALayerWeights"]) -> "PackedLoRALayerWeights":
121 + """Pack a list of LoRAs into a single LoRA.
122 +
123 + If LoRA is None, it signifies that the submodule does not have a LoRA.
124 + """
125 + first_lora = next(lora for lora in loras if lora is not None)
126 + for lora in loras:
127 + if lora is None:
128 + continue
129 + lora.optimize()
130 + rank = first_lora.rank
131 + module_name = first_lora.module_name
132 + obj = cls(
133 + module_name,
134 + rank,
135 + [lora.lora_alpha if lora is not None else None for lora in loras],
136 + [lora.lora_a if lora is not None else None for lora in loras],
137 + [lora.lora_b if lora is not None else None for lora in loras],
138 + scaling=[1 if lora is not None else None for lora in loras])
139 + return obj
140 +
141 + def optimize(self) -> "PackedLoRALayerWeights":
142 + """Optimize the LoRA by merging the scaling into lora_b."""
143 + for i in range(len(self.lora_b)):
144 + if self.scaling[i] == 1 or self.lora_b[i] is None:
145 + continue
146 + self.lora_b[i] *= self.scaling[i]
147 + self.scaling[i] = 1
148 + return self
149 +
150 + @property
151 + def input_dim(self) -> int:
152 + raise NotImplementedError()
153 +
154 + @property
155 + def output_dim(self) -> int:
156 + raise NotImplementedError()
157 +
158 + @property
159 + def is_packed(self) -> bool:
160 + return True
vllm/lora/models.py new
+654
@@ -0,0 +1,654 @@
1 +import copy
2 +import json
3 +import logging
4 +import math
5 +import os
6 +import re
7 +from typing import (Any, Callable, Dict, Hashable, List, Optional, Tuple, Type,
8 + Union)
9 +
10 +import safetensors.torch
11 +import torch
12 +from torch import nn
13 +
14 +from vllm.config import LoRAConfig
15 +from vllm.utils import LRUCache, in_wsl
16 +
17 +from vllm.lora.layers import BaseLayerWithLoRA, LoRAMapping, from_layer, from_layer_sampler
18 +from vllm.lora.lora import LoRALayerWeights, PackedLoRALayerWeights
19 +from vllm.lora.utils import parse_fine_tuned_lora_name, replace_submodule
20 +
21 +logger = logging.getLogger(__name__)
22 +
23 +# TODO: The mappings below should be moved to individual model classes.
24 +
25 +PACKED_MODULES_CFG = {
26 + "qkv_proj": [
27 + "q_proj",
28 + "k_proj",
29 + "v_proj",
30 + ],
31 + "gate_up_proj": [
32 + "gate_proj",
33 + "up_proj",
34 + ],
35 +}
36 +
37 +TARGET_MODULES_QKV = [
38 + "qkv_proj",
39 + "o_proj",
40 + "gate_up_proj",
41 + "down_proj",
42 + "embed_tokens",
43 + "lm_head",
44 +]
45 +
46 +EMBEDDING_MODULES = {
47 + "embed_tokens": "input_embeddings",
48 + "lm_head": "output_embeddings",
49 +}
50 +
51 +EMBEDDING_PADDING_MODULES = ["lm_head"]
52 +
53 +_GLOBAL_LORA_ID = 0
54 +
55 +
56 +def convert_mapping(
57 + mapping: LoRAMapping, lora_index_to_id: List[Optional[int]],
58 + max_loras: int, vocab_size: int, extra_vocab_size: int
59 +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, List[int]]:
60 + """Converts LoRAMapping to index tensors.
61 +
62 + Args:
63 + mapping: LoRAMapping mapping rows in a batch to LoRA ids.
64 + lora_index_to_id: List mapping LoRA ids to LoRA indices.
65 + max_loras: Maximum number of LoRAs.
66 + vocab_size: Model vocab size.
67 + extra_vocab_size: Extra vocab size each LoRA can have.
68 +
69 + Returns:
70 + A tuple of tensors:
71 + base_indices: Tensor of shape [batch_size] mapping batch rows to
72 + LoRA indices.
73 + sampler_indices: Tensor of shape [batch_size] mapping requests to
74 + LoRA indices for sampler. For generation, this will be the
75 + same as base_indicies. For prefill, this will map requests
76 + to LoRA indices.
77 + sampler_indices_padded: Tensor of shape [batch_size] mapping
78 + requests to LoRA indices for sampler with padding.
79 + Same as sampler_indicies, but -1 is replaced with
80 + max_loras.
81 + embeddings_indices: Tensor of shape [2, batch_size] mapping
82 + requests to embedding indices. First row is for embeddings
83 + added by the LoRAs, second row is for the LoRA.lora_a
84 + embeddings.
85 + indices_len: List of lengths of the above tensors.
86 + """
87 + indices = list(mapping.index_mapping).copy()
88 + embedding_indices = indices.copy()
89 + lora_indices = indices.copy()
90 + prompt_mapping = [
91 + lora_index_to_id.index(x) if x > 0 else -1
92 + for x in mapping.prompt_mapping
93 + ]
94 + lora_idx = None
95 + for i in range(len(indices)):
96 + # TODO index can be slow. optimize
97 + lora_idx = (lora_index_to_id.index(indices[i])
98 + if indices[i] > 0 else -1)
99 + embedding_indices[i] = lora_idx if indices[i] > 0 else 0
100 + indices[i] = i
101 + lora_indices[i] = lora_idx
102 +
103 + indices = torch.tensor([indices, lora_indices, embedding_indices],
104 + dtype=torch.long,
105 + device="cuda")
106 + prompt_mapping = torch.tensor(prompt_mapping,
107 + device="cuda",
108 + dtype=torch.long)
109 + embeddings_indices = torch.stack([
110 + indices[2] * extra_vocab_size,
111 + indices[2] * (vocab_size + extra_vocab_size)
112 + ])
113 + embeddings_indices[embeddings_indices == -1] = max_loras - 1
114 + base_indices = indices[1]
115 + sampler_indices = prompt_mapping
116 + sampler_indices_padded = sampler_indices.clone()
117 + sampler_indices_padded[sampler_indices_padded == -1] = max_loras - 1
118 + sampler_indices_padded = (
119 + torch.arange(
120 + 0, len(sampler_indices_padded), device="cuda", dtype=torch.long) +
121 + (sampler_indices_padded * len(sampler_indices_padded)))
122 + indices_len = (base_indices.shape[-1], sampler_indices.shape[-1],
123 + sampler_indices_padded.shape[-1],
124 + embeddings_indices.shape[-1])
125 +
126 + return (base_indices, sampler_indices, sampler_indices_padded,
127 + embeddings_indices, indices_len)
128 +
129 +
130 +def get_lora_id():
131 + global _GLOBAL_LORA_ID
132 + _GLOBAL_LORA_ID += 1
133 + return _GLOBAL_LORA_ID
134 +
135 +
136 +class LoRAModel:
137 + """A LoRA fine-tuned model."""
138 +
139 + def __init__(
140 + self,
141 + lora_model_id: int,
142 + rank: int,
143 + loras: Dict[str, LoRALayerWeights],
144 + ) -> None:
145 + self.id = lora_model_id
146 + assert (lora_model_id >
147 + 0), f"a valid lora id should be greater than 0, got {self.id}"
148 + self.rank = rank
149 + self.loras: Dict[str, LoRALayerWeights] = loras
150 +
151 + @property
152 + def extra_vocab_size(self) -> int:
153 + return max(lora.extra_vocab_size
154 + for lora in self.loras.values()) if self.loras else 0
155 +
156 + def get_lora(self, module_name: str) -> Optional[LoRALayerWeights]:
157 + """Get LoRA for a given module by name"""
158 + return self.loras.get(module_name, None)
159 +
160 + # (yard1): TODO see if we can derive target_embedding_padding automatically
161 + @classmethod
162 + def from_lora_tensors(
163 + cls,
164 + lora_model_id: int,
165 + rank: int,
166 + lora_alpha: int,
167 + tensors: Dict[str, torch.Tensor],
168 + device: str = "cuda",
169 + dtype: Optional[torch.dtype] = None,
170 + embeddings: Optional[Dict[str, torch.Tensor]] = None,
171 + target_embedding_padding: Optional[int] = None,
172 + ) -> "LoRAModel":
173 + """Create a LoRAModel from a dictionary of tensors."""
174 + pin_memory = str(device) == "cpu" and not in_wsl()
175 + loras: Dict[str, LoRALayerWeights] = {}
176 + for tensor_name, tensor in tensors.items():
177 + module_name, is_lora_a = parse_fine_tuned_lora_name(tensor_name)
178 + if module_name not in loras:
179 + lora_embeddings_tensor = None
180 + if embeddings:
181 + embeddings_module = next(
182 + (k for k in EMBEDDING_MODULES if k in module_name),
183 + None)
184 + if embeddings_module:
185 + lora_embeddings_tensor = embeddings[
186 + EMBEDDING_MODULES[embeddings_module]].to(
187 + device=device, dtype=dtype)
188 + if pin_memory:
189 + lora_embeddings_tensor = (
190 + lora_embeddings_tensor.pin_memory())
191 + loras[module_name] = LoRALayerWeights(module_name, rank,
192 + lora_alpha, None, None,
193 + lora_embeddings_tensor)
194 + if is_lora_a:
195 + loras[module_name].lora_a = tensor.to(device=device,
196 + dtype=dtype).t()
197 + if pin_memory:
198 + loras[module_name].lora_a = loras[
199 + module_name].lora_a.pin_memory()
200 + else:
201 + loras[module_name].lora_b = tensor.to(device=device,
202 + dtype=dtype).t()
203 + if any(name in module_name
204 + for name in EMBEDDING_PADDING_MODULES
205 + ) and target_embedding_padding is not None:
206 + lora_b = loras[module_name].lora_b
207 + assert target_embedding_padding >= lora_b.shape[1]
208 + addition = target_embedding_padding - lora_b.shape[1]
209 + loras[module_name].lora_b = torch.nn.functional.pad(
210 + lora_b, (0, addition))
211 + if pin_memory:
212 + loras[module_name].lora_b = loras[
213 + module_name].lora_b.pin_memory()
214 +
215 + for lora in loras.values():
216 + lora.optimize()
217 + return cls(lora_model_id, rank, loras)
218 +
219 + @classmethod
220 + def from_local_checkpoint(
221 + cls,
222 + lora_dir: str,
223 + lora_model_id: Optional[int] = None,
224 + device: str = "cuda",
225 + dtype: Optional[torch.dtype] = None,
226 + target_embedding_padding: Optional[int] = None) -> "LoRAModel":
227 + """Create a LoRAModel from a local checkpoint."""
228 + lora_config_path = os.path.join(lora_dir, "adapter_config.json")
229 + lora_tensor_path = os.path.join(lora_dir, "adapter_model.safetensors")
230 + lora_bin_file_path = os.path.join(lora_dir, "adapter_model.bin")
231 + new_embeddings_tensor_path = os.path.join(
232 + lora_dir, "new_embeddings.safetensors")
233 + new_embeddings_bin_file_path = os.path.join(lora_dir,
234 + "new_embeddings.bin")
235 + if os.path.isfile(lora_tensor_path):
236 + tensors = safetensors.torch.load_file(lora_tensor_path)
237 + elif os.path.isfile(lora_bin_file_path):
238 + tensors = torch.load(lora_bin_file_path)
239 + else:
240 + raise ValueError(f"{lora_dir} doesn't contain tensors")
241 +
242 + embeddings = None
243 + if os.path.isfile(new_embeddings_tensor_path):
244 + embeddings = safetensors.torch.load_file(
245 + new_embeddings_tensor_path)
246 + elif os.path.isfile(new_embeddings_bin_file_path):
247 + embeddings = torch.load(new_embeddings_bin_file_path)
248 +
249 + with open(lora_config_path) as f:
250 + config = json.load(f)
251 + rank = config["r"]
252 + lora_alpha = config["lora_alpha"]
253 + return cls.from_lora_tensors(
254 + lora_model_id=get_lora_id()
255 + if lora_model_id is None else lora_model_id,
256 + rank=rank,
257 + lora_alpha=lora_alpha,
258 + tensors=tensors,
259 + device=device,
260 + dtype=dtype,
261 + embeddings=embeddings,
262 + target_embedding_padding=target_embedding_padding,
263 + )
264 +
265 +
266 +class LoRAModelManager:
267 + """A manager that manages multiple LoRA-fine-tuned models."""
268 +
269 + def __init__(
270 + self,
271 + model: nn.Module,
272 + max_num_seqs: int,
273 + max_num_batched_tokens: int,
274 + vocab_size: int,
275 + lora_config: LoRAConfig,
276 + lora_target_modules: Union[str, List[str]] = TARGET_MODULES_QKV,
277 + packed_modules_mapping: Dict[str, List[str]] = PACKED_MODULES_CFG,
278 + ):
279 + """Create a LoRAModelManager and adapter for a given model.
280 +
281 + Args:
282 + model: the model to be adapted.
283 + max_num_seqs: the maximum number of sequences model can run in a
284 + single batch.
285 + max_num_batched_tokens: the maximum number of tokens model can run
286 + in a single batch.
287 + vocab_size: the vocab size of the model.
288 + lora_config: the LoRA configuration.
289 + lora_target_modules: the target modules patterns to be adapted.
290 + Support both single module name and a list of module names.
291 + packed_modules_mapping: the mapping for packed modules. vLLM
292 + packs some modules into one module, e.g., qkv_proj
293 + is packed of q_proj, k_proj, and v_proj. These modules
294 + have a single layer in the original model, but they are split
295 + into multiple layers in the adapted model.
296 + """
297 + self.lora_config = lora_config
298 + self.max_num_seqs = max_num_seqs
299 + assert self.capacity >= self.lora_slots
300 + self.max_num_batched_tokens = math.ceil(max_num_batched_tokens / 8) * 8
301 + self.lora_index_to_id: List[Optional[int]] = [None] * self.lora_slots
302 + self.vocab_size = vocab_size
303 + self.base_indices = torch.empty(self.max_num_batched_tokens,
304 + dtype=torch.long,
305 + device="cuda")
306 + self.sampler_indices = torch.empty(self.max_num_batched_tokens,
307 + dtype=torch.long,
308 + device="cuda")
309 + self.sampler_indices_padded = torch.empty(self.max_num_batched_tokens,
310 + dtype=torch.long,
311 + device="cuda")
312 + self.embeddings_indices = torch.empty(2,
313 + self.max_num_batched_tokens,
314 + dtype=torch.long,
315 + device="cuda")
316 + self.offsets = []
317 + # 4 is the number of indicies tensors defined above
318 + # base_indices, sampler_indices, sampler_indices_padded,
319 + # embeddings_indices
320 + self.indices_len = [None] * 4
321 +
322 + self.model: nn.Module = model
323 + self.lora_target_modules: List[str] = ([
324 + lora_target_modules
325 + ] if isinstance(lora_target_modules, str) else lora_target_modules)
326 + self.lora_target_modules = copy.deepcopy(lora_target_modules)
327 + self.packed_modules_mapping = copy.deepcopy(packed_modules_mapping)
328 + self.packed_modules: Dict[str, List[str]] = {}
329 + self.modules: Dict[str, "BaseLayerWithLoRA"] = {}
330 + self._registered_loras: Dict[int, LoRAModel] = {}
331 + # Dict instead of a Set for compatibility with LRUCache.
332 + self._active_loras: Dict[int, None] = {}
333 + self._last_mapping = None
334 + self._create_lora_modules()
335 + self.model.lora_manager = self
336 +
337 + @property
338 + def capacity(self) -> int:
339 + return self.lora_config.max_cpu_loras
340 +
341 + @property
342 + def lora_slots(self) -> int:
343 + return self.lora_config.max_loras
344 +
345 + def __len__(self) -> int:
346 + return len(self._registered_loras)
347 +
348 + def activate_lora(
349 + self,
350 + lora_id: int,
351 + ) -> bool:
352 + """Move LoRA into a GPU buffer to be used in the forward pass."""
353 + if lora_id in self._active_loras:
354 + return False
355 + first_free_slot = next(
356 + ((i, lora_id) for i, lora_id in enumerate(self.lora_index_to_id)
357 + if lora_id is None), None)
358 + if first_free_slot is None:
359 + raise ValueError("No free lora slots")
360 + index, _ = first_free_slot
361 + self._active_loras[lora_id] = None
362 + lora_model = self._registered_loras[lora_id]
363 + logger.debug(
364 + f"Activating LoRA. int id: {lora_model.id}, slot index: {index}")
365 + self.lora_index_to_id[index] = lora_model.id
366 + for module_name, module in self.modules.items():
367 + module_lora = lora_model.get_lora(module_name)
368 + if module_lora:
369 + module_lora.optimize()
370 + module.set_lora(index, module_lora.lora_a, module_lora.lora_b,
371 + module_lora.embeddings_tensor)
372 + else:
373 + module.reset_lora(index)
374 + return True
375 +
376 + def _deactivate_lora(self, lora_id: int):
377 + try:
378 + index = self.lora_index_to_id.index(lora_id)
379 + self.lora_index_to_id[index] = None
380 + except ValueError:
381 + pass
382 +
383 + def deactivate_lora(self, lora_id: int) -> bool:
384 + """Remove a LoRA from a GPU buffer."""
385 + if lora_id in self._active_loras:
386 + self._deactivate_lora(lora_id)
387 + self._active_loras.pop(lora_id)
388 + return True
389 + return False
390 +
391 + def _add_lora(self, lora: LoRAModel) -> bool:
392 + self._create_merged_loras_inplace(lora)
393 + self._registered_loras[lora.id] = lora
394 +
395 + def add_lora(self, lora: LoRAModel) -> bool:
396 + """Add a LoRAModel to the manager CPU cache."""
397 + if lora.id not in self._registered_loras:
398 + if len(self._registered_loras) >= self.capacity:
399 + raise RuntimeError("No free LoRA slots.")
400 + self._add_lora(lora)
401 + return True
402 + return False
403 +
404 + def remove_lora(self, lora_id: int) -> bool:
405 + """Remove a LoRAModel from the manager CPU cache."""
406 + # TODO: should we check active lora?
407 + self.deactivate_lora(lora_id)
408 + return bool(self._registered_loras.pop(lora_id, None))
409 +
410 + # TODO see if this can be vectorized
411 + def _set_lora_mapping(self, mapping: LoRAMapping) -> None:
412 + (base_indices, sampler_indices, sampler_indices_padded,
413 + embeddings_indices,
414 + indices_len) = convert_mapping(mapping, self.lora_index_to_id,
415 + self.lora_slots + 1, self.vocab_size,
416 + self.lora_config.lora_extra_vocab_size)
417 + self.base_indices[:base_indices.shape[0]].copy_(base_indices)
418 + self.sampler_indices[:sampler_indices.shape[0]].copy_(sampler_indices)
419 + self.sampler_indices_padded[:sampler_indices_padded.shape[0]].copy_(
420 + sampler_indices_padded)
421 + self.embeddings_indices[:embeddings_indices.
422 + shape[0], :embeddings_indices.shape[1]].copy_(
423 + embeddings_indices)
424 + # Maintain the reference
425 + self.indices_len[:] = indices_len
426 +
427 + def set_lora_mapping(self, lora_mapping: LoRAMapping) -> None:
428 + if self._last_mapping != lora_mapping:
429 + self._set_lora_mapping(lora_mapping)
430 + self._last_mapping = lora_mapping
431 +
432 + def list_loras(self) -> Dict[int, LoRAModel]:
433 + """List all registered LoRAModels."""
434 + return dict(self._registered_loras)
435 +
436 + def get_lora(self, lora_id: int) -> Optional[LoRAModel]:
437 + return self._registered_loras.get(lora_id, None)
438 +
439 + def remove_all_loras(self) -> bool:
440 + """Remove all LoRAModels from the manager."""
441 + self._registered_loras.clear()
442 + self.lora_index_to_id = [None] * self.lora_slots
443 + self._active_loras.clear()
444 +
445 + def _create_lora_modules(self):
446 + for module_name, module in self.model.named_modules():
447 + if not self._match_target_modules(module_name):
448 + continue
449 +
450 + new_module = replace_submodule(
451 + self.model, module_name,
452 + from_layer(module, self.lora_slots, self.lora_config,
453 + self.model.config))
454 + # (yard1): TODO make this more robust
455 + if "lm_head" in module_name:
456 + sampler_module = self.model.get_submodule("sampler")
457 + new_module = replace_submodule(
458 + self.model, "sampler",
459 + from_layer_sampler(sampler_module, module, self.lora_slots,
460 + self.lora_config, self.model.config))
461 + self.register_module(module_name, new_module)
462 + self._register_packed_modules(module_name)
463 + new_module.set_mapping(self.base_indices, self.sampler_indices,
464 + self.sampler_indices_padded,
465 + self.embeddings_indices, self.indices_len)
466 +
467 + def register_module(self, module_name: str, module: "BaseLayerWithLoRA"):
468 + assert isinstance(module, BaseLayerWithLoRA)
469 + self.modules[module_name] = module
470 +
471 + def create_dummy_lora(self, lora_id: int, rank: int) -> LoRAModel:
472 + """Create zero-initialized LoRAModel for warmup."""
473 + model = LoRAModel(lora_id, rank, {})
474 + for module_name, module in self.model.named_modules():
475 + if not self._match_target_modules(module_name) or not isinstance(
476 + module, BaseLayerWithLoRA):
477 + continue
478 + parts = module_name.split(".")
479 + if module_name not in self.packed_modules:
480 + if parts[-1] in EMBEDDING_MODULES:
481 + input_dim = (module.base_layer.org_vocab_size +
482 + self.lora_config.lora_extra_vocab_size if
483 + hasattr(module.base_layer, "org_vocab_size")
484 + else module.base_layer.weight.shape[1])
485 + output_dim = module.base_layer.embedding_dim if hasattr(
486 + module.base_layer,
487 + "embedding_dim") else module.base_layer.weight.shape[0]
488 + embeddings_tensor_dim = (module.base_layer.embedding_dim if
489 + hasattr(module.base_layer,
490 + "embedding_dim") else
491 + module.base_layer.weight.shape[1])
492 + lora = LoRALayerWeights.create_dummy_lora_weights(
493 + module_name,
494 + input_dim,
495 + output_dim,
496 + rank,
497 + module.lora_a_stacked.dtype,
498 + "cpu",
499 + embeddings_tensor_dim=embeddings_tensor_dim)
500 + else:
501 + lora = LoRALayerWeights.create_dummy_lora_weights(
502 + module_name,
503 + module.lora_a_stacked.shape[-1],
504 + module.lora_b_stacked.shape[-2],
505 + rank,
506 + module.lora_a_stacked.dtype,
507 + "cpu",
508 + )
509 + lora.optimize()
510 + else:
511 + parts = module_name.split(".")
512 + replacements = self.packed_modules_mapping[parts[-1]]
513 + subloras = []
514 + for i, r in enumerate(replacements):
515 + lora = LoRALayerWeights.create_dummy_lora_weights(
516 + module_name + "." + r,
517 + module.lora_a_stacked[i].shape[-1],
518 + module.lora_b_stacked[i].shape[-2],
519 + rank,
520 + module.lora_a_stacked[i].dtype,
521 + "cpu",
522 + )
523 + lora.optimize()
524 + subloras.append(lora)
525 + lora = PackedLoRALayerWeights.pack(subloras)
526 + model.loras[module_name] = lora
527 + return model
528 +
529 + def _match_target_modules(self, module_name: str):
530 + return any(
531 + re.match(
532 + r".*\.{target_module}$".format(target_module=target_module),
533 + module_name) or target_module == module_name
534 + for target_module in self.lora_target_modules)
535 +
536 + def _register_packed_modules(self, module_full_name: str) -> None:
537 + parts = module_full_name.split(".")
538 + module_name = parts[-1]
539 + replacements = self.packed_modules_mapping.get(module_name)
540 + if not replacements:
541 + return
542 + prefix = ".".join(parts[:-1])
543 + self.packed_modules[module_full_name] = [
544 + prefix + "." + r if prefix else r for r in replacements
545 + ]
546 +
547 + def _create_merged_loras_inplace(self, lora_model: LoRAModel) -> None:
548 + for module_name, new_module_names in self.packed_modules.items():
549 + replacement_loras = []
550 + has_replacement = False
551 + for r in new_module_names:
552 + lora = lora_model.get_lora(r)
553 + replacement_loras.append(lora)
554 + if lora:
555 + has_replacement = True
556 + if not has_replacement:
557 + continue
558 + for i in range(len(replacement_loras)):
559 + if replacement_loras[i]:
560 + continue
561 + replacement_loras[i] = None
562 + lora_model.loras[module_name] = PackedLoRALayerWeights.pack(
563 + replacement_loras)
564 +
565 +
566 +class LoRALRUCache(LRUCache):
567 +
568 + def __init__(self, capacity: int, deactivate_lora_fn: Callable[[Hashable],
569 + None]):
570 + super().__init__(capacity)
571 + self.deactivate_lora_fn = deactivate_lora_fn
572 +
573 + def _on_remove(self, key: Hashable, value: Any):
574 + logger.debug(f"Removing LoRA. int id: {key}")
575 + self.deactivate_lora_fn(key)
576 + return super()._on_remove(key, value)
577 +
578 +
579 +class LRUCacheLoRAModelManager(LoRAModelManager):
580 + """A model manager that manages multiple LoRAs with LRU cache."""
581 +
582 + def __init__(
583 + self,
584 + model: nn.Module,
585 + max_num_seqs: int,
586 + max_num_batched_tokens: int,
587 + vocab_size: int,
588 + lora_config: LoRAConfig,
589 + lora_target_modules: Union[str, List[str]] = TARGET_MODULES_QKV,
590 + packed_modules_mapping: Dict[str, List[str]] = PACKED_MODULES_CFG,
591 + ):
592 + super().__init__(model, max_num_seqs, max_num_batched_tokens,
593 + vocab_size, lora_config, lora_target_modules,
594 + packed_modules_mapping)
595 + self._registered_loras: LoRALRUCache = LoRALRUCache(
596 + self.capacity, self.deactivate_lora)
597 + self._active_loras: LoRALRUCache = LoRALRUCache(
598 + self.lora_slots, self._deactivate_lora)
599 +
600 + def list_loras(self) -> Dict[int, LoRAModel]:
601 + """List all registered LoRAModels."""
602 + return dict(self._registered_loras.cache)
603 +
604 + def add_lora(self, lora: LoRAModel) -> bool:
605 + """Add a LoRAModel to the manager."""
606 + if lora.id not in self._registered_loras:
607 + self._add_lora(lora)
608 + was_added = True
609 + else:
610 + # We always touch to update the LRU cache order
611 + self._registered_loras.touch(lora.id)
612 + was_added = False
613 + return was_added
614 +
615 + def activate_lora(
616 + self,
617 + lora_id: int,
618 + ) -> bool:
619 + if lora_id not in self._active_loras and len(
620 + self._active_loras) >= self.lora_slots:
621 + self._active_loras.remove_oldest()
622 + result = super().activate_lora(lora_id)
623 + # We always touch to update the LRU cache order
624 + self._active_loras.touch(lora_id)
625 + return result
626 +
627 + def remove_oldest_lora(self) -> bool:
628 + if len(self._registered_loras) > 0:
629 + self._registered_loras.remove_oldest()
630 + return True
631 + return False
632 +
633 +
634 +def create_lora_manager(
635 + model: nn.Module,
636 + max_num_seqs: int,
637 + max_num_batched_tokens: int,
638 + vocab_size: int,
639 + lora_config: LoRAConfig,
640 + target_modules: Union[str, List[str]] = TARGET_MODULES_QKV,
641 + lora_manager_cls: Type[LoRAModelManager] = LoRAModelManager,
642 + **kwargs) -> LoRAModelManager:
643 + """Create a LoRA adapter for a given model."""
644 + if not getattr(model, "supports_lora", False):
645 + raise ValueError(f"Model {type(model)} is not supported for LoRA.")
646 + lora_manager = lora_manager_cls(
647 + model=model,
648 + max_num_seqs=max_num_seqs,
649 + max_num_batched_tokens=max_num_batched_tokens,
650 + vocab_size=vocab_size,
651 + lora_config=lora_config,
652 + lora_target_modules=target_modules,
653 + **kwargs)
654 + return lora_manager
vllm/lora/punica.py new
+173
@@ -0,0 +1,173 @@
1 +# Based on code from https://github.com/punica-ai/punica
2 +
3 +from typing import Optional
4 +
5 +import torch
6 +
7 +import_exc = None
8 +
9 +try:
10 + import vllm._punica_C as punica_kernels
11 +except ImportError as e:
12 + import_exc = e
13 +
14 +if import_exc is None:
15 +
16 + def bgmv(
17 + y: torch.Tensor,
18 + x: torch.Tensor,
19 + w_t_all: torch.Tensor,
20 + indicies: torch.LongTensor,
21 + layer_idx: int,
22 + scale: float,
23 + ):
24 + """
25 + Semantics:
26 + y[i] += (
27 + x[i].unsqueeze(0)
28 + @ w_t_all[indices[i], layer_idx, :, :].transpose(-1, -2)
29 + * scale
30 + ).squeeze(0)
31 +
32 + Args:
33 + y: Shape: `[B, H2]`. Output vectors. Will be changed in-place.
34 + x: Shape: `[B, H1]`. Input vectors.
35 + w_t_all: Shape: `[None, L, H2, H1]`. All of the transposed weight
36 + matrices.
37 + indicies: Shape: `[B]`. Indices of the weight matrices.
38 + layer_idx: Layer index of the weight matrices.
39 + scale: Scaling factor.
40 + """
41 + punica_kernels.dispatch_bgmv(y, x, w_t_all, indicies, layer_idx, scale)
42 +
43 + def add_lora(y: torch.Tensor,
44 + x: torch.Tensor,
45 + wa_t_all: torch.Tensor,
46 + wb_t_all: torch.Tensor,
47 + indicies: torch.LongTensor,
48 + layer_idx: int,
49 + scale: float,
50 + *,
51 + buffer: Optional[torch.Tensor] = None):
52 + """
53 + Semantics:
54 + y[i] += (
55 + x[i].unsqueeze(0)
56 + @ wa_t_all[indices[i], layer_idx, :, :].transpose(-1, -2)
57 + @ wb_t_all[indices[i], layer_idx, :, :].transpose(-1, -2)
58 + * scale
59 + ).squeeze(0)
60 +
61 + Args:
62 + y: Shape: `[B, H2]`. Output vectors. Will be changed in-place.
63 + x: Shape: `[B, H1]`. Input vectors.
64 + wa_t_all: Shape: `[None, L, R, H1]`. All of the transposed
65 + LoRA A matrices.
66 + wb_t_all: Shape: `[None, L, H2, R]`. All of the transposed
67 + LoRA B matrices.
68 + indicies: Shape: `[B]`. Indices of the LoRA weights.
69 + layer_idx: Layer index of LoRA weights.
70 + scale: Scaling factor.
71 + buffer: Optional. Shape: `[B, R]`. Temporary buffer.
72 + """
73 + r = wb_t_all.size(-1)
74 + if buffer is None:
75 + # We set the buffer to be float32 by default to avoid
76 + # numerical innacuracies that would otherwise happen
77 + # due to downcasting.
78 + buffer = torch.zeros((x.size(0), r),
79 + dtype=torch.float32,
80 + device=x.device)
81 + punica_kernels.dispatch_bgmv(buffer, x, wa_t_all, indicies, layer_idx,
82 + 1.0)
83 + punica_kernels.dispatch_bgmv(y, buffer, wb_t_all, indicies, layer_idx,
84 + scale)
85 +
86 + def add_lora_slice(y: torch.Tensor,
87 + x: torch.Tensor,
88 + wa_t_all: torch.Tensor,
89 + wb_t_all: torch.Tensor,
90 + indicies: torch.LongTensor,
91 + layer_idx: int,
92 + scale: float,
93 + y_offset: int,
94 + y_slice_size: int,
95 + *,
96 + buffer: Optional[torch.Tensor] = None):
97 + """
98 + Same as `add_lora` but you can operate on slices of y.
99 + Pass whole y, define y_offset and y_slice_size.
100 +
101 + Semantics:
102 + y[i] += (
103 + x[i].unsqueeze(0)
104 + @ wa_t_all[indices[i], layer_idx, :, :].transpose(-1, -2)
105 + @ wb_t_all[indices[i], layer_idx, :, :].transpose(-1, -2)
106 + * scale
107 + ).squeeze(0)
108 +
109 + Args:
110 + y: Shape: `[B, H2]`. Output vectors. Will be changed in-place.
111 + x: Shape: `[B, H1]`. Input vectors.
112 + wa_t_all: Shape: `[None, L, R, H1]`. All of the transposed
113 + LoRA A matrices.
114 + wb_t_all: Shape: `[None, L, H2, R]`. All of the transposed
115 + LoRA B matrices.
116 + indicies: Shape: `[B]`. Indices of the LoRA weights.
117 + layer_idx: Layer index of LoRA weights.
118 + scale: Scaling factor.
119 + y_offset: Offset to apply to the starting column of y.
120 + y_slice_size: Size of the y column slice.
121 + """
122 + r = wb_t_all.size(-1)
123 + if buffer is None:
124 + # We set the buffer to be float32 by default to avoid
125 + # numerical inaccuracies that would otherwise happen
126 + # due to downcasting.
127 + buffer = torch.zeros((x.size(0), r),
128 + dtype=torch.float32,
129 + device=x.device)
130 + punica_kernels.dispatch_bgmv_low_level(
131 + buffer,
132 + x,
133 + wa_t_all,
134 + indicies,
135 + layer_idx,
136 + 1.0,
137 + x.size(1),
138 + buffer.size(1),
139 + 0,
140 + )
141 + punica_kernels.dispatch_bgmv_low_level(
142 + y,
143 + buffer,
144 + wb_t_all,
145 + indicies,
146 + layer_idx,
147 + scale,
148 + buffer.size(1),
149 + y_slice_size,
150 + y_offset,
151 + )
152 +
153 +else:
154 +
155 + def _raise_exc(
156 + *args, # pylint: disable=unused-argument
157 + **kwargs # pylint: disable=unused-argument
158 + ):
159 + if torch.cuda.get_device_capability() < (8, 0):
160 + raise ImportError(
161 + "LoRA kernels require compute capability>=8.0") from import_exc
162 + else:
163 + raise import_exc
164 +
165 + bgmv = _raise_exc
166 + add_lora = _raise_exc
167 + add_lora_slice = _raise_exc
168 +
169 +__all__ = [
170 + "bgmv",
171 + "add_lora",
172 + "add_lora_slice",
173 +]
vllm/lora/request.py new
+32
@@ -0,0 +1,32 @@
1 +from dataclasses import dataclass
2 +
3 +
4 +@dataclass
5 +class LoRARequest:
6 + """
7 + Request for a LoRA adapter.
8 +
9 + Note that this class should be be used internally. For online
10 + serving, it is recommended to not allow users to use this class but
11 + instead provide another layer of abstraction to prevent users from
12 + accessing unauthorized LoRA adapters.
13 +
14 + lora_int_id must be globally unique for a given adapter.
15 + This is currently not enforced in vLLM.
16 + """
17 +
18 + lora_name: str
19 + lora_int_id: int
20 + lora_local_path: str
21 +
22 + def __post_init__(self):
23 + if self.lora_int_id < 1:
24 + raise ValueError(
25 + f"lora_int_id must be > 0, got {self.lora_int_id}")
26 +
27 + def __eq__(self, value: object) -> bool:
28 + return isinstance(
29 + value, LoRARequest) and self.lora_int_id == value.lora_int_id
30 +
31 + def __hash__(self) -> int:
32 + return self.lora_int_id
vllm/lora/utils.py new
+39
@@ -0,0 +1,39 @@
1 +import logging
2 +from typing import Tuple
3 +
4 +from torch import nn
5 +
6 +logger = logging.getLogger(__name__)
7 +
8 +
9 +def replace_submodule(model: nn.Module, module_name: str,
10 + new_module: nn.Module) -> nn.Module:
11 + """Replace a submodule in a model with a new module."""
12 + parent = model.get_submodule(".".join(module_name.split(".")[:-1]))
13 + target_name = module_name.split(".")[-1]
14 + setattr(parent, target_name, new_module)
15 + return new_module
16 +
17 +
18 +def parse_fine_tuned_lora_name(name: str) -> Tuple[str, bool]:
19 + """Parse the name of lora weights.
20 +
21 + args:
22 + name: the name of the fine-tuned LoRA, e.g.
23 + base_model.model.dense1.weight
24 + return:
25 + Tuple(module_name, is_lora_a):
26 + module_name: the name of the module, e.g. model.dense1,
27 + is_lora_a whether the tensor is lora_a or lora_b.
28 + """
29 + parts = name.split(".")
30 + assert parts[0] == "base_model"
31 + assert parts[1] == "model"
32 + if parts[-1] == "weight":
33 + assert parts[-2] == "lora_A" or parts[-2] == "lora_B"
34 + return ".".join(parts[2:-2]), parts[-2] == "lora_A"
35 +
36 + if parts[-1] == "lora_embedding_A" or parts[-1] == "lora_embedding_B":
37 + return ".".join(parts[2:-1]), parts[-1] == "lora_embedding_A"
38 +
39 + raise ValueError(f"{name} is unsupported format")
vllm/lora/worker_manager.py new
+237
@@ -0,0 +1,237 @@
1 +import logging
2 +from abc import ABC, abstractmethod, abstractproperty
3 +from typing import Any, List, Optional, Set, Type, Union
4 +
5 +import torch
6 +
7 +from vllm.lora.models import (TARGET_MODULES_QKV, LoRAModel, LoRAModelManager,
8 + LRUCacheLoRAModelManager, create_lora_manager)
9 +from vllm.lora.request import LoRARequest
10 +from vllm.lora.layers import LoRAMapping
11 +from vllm.config import LoRAConfig
12 +
13 +logger = logging.getLogger(__name__)
14 +
15 +
16 +class WorkerLoRAManager(ABC):
17 + """Abstract class for managing LoRA models on the worker side."""
18 +
19 + def __init__(self, max_num_seqs: int, max_num_batched_tokens: int,
20 + vocab_size: int, lora_config: LoRAConfig,
21 + device: torch.device):
22 + self.max_num_seqs = max_num_seqs
23 + self.max_num_batched_tokens = max_num_batched_tokens
24 + self.vocab_size = vocab_size
25 + self.device = device
26 + self.lora_config = lora_config
27 +
28 + @abstractproperty
29 + def is_enabled(self) -> bool:
30 + ...
31 +
32 + @abstractmethod
33 + def create_lora_manager(
34 + self,
35 + model: torch.nn.Module,
36 + target_modules: Union[str, List[str]] = TARGET_MODULES_QKV,
37 + ) -> Any:
38 + ...
39 +
40 + @abstractmethod
41 + def set_active_loras(self, lora_requests: List[LoRARequest],
42 + lora_mapping: LoRAMapping) -> None:
43 + ...
44 +
45 + @abstractmethod
46 + def add_lora(self, lora_request: LoRARequest) -> bool:
47 + ...
48 +
49 + @abstractmethod
50 + def add_dummy_lora(self, lora_request: LoRARequest, rank: int) -> bool:
51 + ...
52 +
53 + @abstractmethod
54 + def remove_lora(self, lora_id: int) -> bool:
55 + ...
56 +
57 + @abstractmethod
58 + def remove_all_loras(self) -> bool:
59 + ...
60 +
61 + @abstractmethod
62 + def list_loras(self) -> Set[int]:
63 + ...
64 +
65 +
66 +class WorkerLoRAManager(WorkerLoRAManager):
67 + """WorkerLoRAManager that manages LoRA models on the worker side.
68 +
69 + Every request, the requested LoRAs will be loaded (unless they are already
70 + loaded), and every other LoRA will be unloaded."""
71 +
72 + _lora_manager_cls: Type[LoRAModelManager] = LoRAModelManager
73 +
74 + def __init__(
75 + self,
76 + max_num_seqs: int,
77 + max_num_batched_tokens: int,
78 + vocab_size: int,
79 + lora_config: LoRAConfig,
80 + device: torch.device,
81 + lora_model_cls: Type[LoRAModel] = LoRAModel,
82 + ):
83 + self._lora_manager: Optional[LoRAModelManager] = None
84 + self._lora_model_cls = lora_model_cls
85 + super().__init__(max_num_seqs, max_num_batched_tokens, vocab_size,
86 + lora_config, device)
87 +
88 + @property
89 + def is_enabled(self) -> bool:
90 + return True
91 +
92 + def create_lora_manager(
93 + self,
94 + model: torch.nn.Module,
95 + target_modules: Union[str, List[str]] = TARGET_MODULES_QKV,
96 + ) -> Any:
97 + lora_manager = create_lora_manager(
98 + model,
99 + max_num_seqs=self.max_num_seqs,
100 + max_num_batched_tokens=self.max_num_batched_tokens,
101 + target_modules=target_modules,
102 + vocab_size=self.vocab_size,
103 + lora_config=self.lora_config,
104 + lora_manager_cls=self._lora_manager_cls,
105 + )
106 + self._lora_manager: LoRAModelManager = lora_manager
107 + return lora_manager.model
108 +
109 + def set_active_loras(self, lora_requests: List[LoRARequest],
110 + lora_mapping: LoRAMapping) -> None:
111 + self._apply_loras(lora_requests)
112 + self._lora_manager.set_lora_mapping(lora_mapping)
113 +
114 + def _apply_loras(self, lora_requests: List[LoRARequest]) -> None:
115 + loras_that_exist = self.list_loras()
116 + loras_map = {
117 + lora_request.lora_int_id: lora_request
118 + for lora_request in lora_requests if lora_request
119 + }
120 + if len(loras_map) > self._lora_manager.lora_slots:
121 + raise RuntimeError(
122 + f"Number of requested LoRAs ({len(loras_map)}) is greater "
123 + "than the number of GPU LoRA slots "
124 + f"({self._lora_manager.lora_slots}).")
125 +
126 + new_loras = set(loras_map)
127 + loras_to_add = new_loras - loras_that_exist
128 + loras_to_remove = loras_that_exist - new_loras
129 +
130 + for lora_id in loras_to_remove:
131 + self.remove_lora(lora_id)
132 +
133 + for lora_id in loras_to_add:
134 + self.add_lora(loras_map[lora_id])
135 +
136 + def _load_lora(self, lora_request: LoRARequest) -> LoRAModel:
137 + try:
138 + lora = self._lora_model_cls.from_local_checkpoint(
139 + lora_request.lora_local_path,
140 + lora_model_id=lora_request.lora_int_id,
141 + device="cpu",
142 + dtype=self.lora_config.lora_dtype,
143 + target_embedding_padding=self.vocab_size +
144 + self.lora_config.lora_extra_vocab_size,
145 + )
146 + except Exception as e:
147 + raise RuntimeError(
148 + f"Loading lora {lora_request.lora_local_path} failed") from e
149 + if lora.rank > self.lora_config.max_lora_rank:
150 + raise ValueError(
151 + f"LoRA rank {lora.rank} is greater than max_lora_rank "
152 + f"{self.lora_config.max_lora_rank}.")
153 + if lora.extra_vocab_size > self.lora_config.lora_extra_vocab_size:
154 + raise ValueError(
155 + f"LoRA added vocab size {lora.extra_vocab_size} is greater than "
156 + f"lora_extra_vocab_size {self.lora_config.lora_extra_vocab_size}."
157 + )
158 + return lora
159 +
160 + def add_dummy_lora(self, lora_request: LoRARequest, rank: int) -> bool:
161 + if lora_request.lora_int_id in self.list_loras():
162 + return False
163 + return self._lora_manager.add_lora(
164 + self._lora_manager.create_dummy_lora(lora_request.lora_int_id,
165 + rank))
166 +
167 + def add_lora(self, lora_request: LoRARequest) -> bool:
168 + if lora_request.lora_int_id in self.list_loras():
169 + return False
170 + lora = self._load_lora(lora_request)
171 + loaded = self._lora_manager.add_lora(lora)
172 + self._lora_manager.activate_lora(lora.id)
173 + return loaded
174 +
175 + def remove_lora(self, lora_id: int) -> bool:
176 + return self._lora_manager.remove_lora(lora_id)
177 +
178 + def remove_all_loras(self) -> bool:
179 + self._lora_manager.remove_all_loras()
180 +
181 + def list_loras(self) -> Set[int]:
182 + return set(self._lora_manager.list_loras())
183 +
184 +
185 +class LRUCacheWorkerLoRAManager(WorkerLoRAManager):
186 + """WorkerLoRAManager that manages LoRA models on the worker side.
187 +
188 + Uses an LRU Cache. Every request, the requested LoRAs will be loaded
189 + (unless they are already loaded) and least recently used LoRAs will
190 + be unloaded if the cache is above capacity."""
191 +
192 + _lora_manager_cls: Type[
193 + LRUCacheLoRAModelManager] = LRUCacheLoRAModelManager
194 +
195 + def create_lora_manager(
196 + self,
197 + model: torch.nn.Module,
198 + target_modules: Union[str, List[str]] = TARGET_MODULES_QKV,
199 + ) -> Any:
200 + lora_manager = create_lora_manager(
201 + model,
202 + target_modules=target_modules,
203 + lora_manager_cls=self._lora_manager_cls,
204 + max_num_seqs=self.max_num_seqs,
205 + vocab_size=self.vocab_size,
206 + lora_config=self.lora_config,
207 + max_num_batched_tokens=self.max_num_batched_tokens,
208 + )
209 + self._lora_manager: LRUCacheLoRAModelManager = lora_manager
210 + return lora_manager.model
211 +
212 + def _apply_loras(self, lora_requests: List[LoRARequest]) -> None:
213 + loras_map = {
214 + lora_request.lora_int_id: lora_request
215 + for lora_request in lora_requests if lora_request
216 + }
217 + if len(loras_map) > self._lora_manager.lora_slots:
218 + raise RuntimeError(
219 + f"Number of requested LoRAs ({len(loras_map)}) is greater "
220 + "than the number of GPU LoRA slots "
221 + f"({self._lora_manager.lora_slots}).")
222 + for lora in loras_map.values():
223 + self.add_lora(lora)
224 +
225 + def add_lora(self, lora_request: LoRARequest) -> bool:
226 + if lora_request.lora_int_id not in self.list_loras():
227 + # Remove before we load the new lora to save memory
228 + if len(self._lora_manager) + 1 > self._lora_manager.capacity:
229 + self._lora_manager.remove_oldest_lora()
230 + lora = self._load_lora(lora_request)
231 + loaded = self._lora_manager.add_lora(lora)
232 + else:
233 + # If the lora is already loaded, just touch it to
234 + # update its position in the caches
235 + loaded = self._lora_manager.get_lora(lora_request.lora_int_id)
236 + self._lora_manager.activate_lora(lora_request.lora_int_id)
237 + return loaded
vllm/model_executor/layers/sampler.py
+18 -17
@@ -27,9 +27,25 @@ class Sampler(nn.Module):
27 parameters (e.g., sampling method, temperature, top-p, top-k, etc.).
28 """
29
30 - def __init__(self, vocab_size: int) -> None:
30 + def __init__(self,
31 + vocab_size: int,
32 + org_vocab_size: Optional[int] = None) -> None:
33 super().__init__()
34 self.vocab_size = vocab_size
35 + # original vocabulary size (without LoRA).
36 + self.org_vocab_size = org_vocab_size or vocab_size
37 +
38 + def _get_logits(self, hidden_states: torch.Tensor, embedding: torch.Tensor,
39 + embedding_bias: Optional[torch.Tensor]) -> torch.Tensor:
40 + # Get the logits for the next tokens.
41 + logits = torch.matmul(hidden_states, embedding.t())
42 + if embedding_bias is not None:
43 + logits += embedding_bias
44 + logits = tensor_model_parallel_gather(logits)
45 + # Remove paddings in vocab (if any).
46 + if logits is not None:
47 + logits = logits[:, :self.org_vocab_size]
48 + return logits
49
50 def forward(
51 self,
@@ -42,8 +58,7 @@ class Sampler(nn.Module):
58 hidden_states = _prune_hidden_states(hidden_states, sampling_metadata)
59
60 # Get the logits for the next tokens.
45 - logits = _get_logits(hidden_states, embedding, embedding_bias,
46 - self.vocab_size)
61 + logits = self._get_logits(hidden_states, embedding, embedding_bias)
62
63 # Only perform sampling in the driver worker.
64 # Note: `_get_logits` is still distributed across TP workers because
@@ -98,20 +113,6 @@ class Sampler(nn.Module):
113 prompt_logprobs, sample_logprobs)
114
115
101 -def _get_logits(hidden_states: torch.Tensor, embedding: torch.Tensor,
102 - embedding_bias: Optional[torch.Tensor],
103 - vocab_size: int) -> Optional[torch.Tensor]:
104 - # Get the logits for the next tokens.
105 - logits = torch.matmul(hidden_states, embedding.t())
106 - if embedding_bias is not None:
107 - logits += embedding_bias
108 - logits = tensor_model_parallel_gather(logits)
109 - # Remove paddings in vocab (if any).
110 - if logits is not None:
111 - logits = logits[:, :vocab_size]
112 - return logits
113 -
114 -
116 def _prune_hidden_states(
117 hidden_states: torch.Tensor,
118 sampling_metadata: SamplingMetadata,
vllm/model_executor/layers/vocab_parallel_embedding.py
+20 -6
@@ -13,8 +13,11 @@ from vllm.model_executor.parallel_utils.communication_op import (
13 tensor_model_parallel_all_reduce)
14 from vllm.model_executor.utils import set_weight_attrs
15
16 +DEFAULT_VOCAB_PADDING_SIZE = 64
17
17 -def pad_vocab_size(vocab_size: int, pad_to: int = 64) -> int:
18 +
19 +def pad_vocab_size(vocab_size: int,
20 + pad_to: int = DEFAULT_VOCAB_PADDING_SIZE) -> int:
21 """Pad the vocab size to the given value."""
22 return ((vocab_size + pad_to - 1) // pad_to) * pad_to
23
@@ -43,17 +46,23 @@ class VocabParallelEmbedding(torch.nn.Module):
46 num_embeddings: vocabulary size.
47 embedding_dim: size of hidden state.
48 params_dtype: type of the parameters.
49 + org_num_embeddings: original vocabulary size (without LoRA).
50 + padding_size: padding size for the vocabulary.
51 """
52
53 def __init__(self,
54 num_embeddings: int,
55 embedding_dim: int,
51 - params_dtype: Optional[torch.dtype] = None):
56 + params_dtype: Optional[torch.dtype] = None,
57 + org_num_embeddings: Optional[int] = None,
58 + padding_size: int = DEFAULT_VOCAB_PADDING_SIZE):
59 super().__init__()
60
61 # Keep the input dimensions.
62 self.num_embeddings = num_embeddings
56 - self.num_embeddings_padded = pad_vocab_size(num_embeddings)
63 + self.org_vocab_size = org_num_embeddings or num_embeddings
64 + self.num_embeddings_padded = pad_vocab_size(num_embeddings,
65 + padding_size)
66 self.embedding_dim = embedding_dim
67 if params_dtype is None:
68 params_dtype = torch.get_default_dtype()
@@ -77,7 +86,7 @@ class VocabParallelEmbedding(torch.nn.Module):
86
87 def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor):
88 parallel_dim = param.parallel_dim
80 - assert loaded_weight.shape[parallel_dim] == self.num_embeddings
89 + assert loaded_weight.shape[parallel_dim] == self.org_vocab_size
90 loaded_weight = loaded_weight[self.vocab_start_index:self.
91 vocab_end_index]
92 param[:loaded_weight.shape[0]].data.copy_(loaded_weight)
@@ -114,14 +123,19 @@ class ParallelLMHead(VocabParallelEmbedding):
123 embedding_dim: size of hidden state.
124 bias: whether to use bias.
125 params_dtype: type of the parameters.
126 + org_num_embeddings: original vocabulary size (without LoRA).
127 + padding_size: padding size for the vocabulary.
128 """
129
130 def __init__(self,
131 num_embeddings: int,
132 embedding_dim: int,
133 bias: bool = False,
123 - params_dtype: Optional[torch.dtype] = None):
124 - super().__init__(num_embeddings, embedding_dim, params_dtype)
134 + params_dtype: Optional[torch.dtype] = None,
135 + org_num_embeddings: Optional[int] = None,
136 + padding_size: int = DEFAULT_VOCAB_PADDING_SIZE):
137 + super().__init__(num_embeddings, embedding_dim, params_dtype,
138 + org_num_embeddings, padding_size)
139 if bias:
140 self.bias = Parameter(
141 torch.empty(self.num_embeddings_per_partition,
vllm/model_executor/model_loader.py
+15 -4
@@ -1,12 +1,12 @@
1 """Utilities for selecting and loading models."""
2 import contextlib
3 -from typing import Type
3 +from typing import Optional, Type
4
5 import torch
6 import torch.nn as nn
7 from transformers import PretrainedConfig
8
9 -from vllm.config import ModelConfig
9 +from vllm.config import ModelConfig, LoRAConfig
10 from vllm.model_executor.models import ModelRegistry
11 from vllm.model_executor.weight_utils import (get_quant_config,
12 initialize_dummy_weights)
@@ -32,7 +32,8 @@ def _get_model_architecture(config: PretrainedConfig) -> Type[nn.Module]:
32 f"Supported architectures: {ModelRegistry.get_supported_archs()}")
33
34
35 -def get_model(model_config: ModelConfig) -> nn.Module:
35 +def get_model(model_config: ModelConfig,
36 + lora_config: Optional[LoRAConfig] = None) -> nn.Module:
37 model_class = _get_model_architecture(model_config.hf_config)
38
39 # Get the (maybe quantized) linear method.
@@ -62,7 +63,17 @@ def get_model(model_config: ModelConfig) -> nn.Module:
63 # Create a model instance.
64 # The weights will be initialized as empty tensors.
65 with torch.device("cuda"):
65 - model = model_class(model_config.hf_config, linear_method)
66 + if getattr(model_class, "supports_lora", False):
67 + model = model_class(model_config.hf_config, linear_method,
68 + lora_config)
69 + elif lora_config:
70 + raise ValueError(
71 + f"Model {model_class.__name__} does not support LoRA, "
72 + "but LoRA is enabled. Support for this model may "
73 + "be added in the future. If this is important to you, "
74 + "please open an issue on github.")
75 + else:
76 + model = model_class(model_config.hf_config, linear_method)
77 if model_config.load_format == "dummy":
78 # NOTE(woosuk): For accurate performance evaluation, we assign
79 # random values to the weights.
vllm/model_executor/models/llama.py
+25 -6
@@ -38,13 +38,14 @@ from vllm.model_executor.layers.linear import (LinearMethodBase,
38 from vllm.model_executor.layers.rotary_embedding import get_rope
39 from vllm.model_executor.layers.sampler import Sampler
40 from vllm.model_executor.layers.vocab_parallel_embedding import (
41 - VocabParallelEmbedding, ParallelLMHead)
41 + VocabParallelEmbedding, ParallelLMHead, DEFAULT_VOCAB_PADDING_SIZE)
42 from vllm.model_executor.parallel_utils.parallel_state import (
43 get_tensor_model_parallel_world_size)
44 from vllm.model_executor.sampling_metadata import SamplingMetadata
45 from vllm.model_executor.weight_utils import (default_weight_loader,
46 hf_model_weights_iterator)
47 from vllm.sequence import SamplerOutput
48 +from vllm.config import LoRAConfig
49
50 KVCache = Tuple[torch.Tensor, torch.Tensor]
51
@@ -225,14 +226,19 @@ class LlamaModel(nn.Module):
226 self,
227 config: LlamaConfig,
228 linear_method: Optional[LinearMethodBase] = None,
229 + lora_config: Optional[LoRAConfig] = None,
230 ) -> None:
231 super().__init__()
232 self.config = config
233 self.padding_idx = config.pad_token_id
232 - self.vocab_size = config.vocab_size
234 + lora_vocab = (lora_config.lora_extra_vocab_size *
235 + (lora_config.max_loras or 1)) if lora_config else 0
236 + self.vocab_size = config.vocab_size + lora_vocab
237 + self.org_vocab_size = config.vocab_size
238 self.embed_tokens = VocabParallelEmbedding(
234 - config.vocab_size,
239 + self.vocab_size,
240 config.hidden_size,
241 + org_num_embeddings=config.vocab_size,
242 )
243 self.layers = nn.ModuleList([
244 LlamaDecoderLayer(config, linear_method)
@@ -263,18 +269,31 @@ class LlamaModel(nn.Module):
269
270
271 class LlamaForCausalLM(nn.Module):
272 + supports_lora = True
273
274 def __init__(
275 self,
276 config: LlamaConfig,
277 linear_method: Optional[LinearMethodBase] = None,
278 + lora_config: Optional[LoRAConfig] = None,
279 ) -> None:
280 super().__init__()
281 self.config = config
282 self.linear_method = linear_method
275 - self.model = LlamaModel(config, linear_method)
276 - self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
277 - self.sampler = Sampler(config.vocab_size)
283 + self.model = LlamaModel(config, linear_method, lora_config=lora_config)
284 + unpadded_vocab_size = config.vocab_size
285 + if lora_config:
286 + unpadded_vocab_size += lora_config.lora_extra_vocab_size
287 + self.lm_head = ParallelLMHead(
288 + unpadded_vocab_size,
289 + config.hidden_size,
290 + org_num_embeddings=config.vocab_size,
291 + padding_size=DEFAULT_VOCAB_PADDING_SIZE
292 + # We need bigger padding if using lora for kernel
293 + # compatibility
294 + if not lora_config else lora_config.lora_vocab_padding_size,
295 + )
296 + self.sampler = Sampler(unpadded_vocab_size, config.vocab_size)
297
298 def forward(
299 self,
vllm/model_executor/models/mistral.py
+27 -6
@@ -38,13 +38,14 @@ from vllm.model_executor.layers.linear import (LinearMethodBase,
38 from vllm.model_executor.layers.rotary_embedding import get_rope
39 from vllm.model_executor.layers.sampler import Sampler
40 from vllm.model_executor.layers.vocab_parallel_embedding import (
41 - VocabParallelEmbedding, ParallelLMHead)
41 + VocabParallelEmbedding, ParallelLMHead, DEFAULT_VOCAB_PADDING_SIZE)
42 from vllm.model_executor.parallel_utils.parallel_state import (
43 get_tensor_model_parallel_world_size)
44 from vllm.model_executor.sampling_metadata import SamplingMetadata
45 from vllm.model_executor.weight_utils import (default_weight_loader,
46 hf_model_weights_iterator)
47 from vllm.sequence import SamplerOutput
48 +from vllm.config import LoRAConfig
49
50 KVCache = Tuple[torch.Tensor, torch.Tensor]
51
@@ -220,15 +221,20 @@ class MistralModel(nn.Module):
221 self,
222 config: MistralConfig,
223 linear_method: Optional[LinearMethodBase] = None,
224 + lora_config: Optional[LoRAConfig] = None,
225 ) -> None:
226 super().__init__()
227 self.config = config
228 self.padding_idx = config.pad_token_id
227 - self.vocab_size = config.vocab_size
229 + lora_vocab = (lora_config.lora_extra_vocab_size *
230 + (lora_config.max_loras or 1)) if lora_config else 0
231 + self.vocab_size = config.vocab_size + lora_vocab
232 + self.org_vocab_size = config.vocab_size
233
234 self.embed_tokens = VocabParallelEmbedding(
230 - config.vocab_size,
235 + self.vocab_size,
236 config.hidden_size,
237 + org_num_embeddings=config.vocab_size,
238 )
239 self.layers = nn.ModuleList([
240 MistralDecoderLayer(config, linear_method)
@@ -259,18 +265,33 @@ class MistralModel(nn.Module):
265
266
267 class MistralForCausalLM(nn.Module):
268 + supports_lora = True
269
270 def __init__(
271 self,
272 config: MistralConfig,
273 linear_method: Optional[LinearMethodBase] = None,
274 + lora_config: Optional[LoRAConfig] = None,
275 ) -> None:
276 super().__init__()
277 self.config = config
278 self.linear_method = linear_method
271 - self.model = MistralModel(config, linear_method)
272 - self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
273 - self.sampler = Sampler(config.vocab_size)
279 + self.model = MistralModel(config,
280 + linear_method,
281 + lora_config=lora_config)
282 + unpadded_vocab_size = config.vocab_size
283 + if lora_config:
284 + unpadded_vocab_size += lora_config.lora_extra_vocab_size
285 + self.lm_head = ParallelLMHead(
286 + unpadded_vocab_size,
287 + config.hidden_size,
288 + org_num_embeddings=config.vocab_size,
289 + padding_size=DEFAULT_VOCAB_PADDING_SIZE
290 + # We need bigger padding if using lora for kernel
291 + # compatibility
292 + if not lora_config else lora_config.lora_vocab_padding_size,
293 + )
294 + self.sampler = Sampler(unpadded_vocab_size, config.vocab_size)
295
296 def forward(
297 self,
vllm/model_executor/parallel_utils/parallel_state.py
+5 -1
@@ -195,10 +195,14 @@ def get_pipeline_model_parallel_prev_rank():
195
196
197 def destroy_model_parallel():
198 - """Set the groups to none."""
198 + """Set the groups to none and destroy them."""
199 global _TENSOR_MODEL_PARALLEL_GROUP
200 + if _TENSOR_MODEL_PARALLEL_GROUP:
201 + torch.distributed.destroy_process_group(_TENSOR_MODEL_PARALLEL_GROUP)
202 _TENSOR_MODEL_PARALLEL_GROUP = None
203 global _PIPELINE_MODEL_PARALLEL_GROUP
204 + if _PIPELINE_MODEL_PARALLEL_GROUP:
205 + torch.distributed.destroy_process_group(_PIPELINE_MODEL_PARALLEL_GROUP)
206 _PIPELINE_MODEL_PARALLEL_GROUP = None
207 global _PIPELINE_GLOBAL_RANKS
208 _PIPELINE_GLOBAL_RANKS = None
vllm/outputs.py
+16 -3
@@ -2,6 +2,7 @@ from typing import List, Optional
2
3 from vllm.sequence import (PromptLogprobs, SampleLogprobs, SequenceGroup,
4 SequenceStatus)
5 +from vllm.lora.request import LoRARequest
6
7
8 class CompletionOutput:
@@ -16,6 +17,7 @@ class CompletionOutput:
17 logprobs: The log probabilities of the top probability words at each
18 position if the logprobs are requested.
19 finish_reason: The reason why the sequence is finished.
20 + lora_request: The LoRA request that was used to generate the output.
21 """
22
23 def __init__(
@@ -26,6 +28,7 @@ class CompletionOutput:
28 cumulative_logprob: float,
29 logprobs: Optional[SampleLogprobs],
30 finish_reason: Optional[str] = None,
31 + lora_request: Optional[LoRARequest] = None,
32 ) -> None:
33 self.index = index
34 self.text = text
@@ -33,6 +36,7 @@ class CompletionOutput:
36 self.cumulative_logprob = cumulative_logprob
37 self.logprobs = logprobs
38 self.finish_reason = finish_reason
39 + self.lora_request = lora_request
40
41 def finished(self) -> bool:
42 return self.finish_reason is not None
@@ -56,6 +60,7 @@ class RequestOutput:
60 prompt_logprobs: The log probabilities to return per prompt token.
61 outputs: The output sequences of the request.
62 finished: Whether the whole request is finished.
63 + lora_request: The LoRA request that was used to generate the output.
64 """
65
66 def __init__(
@@ -66,6 +71,7 @@ class RequestOutput:
71 prompt_logprobs: Optional[PromptLogprobs],
72 outputs: List[CompletionOutput],
73 finished: bool,
74 + lora_request: Optional[LoRARequest] = None,
75 ) -> None:
76 self.request_id = request_id
77 self.prompt = prompt
@@ -73,6 +79,7 @@ class RequestOutput:
79 self.prompt_logprobs = prompt_logprobs
80 self.outputs = outputs
81 self.finished = finished
82 + self.lora_request = lora_request
83
84 @classmethod
85 def from_seq_group(cls, seq_group: SequenceGroup) -> "RequestOutput":
@@ -108,8 +115,13 @@ class RequestOutput:
115 prompt_token_ids = seq_group.prompt_token_ids
116 prompt_logprobs = seq_group.prompt_logprobs
117 finished = seq_group.is_finished()
111 - return cls(seq_group.request_id, prompt, prompt_token_ids,
112 - prompt_logprobs, outputs, finished)
118 + return cls(seq_group.request_id,
119 + prompt,
120 + prompt_token_ids,
121 + prompt_logprobs,
122 + outputs,
123 + finished,
124 + lora_request=seq_group.lora_request)
125
126 def __repr__(self) -> str:
127 return (f"RequestOutput(request_id={self.request_id}, "
@@ -117,4 +129,5 @@ class RequestOutput:
129 f"prompt_token_ids={self.prompt_token_ids}, "
130 f"prompt_logprobs={self.prompt_logprobs}, "
131 f"outputs={self.outputs}, "
120 - f"finished={self.finished})")
132 + f"finished={self.finished}, "
133 + f"lora_request={self.lora_request})")
vllm/prefix.py
+3 -2
@@ -74,13 +74,14 @@ class PrefixPool:
74 new_length = len(token_ids) // self.block_size * self.block_size
75 return tuple(token_ids[:new_length])
76
77 - def add_or_get_prefix(self, token_ids: Sequence[int]) -> Optional[Prefix]:
77 + def add_or_get_prefix(self, token_ids: Sequence[int],
78 + lora_int_id: int) -> Optional[Prefix]:
79 token_ids = self._truncate_token_ids(token_ids)
80 if len(token_ids) == 0:
81 # Prefix is empty.
82 return None
83 prefix = Prefix(token_ids, self.block_size)
83 - prefix_hash = hash(prefix)
84 + prefix_hash = hash((prefix, lora_int_id))
85 if prefix_hash not in self.prefixes:
86 self.prefixes[prefix_hash] = prefix
87 return self.prefixes[prefix_hash]
vllm/sequence.py
+22
@@ -6,6 +6,7 @@ from typing import Dict, List, Optional, Union
6 from vllm.block import LogicalTokenBlock
7 from vllm.prefix import Prefix
8 from vllm.sampling_params import SamplingParams
9 +from vllm.lora.request import LoRARequest
10
11 PromptLogprobs = List[Optional[Dict[int, float]]]
12 SampleLogprobs = List[Dict[int, float]]
@@ -106,6 +107,7 @@ class Sequence:
107 prompt_token_ids: The token IDs of the prompt.
108 block_size: The block size of the sequence. Should be the same as the
109 block size used by the block manager and cache engine.
110 + lora_request: LoRA request.
111 """
112
113 def __init__(
@@ -114,10 +116,12 @@ class Sequence:
116 prompt: str,
117 prompt_token_ids: List[int],
118 block_size: int,
119 + lora_request: Optional[LoRARequest] = None,
120 ) -> None:
121 self.seq_id = seq_id
122 self.prompt = prompt
123 self.block_size = block_size
124 + self.lora_request = lora_request
125
126 self.data = SequenceData(prompt_token_ids)
127 self.output_logprobs: SampleLogprobs = []
@@ -134,6 +138,10 @@ class Sequence:
138 # Input + output tokens
139 self.tokens: Optional[List[str]] = None
140
141 + @property
142 + def lora_int_id(self) -> int:
143 + return self.lora_request.lora_int_id if self.lora_request else 0
144 +
145 def _append_logical_block(self) -> None:
146 block = LogicalTokenBlock(
147 block_number=len(self.logical_token_blocks),
@@ -229,6 +237,7 @@ class SequenceGroup:
237 seqs: The list of sequences.
238 sampling_params: The sampling parameters used to generate the outputs.
239 arrival_time: The arrival time of the request.
240 + lora_request: LoRA request.
241 prefix: The prefix of the prompt of the sequence group.
242 """
243
@@ -238,12 +247,14 @@ class SequenceGroup:
247 seqs: List[Sequence],
248 sampling_params: SamplingParams,
249 arrival_time: float,
250 + lora_request: Optional[LoRARequest] = None,
251 prefix: Optional[Prefix] = None,
252 ) -> None:
253 self.request_id = request_id
254 self.seqs_dict = {seq.seq_id: seq for seq in seqs}
255 self.sampling_params = sampling_params
256 self.arrival_time = arrival_time
257 + self.lora_request = lora_request
258 self.prefix: Optional[Prefix] = prefix
259 self.prompt_logprobs: Optional[PromptLogprobs] = None
260
@@ -259,6 +270,10 @@ class SequenceGroup:
270 # We use the prompt of an arbitrary sequence.
271 return next(iter(self.seqs_dict.values())).data.prompt_token_ids
272
273 + @property
274 + def lora_int_id(self) -> int:
275 + return self.lora_request.lora_int_id if self.lora_request else 0
276 +
277 def get_max_num_running_seqs(self) -> int:
278 """The maximum number of sequences running in parallel in the remaining
279 lifetime of the request."""
@@ -338,6 +353,7 @@ class SequenceGroupMetadata:
353 sampling_params: The sampling parameters used to generate the outputs.
354 block_tables: The block tables. (Seq id -> list of physical block
355 numbers)
356 + lora_request: LoRA request.
357 prefix: The prefix of the prompt of the sequence group.
358 """
359
@@ -348,6 +364,7 @@ class SequenceGroupMetadata:
364 seq_data: Dict[int, SequenceData],
365 sampling_params: SamplingParams,
366 block_tables: Dict[int, List[int]],
367 + lora_request: Optional[LoRARequest] = None,
368 prefix: Optional[Prefix] = None,
369 ) -> None:
370 self.request_id = request_id
@@ -355,8 +372,13 @@ class SequenceGroupMetadata:
372 self.seq_data = seq_data
373 self.sampling_params = sampling_params
374 self.block_tables = block_tables
375 + self.lora_request = lora_request
376 self.prefix = prefix
377
378 + @property
379 + def lora_int_id(self) -> int:
380 + return self.lora_request.lora_int_id if self.lora_request else 0
381 +
382
383 class SequenceOutput:
384 """The model output associated with a sequence.
vllm/transformers_utils/tokenizer.py
+80
@@ -4,6 +4,8 @@ from transformers import (AutoTokenizer, PreTrainedTokenizer,
4 PreTrainedTokenizerFast)
5
6 from vllm.logger import init_logger
7 +from vllm.lora.request import LoRARequest
8 +from vllm.utils import make_async, LRUCache
9 from vllm.transformers_utils.tokenizers import *
10
11 logger = init_logger(__name__)
@@ -65,6 +67,84 @@ def get_tokenizer(
67 return tokenizer
68
69
70 +def get_lora_tokenizer(lora_request: LoRARequest, *args,
71 + **kwargs) -> Optional[PreTrainedTokenizer]:
72 + if lora_request is None:
73 + return None
74 + try:
75 + tokenizer = get_tokenizer(lora_request.lora_local_path, *args,
76 + **kwargs)
77 + except OSError as e:
78 + # No tokenizer was found in the LoRA folder,
79 + # use base model tokenizer
80 + logger.warning(
81 + f"No tokenizer found in {lora_request.lora_local_path}, "
82 + "using base model tokenizer instead. "
83 + f"(Exception: {str(e)})")
84 + tokenizer = None
85 + return tokenizer
86 +
87 +
88 +get_lora_tokenizer_async = make_async(get_lora_tokenizer)
89 +
90 +
91 +class TokenizerGroup:
92 + """A group of tokenizers that can be used for LoRA adapters."""
93 +
94 + def __init__(self, tokenizer_id: str, enable_lora: bool, max_num_seqs: int,
95 + max_input_length: Optional[int], **tokenizer_config):
96 + self.tokenizer_id = tokenizer_id
97 + self.tokenizer_config = tokenizer_config
98 + self.enable_lora = enable_lora
99 + self.max_input_length = max_input_length
100 + self.tokenizer = get_tokenizer(self.tokenizer_id, **tokenizer_config)
101 + if enable_lora:
102 + self.lora_tokenizers = LRUCache(capacity=max_num_seqs)
103 + else:
104 + self.lora_tokenizers = None
105 +
106 + def encode(self,
107 + prompt: str,
108 + request_id: Optional[str] = None,
109 + lora_request: Optional[LoRARequest] = None) -> List[int]:
110 + tokenizer = self.get_lora_tokenizer(lora_request)
111 + return tokenizer.encode(prompt)
112 +
113 + async def encode_async(
114 + self,
115 + prompt: str,
116 + request_id: Optional[str] = None,
117 + lora_request: Optional[LoRARequest] = None) -> List[int]:
118 + tokenizer = await self.get_lora_tokenizer_async(lora_request)
119 + return tokenizer.encode(prompt)
120 +
121 + def get_lora_tokenizer(
122 + self,
123 + lora_request: Optional[LoRARequest]) -> "PreTrainedTokenizer":
124 + if not lora_request or not self.enable_lora:
125 + return self.tokenizer
126 + if lora_request.lora_int_id not in self.lora_tokenizers:
127 + tokenizer = (get_lora_tokenizer(
128 + lora_request, **self.tokenizer_config) or self.tokenizer)
129 + self.lora_tokenizers.put(lora_request.lora_int_id, tokenizer)
130 + return tokenizer
131 + else:
132 + return self.lora_tokenizers.get(lora_request.lora_int_id)
133 +
134 + async def get_lora_tokenizer_async(
135 + self,
136 + lora_request: Optional[LoRARequest]) -> "PreTrainedTokenizer":
137 + if not lora_request or not self.enable_lora:
138 + return self.tokenizer
139 + if lora_request.lora_int_id not in self.lora_tokenizers:
140 + tokenizer = (await get_lora_tokenizer_async(
141 + lora_request, **self.tokenizer_config) or self.tokenizer)
142 + self.lora_tokenizers.put(lora_request.lora_int_id, tokenizer)
143 + return tokenizer
144 + else:
145 + return self.lora_tokenizers.get(lora_request.lora_int_id)
146 +
147 +
148 def _convert_tokens_to_string_with_added_encoders(
149 tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
150 output_tokens: List[str],
vllm/utils.py
+90
@@ -7,6 +7,17 @@ from typing import List
7
8 import psutil
9 import torch
10 +import asyncio
11 +from functools import partial
12 +from typing import (
13 + Awaitable,
14 + Callable,
15 + TypeVar,
16 +)
17 +from collections import OrderedDict
18 +from typing import Any, Hashable, Optional
19 +
20 +T = TypeVar("T")
21
22
23 class Device(enum.Enum):
@@ -28,6 +39,69 @@ class Counter:
39 self.counter = 0
40
41
42 +class LRUCache:
43 +
44 + def __init__(self, capacity: int):
45 + self.cache = OrderedDict()
46 + self.capacity = capacity
47 +
48 + def __contains__(self, key: Hashable) -> bool:
49 + return key in self.cache
50 +
51 + def __len__(self) -> int:
52 + return len(self.cache)
53 +
54 + def __getitem__(self, key: Hashable) -> Any:
55 + return self.get(key)
56 +
57 + def __setitem__(self, key: Hashable, value: Any) -> None:
58 + self.put(key, value)
59 +
60 + def __delitem__(self, key: Hashable) -> None:
61 + self.pop(key)
62 +
63 + def touch(self, key: Hashable) -> None:
64 + self.cache.move_to_end(key)
65 +
66 + def get(self, key: Hashable, default_value: Optional[Any] = None) -> int:
67 + if key in self.cache:
68 + value = self.cache[key]
69 + self.cache.move_to_end(key)
70 + else:
71 + value = default_value
72 + return value
73 +
74 + def put(self, key: Hashable, value: Any) -> None:
75 + self.cache[key] = value
76 + self.cache.move_to_end(key)
77 + self._remove_old_if_needed()
78 +
79 + def _on_remove(self, key: Hashable, value: Any):
80 + pass
81 +
82 + def remove_oldest(self):
83 + if not self.cache:
84 + return
85 + key, value = self.cache.popitem(last=False)
86 + self._on_remove(key, value)
87 +
88 + def _remove_old_if_needed(self) -> None:
89 + while len(self.cache) > self.capacity:
90 + self.remove_oldest()
91 +
92 + def pop(self, key: int, default_value: Optional[Any] = None) -> Any:
93 + run_on_remove = key in self.cache
94 + value = self.cache.pop(key, default_value)
95 + if run_on_remove:
96 + self._on_remove(key, value)
97 + return value
98 +
99 + def clear(self):
100 + while len(self.cache) > 0:
101 + self.remove_oldest()
102 + self.cache.clear()
103 +
104 +
105 def is_hip() -> bool:
106 return torch.version.hip is not None
107
@@ -59,6 +133,22 @@ def in_wsl() -> bool:
133 return "microsoft" in " ".join(uname()).lower()
134
135
136 +def make_async(func: Callable[..., T]) -> Callable[..., Awaitable[T]]:
137 + """Take a blocking function, and run it on in an executor thread.
138 +
139 + This function prevents the blocking function from blocking the
140 + asyncio event loop.
141 + The code in this function needs to be thread safe.
142 + """
143 +
144 + def _async_wrapper(*args, **kwargs) -> asyncio.Future:
145 + loop = asyncio.get_event_loop()
146 + p_func = partial(func, *args, **kwargs)
147 + return loop.run_in_executor(executor=None, func=p_func)
148 +
149 + return _async_wrapper
150 +
151 +
152 def get_ip() -> str:
153 s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
154 s.connect(("8.8.8.8", 80)) # Doesn't need to be reachable
vllm/worker/model_runner.py
+146 -15
@@ -1,23 +1,27 @@
1 import time
2 -from typing import Dict, List, Optional, Tuple, Union
2 +from typing import Dict, List, Optional, Tuple, Set, Union
3
4 import numpy as np
5 import torch
6 import torch.nn as nn
7
8 -from vllm.config import ModelConfig, ParallelConfig, SchedulerConfig
8 +from vllm.config import ModelConfig, LoRAConfig, ParallelConfig, SchedulerConfig
9 from vllm.logger import init_logger
10 from vllm.model_executor import get_model, InputMetadata, SamplingMetadata
11 from vllm.model_executor.parallel_utils.communication_op import (
12 broadcast_tensor_dict)
13 from vllm.sampling_params import SamplingParams, SamplingType
14 from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
15 +from vllm.lora.worker_manager import LRUCacheWorkerLoRAManager
16 +from vllm.lora.layers import LoRAMapping
17 +from vllm.lora.request import LoRARequest
18 from vllm.utils import in_wsl
19
20 logger = init_logger(__name__)
21
22 KVCache = Tuple[torch.Tensor, torch.Tensor]
23 _PAD_SLOT_ID = -1
24 +LORA_WARMUP_RANK = 8
25 # Capture graphs for batch size 1, 2, 4, 8, 16, 24, 32, 40, ..., 256.
26 # NOTE: _get_graph_batch_size needs to be updated if this list is changed.
27 _BATCH_SIZES_TO_CAPTURE = [1, 2, 4] + [8 * i for i in range(1, 33)]
@@ -30,19 +34,23 @@ class ModelRunner:
34 model_config: ModelConfig,
35 parallel_config: ParallelConfig,
36 scheduler_config: SchedulerConfig,
37 + lora_config: Optional[LoRAConfig],
38 is_driver_worker: bool = False,
39 ):
40 self.model_config = model_config
41 self.parallel_config = parallel_config
42 self.scheduler_config = scheduler_config
43 + self.lora_config = lora_config
44 self.is_driver_worker = is_driver_worker
45
46 # model_config can be None in tests/samplers/test_sampler.py.
47 # FIXME(woosuk): This is a hack to make the tests work. Refactor this.
48 self.sliding_window = (model_config.get_sliding_window()
49 if model_config is not None else None)
50 + self.device = torch.device(torch.cuda.current_device())
51 self.model = None
52 self.block_size = None # Set after initial profiling.
53 + self.lora_manager = None
54
55 self.graph_runners: Dict[int, CUDAGraphRunner] = {}
56 self.graph_memory_pool = None # Set during graph capture.
@@ -61,7 +69,17 @@ class ModelRunner:
69 self.in_wsl = in_wsl()
70
71 def load_model(self) -> None:
64 - self.model = get_model(self.model_config)
72 + self.model = get_model(self.model_config, self.lora_config)
73 +
74 + vocab_size = self.model.config.vocab_size
75 +
76 + if self.lora_config:
77 + self.lora_manager = LRUCacheWorkerLoRAManager(
78 + self.scheduler_config.max_num_seqs,
79 + self.scheduler_config.max_num_batched_tokens +
80 + self.scheduler_config.max_paddings, vocab_size,
81 + self.lora_config, self.device)
82 + self.model = self.lora_manager.create_lora_manager(self.model)
83
84 def set_block_size(self, block_size: int) -> None:
85 self.block_size = block_size
@@ -74,12 +92,15 @@ class ModelRunner:
92 def _prepare_prompt(
93 self,
94 seq_group_metadata_list: List[SequenceGroupMetadata],
77 - ) -> Tuple[torch.Tensor, torch.Tensor, InputMetadata, List[int],
78 - List[int]]:
95 + ) -> Tuple[torch.Tensor, torch.Tensor, InputMetadata, List[int], List[int],
96 + List[int], List[int], Set[LoRARequest]]:
97 assert len(seq_group_metadata_list) > 0
98 input_tokens: List[List[int]] = []
99 input_positions: List[List[int]] = []
100 slot_mapping: List[List[int]] = []
101 + lora_index_mapping: List[int] = []
102 + lora_prompt_mapping: List[int] = []
103 + lora_requests: Set[LoRARequest] = set()
104
105 prompt_lens: List[int] = []
106 context_lens: List[int] = []
@@ -113,6 +134,17 @@ class ModelRunner:
134 input_positions.append(
135 list(range(prefix_len, prefix_len + len(prompt_tokens))))
136
137 + lora_id = seq_group_metadata.lora_int_id
138 +
139 + if lora_id > 0:
140 + lora_requests.add(seq_group_metadata.lora_request)
141 +
142 + lora_index_mapping.append([lora_id] * prompt_len)
143 + lora_prompt_mapping.extend(
144 + [lora_id] *
145 + (prompt_len
146 + if seq_group_metadata.sampling_params.prompt_logprobs else 1))
147 +
148 if seq_group_metadata.block_tables is None:
149 # During memory profiling, the block tables are not initialized
150 # yet. In this case, we just use a dummy slot mapping.
@@ -156,6 +188,10 @@ class ModelRunner:
188 max_prompt_len,
189 pad=_PAD_SLOT_ID,
190 dtype=torch.long)
191 + lora_index_mapping = [
192 + _pad_to_max(mapping, max_prompt_len, pad=0)
193 + for mapping in lora_index_mapping
194 + ]
195 context_lens_tensor = torch.tensor(context_lens,
196 dtype=torch.int,
197 device='cuda')
@@ -188,23 +224,33 @@ class ModelRunner:
224 use_cuda_graph=False,
225 )
226 return (input_tokens, input_positions, input_metadata, prompt_lens,
191 - subquery_lens)
227 + subquery_lens, lora_index_mapping, lora_prompt_mapping,
228 + lora_requests)
229
230 def _prepare_decode(
231 self,
232 seq_group_metadata_list: List[SequenceGroupMetadata],
196 - ) -> Tuple[torch.Tensor, torch.Tensor, InputMetadata]:
233 + ) -> Tuple[torch.Tensor, torch.Tensor, InputMetadata, List[int], List[int],
234 + Set[LoRARequest]]:
235 assert len(seq_group_metadata_list) > 0
236 input_tokens: List[List[int]] = []
237 input_positions: List[List[int]] = []
238 slot_mapping: List[List[int]] = []
239 context_lens: List[int] = []
240 block_tables: List[List[int]] = []
241 + lora_index_mapping: List[int] = []
242 + lora_prompt_mapping: List[int] = []
243 + lora_requests: Set[LoRARequest] = set()
244
245 for seq_group_metadata in seq_group_metadata_list:
246 assert not seq_group_metadata.is_prompt
247
248 seq_ids = list(seq_group_metadata.seq_data.keys())
249 + lora_id = seq_group_metadata.lora_int_id
250 +
251 + if lora_id > 0:
252 + lora_requests.add(seq_group_metadata.lora_request)
253 +
254 for seq_id in seq_ids:
255 seq_data = seq_group_metadata.seq_data[seq_id]
256 generation_token = seq_data.get_last_token_id()
@@ -223,6 +269,8 @@ class ModelRunner:
269 block_offset = position % self.block_size
270 slot = block_number * self.block_size + block_offset
271 slot_mapping.append([slot])
272 + lora_index_mapping.append([lora_id])
273 + lora_prompt_mapping.append(lora_id)
274
275 if self.sliding_window is not None:
276 sliding_window_blocks = (self.sliding_window //
@@ -287,6 +335,10 @@ class ModelRunner:
335 device="cuda",
336 )
337
338 + lora_index_mapping = [
339 + _pad_to_max(mapping, 1, pad=0) for mapping in lora_index_mapping
340 + ]
341 +
342 input_metadata = InputMetadata(
343 is_prompt=False,
344 slot_mapping=slot_mapping,
@@ -298,7 +350,7 @@ class ModelRunner:
350 block_tables=block_tables,
351 use_cuda_graph=use_captured_graph,
352 )
301 - return input_tokens, input_positions, input_metadata
353 + return input_tokens, input_positions, input_metadata, lora_index_mapping, lora_prompt_mapping, lora_requests
354
355 def _prepare_sample(
356 self,
@@ -375,7 +427,8 @@ class ModelRunner:
427 def prepare_input_tensors(
428 self,
429 seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
378 - ) -> Tuple[torch.Tensor, torch.Tensor, InputMetadata, SamplingMetadata]:
430 + ) -> Tuple[torch.Tensor, torch.Tensor, InputMetadata, SamplingMetadata,
431 + Set[int], LoRAMapping]:
432 if self.is_driver_worker:
433 # NOTE: We assume that all sequences in the group are all prompts or
434 # all decodes.
@@ -383,16 +436,29 @@ class ModelRunner:
436 # Prepare input tensors.
437 if is_prompt:
438 (input_tokens, input_positions, input_metadata, prompt_lens,
386 - subquery_lens) = self._prepare_prompt(seq_group_metadata_list)
439 + subquery_lens, lora_index_mapping, lora_prompt_mapping,
440 + lora_requests) = self._prepare_prompt(seq_group_metadata_list)
441 else:
388 - (input_tokens, input_positions, input_metadata
389 - ) = self._prepare_decode(seq_group_metadata_list)
390 - subquery_lens = None
442 + (input_tokens, input_positions, input_metadata,
443 + lora_index_mapping, lora_prompt_mapping,
444 + lora_requests) = self._prepare_decode(seq_group_metadata_list)
445 prompt_lens = []
446 + subquery_lens = None
447 sampling_metadata = self._prepare_sample(seq_group_metadata_list,
448 prompt_lens,
449 subquery_lens)
450
451 + if self.lora_config:
452 + flat_lora_index_mapping = [
453 + item for sublist in lora_index_mapping for item in sublist
454 + ]
455 + lora_mapping = LoRAMapping(
456 + flat_lora_index_mapping,
457 + lora_prompt_mapping,
458 + )
459 + else:
460 + lora_mapping = None
461 +
462 # Broadcast the metadata.
463 metadata_dict = {
464 "input_tokens": input_tokens,
@@ -408,12 +474,16 @@ class ModelRunner:
474 "use_cuda_graph": input_metadata.use_cuda_graph,
475 "selected_token_indices":
476 sampling_metadata.selected_token_indices,
477 + "lora_requests": lora_requests,
478 + "lora_mapping": lora_mapping,
479 }
480 broadcast_tensor_dict(metadata_dict, src=0)
481 else:
482 metadata_dict = broadcast_tensor_dict(src=0)
483 input_tokens = metadata_dict["input_tokens"]
484 input_positions = metadata_dict["input_positions"]
485 + lora_mapping = metadata_dict["lora_mapping"]
486 + lora_requests = metadata_dict["lora_requests"]
487 input_metadata = InputMetadata(
488 is_prompt=metadata_dict["is_prompt"],
489 slot_mapping=metadata_dict["slot_mapping"],
@@ -434,7 +504,7 @@ class ModelRunner:
504 perform_sampling=False,
505 )
506
437 - return input_tokens, input_positions, input_metadata, sampling_metadata
507 + return input_tokens, input_positions, input_metadata, sampling_metadata, lora_requests, lora_mapping
508
509 @torch.inference_mode()
510 def execute_model(
@@ -442,8 +512,12 @@ class ModelRunner:
512 seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
513 kv_caches: List[Tuple[torch.Tensor, torch.Tensor]],
514 ) -> Optional[SamplerOutput]:
445 - input_tokens, input_positions, input_metadata, sampling_metadata = (
515 + input_tokens, input_positions, input_metadata, sampling_metadata, lora_requests, lora_mapping = (
516 self.prepare_input_tensors(seq_group_metadata_list))
517 +
518 + if self.lora_config:
519 + self.set_active_loras(lora_requests, lora_mapping)
520 +
521 # Execute the model.
522 if input_metadata.use_cuda_graph:
523 graph_batch_size = input_tokens.shape[0]
@@ -472,6 +546,28 @@ class ModelRunner:
546 max_num_batched_tokens = self.scheduler_config.max_num_batched_tokens
547 max_num_seqs = self.scheduler_config.max_num_seqs
548
549 + # This represents the maximum number of different requests
550 + # that will have unique loras, an therefore the max amount of memory
551 + # consumption create dummy lora request copies from the lora request
552 + # passed in, which contains a lora from the lora warmup path.
553 + dummy_lora_requests = []
554 + dummy_lora_requests_per_seq = []
555 + if self.lora_config:
556 + for idx in range(self.lora_config.max_loras):
557 + lora_id = idx + 1
558 + dummy_lora_request = LoRARequest(
559 + lora_name=f"warmup_{lora_id}",
560 + lora_int_id=lora_id,
561 + lora_local_path="/not/a/real/path",
562 + )
563 + self.lora_manager.add_dummy_lora(dummy_lora_request,
564 + rank=LORA_WARMUP_RANK)
565 + dummy_lora_requests.append(dummy_lora_request)
566 + dummy_lora_requests_per_seq = [
567 + dummy_lora_requests[idx % len(dummy_lora_requests)]
568 + for idx in range(max_num_seqs)
569 + ]
570 +
571 # Profile memory usage with max_num_sequences sequences and the total
572 # number of tokens equal to max_num_batched_tokens.
573 seqs: List[SequenceGroupMetadata] = []
@@ -485,6 +581,8 @@ class ModelRunner:
581 seq_data={group_id: seq_data},
582 sampling_params=sampling_params,
583 block_tables=None,
584 + lora_request=dummy_lora_requests_per_seq[group_id]
585 + if dummy_lora_requests_per_seq else None,
586 )
587 seqs.append(seq)
588
@@ -495,6 +593,32 @@ class ModelRunner:
593 torch.cuda.synchronize()
594 return
595
596 + def remove_all_loras(self) -> bool:
597 + if not self.lora_manager:
598 + raise RuntimeError("LoRA is not enabled.")
599 + return self.lora_manager.remove_all_loras()
600 +
601 + def set_active_loras(self, lora_requests: List[LoRARequest],
602 + lora_mapping: LoRAMapping) -> None:
603 + if not self.lora_manager:
604 + raise RuntimeError("LoRA is not enabled.")
605 + self.lora_manager.set_active_loras(lora_requests, lora_mapping)
606 +
607 + def add_lora(self, lora_request: LoRARequest) -> bool:
608 + if not self.lora_manager:
609 + raise RuntimeError("LoRA is not enabled.")
610 + return self.lora_manager.add_lora(lora_request)
611 +
612 + def remove_lora(self, lora_id: int) -> bool:
613 + if not self.lora_manager:
614 + raise RuntimeError("LoRA is not enabled.")
615 + return self.lora_manager.remove_lora(lora_id)
616 +
617 + def list_loras(self) -> Set[int]:
618 + if not self.lora_manager:
619 + raise RuntimeError("LoRA is not enabled.")
620 + return self.lora_manager.list_loras()
621 +
622 @torch.inference_mode()
623 def capture_model(self, kv_caches: List[KVCache]) -> None:
624 assert not self.model_config.enforce_eager
@@ -541,6 +665,13 @@ class ModelRunner:
665 use_cuda_graph=True,
666 )
667
668 + if self.lora_config:
669 + lora_mapping = LoRAMapping(
670 + [0] * batch_size,
671 + [0] * batch_size,
672 + )
673 + self.set_active_loras(set(), lora_mapping)
674 +
675 graph_runner = CUDAGraphRunner(self.model)
676 graph_runner.capture(
677 input_tokens[:batch_size],
vllm/worker/worker.py
+23 -4
@@ -1,12 +1,13 @@
1 """A GPU worker class."""
2 +import gc
3 import os
3 -from typing import Dict, List, Optional, Tuple
4 +from typing import Dict, List, Tuple, Set, Optional
5
6 import torch
7 import torch.distributed
8
9 from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
9 - SchedulerConfig)
10 + SchedulerConfig, LoRAConfig)
11 from vllm.model_executor import set_random_seed
12 from vllm.model_executor.parallel_utils.communication_op import (
13 broadcast_tensor_dict)
@@ -15,6 +16,7 @@ from vllm.model_executor.parallel_utils.parallel_state import (
16 from vllm.sequence import SamplerOutput, SequenceGroupMetadata
17 from vllm.worker.cache_engine import CacheEngine
18 from vllm.worker.model_runner import ModelRunner
19 +from vllm.lora.request import LoRARequest
20
21
22 class Worker:
@@ -33,6 +35,7 @@ class Worker:
35 local_rank: int,
36 rank: int,
37 distributed_init_method: str,
38 + lora_config: Optional[LoRAConfig] = None,
39 is_driver_worker: bool = False,
40 ) -> None:
41 self.model_config = model_config
@@ -41,12 +44,16 @@ class Worker:
44 self.local_rank = local_rank
45 self.rank = rank
46 self.distributed_init_method = distributed_init_method
47 + self.lora_config = lora_config
48 self.is_driver_worker = is_driver_worker
49 if self.is_driver_worker:
50 assert self.rank == 0, "The driver worker must have rank 0."
51
48 - self.model_runner = ModelRunner(model_config, parallel_config,
49 - scheduler_config, is_driver_worker)
52 + self.model_runner = ModelRunner(model_config,
53 + parallel_config,
54 + scheduler_config,
55 + lora_config=self.lora_config,
56 + is_driver_worker=is_driver_worker)
57 # Uninitialized cache engine. Will be initialized by
58 # self.init_cache_engine().
59 self.cache_config = None
@@ -117,6 +124,9 @@ class Worker:
124 num_cpu_blocks = int(cpu_swap_space // cache_block_size)
125 num_gpu_blocks = max(num_gpu_blocks, 0)
126 num_cpu_blocks = max(num_cpu_blocks, 0)
127 + if self.model_runner.lora_manager:
128 + self.model_runner.remove_all_loras()
129 + gc.collect()
130 torch.cuda.empty_cache()
131 return num_gpu_blocks, num_cpu_blocks
132
@@ -199,6 +209,15 @@ class Worker:
209 self.gpu_cache)
210 return output
211
212 + def add_lora(self, lora_request: LoRARequest) -> bool:
213 + return self.model_runner.add_lora(lora_request)
214 +
215 + def remove_lora(self, lora_id: int) -> bool:
216 + return self.model_runner.remove_lora(lora_id)
217 +
218 + def list_loras(self) -> Set[int]:
219 + return self.model_runner.list_loras()
220 +
221
222 def _init_distributed_environment(
223 parallel_config: ParallelConfig,