@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7ecee34321
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Kernel][RFC] Refactor the punica kernel based on Triton (#5036)

Jee Jee Li committed Aug 1, 2024 at 08:12 UTC 7ecee3432110bae563c8756a66b54e5f08dc777d
47 files changed +3175 -4364
.github/workflows/scripts/build.sh
-2
@@ -13,8 +13,6 @@ $python_executable -m pip install -r requirements-cuda.txt
13
14 # Limit the number of parallel jobs to avoid OOM
15 export MAX_JOBS=1
16 -# Make sure punica is built for the release (for LoRA)
17 -export VLLM_INSTALL_PUNICA_KERNELS=1
16 # Make sure release wheels are built for the following architectures
17 export TORCH_CUDA_ARCH_LIST="7.0 7.5 8.0 8.6 8.9 9.0+PTX"
18 # Build
CMakeLists.txt
-62
@@ -223,61 +223,7 @@ define_gpu_extension_target(
223 USE_SABI 3
224 WITH_SOABI)
225
226 -#
227 -# _punica_C extension
228 -#
229 -
230 -set(VLLM_PUNICA_EXT_SRC
231 - "csrc/punica/bgmv/bgmv_bf16_bf16_bf16.cu"
232 - "csrc/punica/bgmv/bgmv_bf16_fp32_bf16.cu"
233 - "csrc/punica/bgmv/bgmv_fp16_fp16_fp16.cu"
234 - "csrc/punica/bgmv/bgmv_fp16_fp32_fp16.cu"
235 - "csrc/punica/bgmv/bgmv_fp32_bf16_bf16.cu"
236 - "csrc/punica/bgmv/bgmv_fp32_fp16_fp16.cu"
237 - "csrc/punica/punica_ops.cu"
238 - "csrc/punica/torch_bindings.cpp")
239 -
240 -#
241 -# Copy GPU compilation flags+update for punica
242 -#
243 -set(VLLM_PUNICA_GPU_FLAGS ${VLLM_GPU_FLAGS})
244 -list(REMOVE_ITEM VLLM_PUNICA_GPU_FLAGS
245 - "-D__CUDA_NO_HALF_OPERATORS__"
246 - "-D__CUDA_NO_HALF_CONVERSIONS__"
247 - "-D__CUDA_NO_BFLOAT16_CONVERSIONS__"
248 - "-D__CUDA_NO_HALF2_OPERATORS__")
249 -
250 -#
251 -# Filter out CUDA architectures < 8.0 for punica.
252 -#
253 -if (${VLLM_GPU_LANG} STREQUAL "CUDA")
254 - set(VLLM_PUNICA_GPU_ARCHES)
255 - foreach(ARCH ${VLLM_GPU_ARCHES})
256 - string_to_ver(CODE_VER ${ARCH})
257 - if (CODE_VER GREATER_EQUAL 8.0)
258 - list(APPEND VLLM_PUNICA_GPU_ARCHES ${ARCH})
259 - endif()
260 - endforeach()
261 - message(STATUS "Punica target arches: ${VLLM_PUNICA_GPU_ARCHES}")
262 -elseif(${VLLM_GPU_LANG} STREQUAL "HIP")
263 - set(VLLM_PUNICA_GPU_ARCHES ${VLLM_GPU_ARCHES})
264 - message(STATUS "Punica target arches: ${VLLM_PUNICA_GPU_ARCHES}")
265 -endif()
226
267 -if (VLLM_PUNICA_GPU_ARCHES)
268 - define_gpu_extension_target(
269 - _punica_C
270 - DESTINATION vllm
271 - LANGUAGE ${VLLM_GPU_LANG}
272 - SOURCES ${VLLM_PUNICA_EXT_SRC}
273 - COMPILE_FLAGS ${VLLM_PUNICA_GPU_FLAGS}
274 - ARCHITECTURES ${VLLM_PUNICA_GPU_ARCHES}
275 - USE_SABI 3
276 - WITH_SOABI)
277 -else()
278 - message(WARNING "Unable to create _punica_C target because none of the "
279 - "requested architectures (${VLLM_GPU_ARCHES}) are supported, i.e. >= 8.0")
280 -endif()
227
228 #
229 # Add the `default` target which detects which extensions should be
@@ -301,12 +247,4 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
247 message(STATUS "Enabling moe extension.")
248 add_dependencies(default _moe_C)
249
304 - # Enable punica if -DVLLM_INSTALL_PUNICA_KERNELS=ON or
305 - # VLLM_INSTALL_PUNICA_KERNELS is set in the environment and
306 - # there are supported target arches.
307 - if (VLLM_PUNICA_GPU_ARCHES AND
308 - (ENV{VLLM_INSTALL_PUNICA_KERNELS} OR VLLM_INSTALL_PUNICA_KERNELS))
309 - message(STATUS "Enabling punica extension.")
310 - add_dependencies(default _punica_C)
311 - endif()
250 endif()
Dockerfile
-2
@@ -88,8 +88,6 @@ ENV MAX_JOBS=${max_jobs}
88 # number of threads used by nvcc
89 ARG nvcc_threads=8
90 ENV NVCC_THREADS=$nvcc_threads
91 -# make sure punica kernels are built (for LoRA)
92 -ENV VLLM_INSTALL_PUNICA_KERNELS=1
91
92 ARG buildkite_commit
93 ENV BUILDKITE_COMMIT=${buildkite_commit}
Dockerfile.rocm
+1 -2
@@ -131,8 +131,7 @@ COPY . .
131 RUN --mount=type=cache,target=/root/.cache/pip \
132 python3 -m pip install --upgrade numba scipy huggingface-hub[cli]
133
134 -# Make sure punica kernels are built (for LoRA)
135 -ENV VLLM_INSTALL_PUNICA_KERNELS=1
134 +
135 # Workaround for ray >= 2.10.0
136 ENV RAY_EXPERIMENTAL_NOSET_ROCR_VISIBLE_DEVICES=1
137 # Silences the HF Tokenizers warning
csrc/punica/LICENSE deleted
-217
@@ -1,217 +0,0 @@
1 -Contains code from https://github.com/punica-ai/punica
2 -
3 - Apache License
4 - Version 2.0, January 2004
5 - http://www.apache.org/licenses/
6 -
7 - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
8 -
9 - 1. Definitions.
10 -
11 - "License" shall mean the terms and conditions for use, reproduction,
12 - and distribution as defined by Sections 1 through 9 of this document.
13 -
14 - "Licensor" shall mean the copyright owner or entity authorized by
15 - the copyright owner that is granting the License.
16 -
17 - "Legal Entity" shall mean the union of the acting entity and all
18 - other entities that control, are controlled by, or are under common
19 - control with that entity. For the purposes of this definition,
20 - "control" means (i) the power, direct or indirect, to cause the
21 - direction or management of such entity, whether by contract or
22 - otherwise, or (ii) ownership of fifty percent (50%) or more of the
23 - outstanding shares, or (iii) beneficial ownership of such entity.
24 -
25 - "You" (or "Your") shall mean an individual or Legal Entity
26 - exercising permissions granted by this License.
27 -
28 - "Source" form shall mean the preferred form for making modifications,
29 - including but not limited to software source code, documentation
30 - source, and configuration files.
31 -
32 - "Object" form shall mean any form resulting from mechanical
33 - transformation or translation of a Source form, including but
34 - not limited to compiled object code, generated documentation,
35 - and conversions to other media types.
36 -
37 - "Work" shall mean the work of authorship, whether in Source or
38 - Object form, made available under the License, as indicated by a
39 - copyright notice that is included in or attached to the work
40 - (an example is provided in the Appendix below).
41 -
42 - "Derivative Works" shall mean any work, whether in Source or Object
43 - form, that is based on (or derived from) the Work and for which the
44 - editorial revisions, annotations, elaborations, or other modifications
45 - represent, as a whole, an original work of authorship. For the purposes
46 - of this License, Derivative Works shall not include works that remain
47 - separable from, or merely link (or bind by name) to the interfaces of,
48 - the Work and Derivative Works thereof.
49 -
50 - "Contribution" shall mean any work of authorship, including
51 - the original version of the Work and any modifications or additions
52 - to that Work or Derivative Works thereof, that is intentionally
53 - submitted to Licensor for inclusion in the Work by the copyright owner
54 - or by an individual or Legal Entity authorized to submit on behalf of
55 - the copyright owner. For the purposes of this definition, "submitted"
56 - means any form of electronic, verbal, or written communication sent
57 - to the Licensor or its representatives, including but not limited to
58 - communication on electronic mailing lists, source code control systems,
59 - and issue tracking systems that are managed by, or on behalf of, the
60 - Licensor for the purpose of discussing and improving the Work, but
61 - excluding communication that is conspicuously marked or otherwise
62 - designated in writing by the copyright owner as "Not a Contribution."
63 -
64 - "Contributor" shall mean Licensor and any individual or Legal Entity
65 - on behalf of whom a Contribution has been received by Licensor and
66 - subsequently incorporated within the Work.
67 -
68 - 2. Grant of Copyright License. Subject to the terms and conditions of
69 - this License, each Contributor hereby grants to You a perpetual,
70 - worldwide, non-exclusive, no-charge, royalty-free, irrevocable
71 - copyright license to reproduce, prepare Derivative Works of,
72 - publicly display, publicly perform, sublicense, and distribute the
73 - Work and such Derivative Works in Source or Object form.
74 -
75 - 3. Grant of Patent License. Subject to the terms and conditions of
76 - this License, each Contributor hereby grants to You a perpetual,
77 - worldwide, non-exclusive, no-charge, royalty-free, irrevocable
78 - (except as stated in this section) patent license to make, have made,
79 - use, offer to sell, sell, import, and otherwise transfer the Work,
80 - where such license applies only to those patent claims licensable
81 - by such Contributor that are necessarily infringed by their
82 - Contribution(s) alone or by combination of their Contribution(s)
83 - with the Work to which such Contribution(s) was submitted. If You
84 - institute patent litigation against any entity (including a
85 - cross-claim or counterclaim in a lawsuit) alleging that the Work
86 - or a Contribution incorporated within the Work constitutes direct
87 - or contributory patent infringement, then any patent licenses
88 - granted to You under this License for that Work shall terminate
89 - as of the date such litigation is filed.
90 -
91 - 4. Redistribution. You may reproduce and distribute copies of the
92 - Work or Derivative Works thereof in any medium, with or without
93 - modifications, and in Source or Object form, provided that You
94 - meet the following conditions:
95 -
96 - (a) You must give any other recipients of the Work or
97 - Derivative Works a copy of this License; and
98 -
99 - (b) You must cause any modified files to carry prominent notices
100 - stating that You changed the files; and
101 -
102 - (c) You must retain, in the Source form of any Derivative Works
103 - that You distribute, all copyright, patent, trademark, and
104 - attribution notices from the Source form of the Work,
105 - excluding those notices that do not pertain to any part of
106 - the Derivative Works; and
107 -
108 - (d) If the Work includes a "NOTICE" text file as part of its
109 - distribution, then any Derivative Works that You distribute must
110 - include a readable copy of the attribution notices contained
111 - within such NOTICE file, excluding those notices that do not
112 - pertain to any part of the Derivative Works, in at least one
113 - of the following places: within a NOTICE text file distributed
114 - as part of the Derivative Works; within the Source form or
115 - documentation, if provided along with the Derivative Works; or,
116 - within a display generated by the Derivative Works, if and
117 - wherever such third-party notices normally appear. The contents
118 - of the NOTICE file are for informational purposes only and
119 - do not modify the License. You may add Your own attribution
120 - notices within Derivative Works that You distribute, alongside
121 - or as an addendum to the NOTICE text from the Work, provided
122 - that such additional attribution notices cannot be construed
123 - as modifying the License.
124 -
125 - You may add Your own copyright statement to Your modifications and
126 - may provide additional or different license terms and conditions
127 - for use, reproduction, or distribution of Your modifications, or
128 - for any such Derivative Works as a whole, provided Your use,
129 - reproduction, and distribution of the Work otherwise complies with
130 - the conditions stated in this License.
131 -
132 - 5. Submission of Contributions. Unless You explicitly state otherwise,
133 - any Contribution intentionally submitted for inclusion in the Work
134 - by You to the Licensor shall be under the terms and conditions of
135 - this License, without any additional terms or conditions.
136 - Notwithstanding the above, nothing herein shall supersede or modify
137 - the terms of any separate license agreement you may have executed
138 - with Licensor regarding such Contributions.
139 -
140 - 6. Trademarks. This License does not grant permission to use the trade
141 - names, trademarks, service marks, or product names of the Licensor,
142 - except as required for reasonable and customary use in describing the
143 - origin of the Work and reproducing the content of the NOTICE file.
144 -
145 - 7. Disclaimer of Warranty. Unless required by applicable law or
146 - agreed to in writing, Licensor provides the Work (and each
147 - Contributor provides its Contributions) on an "AS IS" BASIS,
148 - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
149 - implied, including, without limitation, any warranties or conditions
150 - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
151 - PARTICULAR PURPOSE. You are solely responsible for determining the
152 - appropriateness of using or redistributing the Work and assume any
153 - risks associated with Your exercise of permissions under this License.
154 -
155 - 8. Limitation of Liability. In no event and under no legal theory,
156 - whether in tort (including negligence), contract, or otherwise,
157 - unless required by applicable law (such as deliberate and grossly
158 - negligent acts) or agreed to in writing, shall any Contributor be
159 - liable to You for damages, including any direct, indirect, special,
160 - incidental, or consequential damages of any character arising as a
161 - result of this License or out of the use or inability to use the
162 - Work (including but not limited to damages for loss of goodwill,
163 - work stoppage, computer failure or malfunction, or any and all
164 - other commercial damages or losses), even if such Contributor
165 - has been advised of the possibility of such damages.
166 -
167 - 9. Accepting Warranty or Additional Liability. While redistributing
168 - the Work or Derivative Works thereof, You may choose to offer,
169 - and charge a fee for, acceptance of support, warranty, indemnity,
170 - or other liability obligations and/or rights consistent with this
171 - License. However, in accepting such obligations, You may act only
172 - on Your own behalf and on Your sole responsibility, not on behalf
173 - of any other Contributor, and only if You agree to indemnify,
174 - defend, and hold each Contributor harmless for any liability
175 - incurred by, or claims asserted against, such Contributor by reason
176 - of your accepting any such warranty or additional liability.
177 -
178 - END OF TERMS AND CONDITIONS
179 -
180 - APPENDIX: How to apply the Apache License to your work.
181 -
182 - To apply the Apache License to your work, attach the following
183 - boilerplate notice, with the fields enclosed by brackets "{}"
184 - replaced with your own identifying information. (Don't include
185 - the brackets!) The text should be enclosed in the appropriate
186 - comment syntax for the file format. We also recommend that a
187 - file or class name and description of purpose be included on the
188 - same "printed page" as the copyright notice for easier
189 - identification within third-party archives.
190 -
191 - Copyright {yyyy} {name of copyright owner}
192 -
193 - Licensed under the Apache License, Version 2.0 (the "License");
194 - you may not use this file except in compliance with the License.
195 - You may obtain a copy of the License at
196 -
197 - http://www.apache.org/licenses/LICENSE-2.0
198 -
199 - Unless required by applicable law or agreed to in writing, software
200 - distributed under the License is distributed on an "AS IS" BASIS,
201 - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
202 - See the License for the specific language governing permissions and
203 - limitations under the License.
204 -
205 -------------------------------------------------------------------------------------
206 -
207 -This product bundles various third-party components under other open source licenses.
208 -This section summarizes those components and their licenses. See licenses/
209 -for text of these licenses.
210 -
211 -
212 -Apache-2.0
213 -* third_party/nvbench (with LLVM exception)
214 -* third_party/flashinfer
215 -
216 -BSD-3-Clause:
217 -* third_party/cutlass
\ No newline at end of file
csrc/punica/bgmv/bgmv_bf16_bf16_bf16.cu deleted
-5
@@ -1,5 +0,0 @@
1 -#include "bgmv_config.h"
2 -#include "bgmv_impl.cuh"
3 -
4 -FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_bfloat16, nv_bfloat16, nv_bfloat16)
5 -FOR_INST_BGMV_WIDE_NARROW(INST_BGMV_ONESIDE, nv_bfloat16, nv_bfloat16, nv_bfloat16)
csrc/punica/bgmv/bgmv_bf16_fp32_bf16.cu deleted
-5
@@ -1,5 +0,0 @@
1 -#include "bgmv_config.h"
2 -#include "bgmv_impl.cuh"
3 -
4 -FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_bfloat16, float, nv_bfloat16)
5 -FOR_INST_BGMV_WIDE_NARROW(INST_BGMV_ONESIDE, nv_bfloat16, float, nv_bfloat16)
csrc/punica/bgmv/bgmv_config.h deleted
-218
@@ -1,218 +0,0 @@
1 -#pragma once
2 -
3 -template <int feat_in, int feat_out, typename in_T, typename out_T,
4 - typename W_T>
5 -void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
6 - const W_T *__restrict__ W,
7 - const int64_t *__restrict__ indicies, int64_t y_offset,
8 - int64_t full_y_size, int64_t batch_size, int64_t num_layers,
9 - int64_t layer_idx, float scale);
10 -
11 -// clang-format off
12 -
13 -#define FOR_BGMV_WIDE(f, in_T, out_T, W_T, narrow) \
14 - f(in_T, out_T, W_T, narrow, 128) \
15 - f(in_T, out_T, W_T, narrow, 256) \
16 - f(in_T, out_T, W_T, narrow, 512) \
17 - f(in_T, out_T, W_T, narrow, 640) \
18 - f(in_T, out_T, W_T, narrow, 768) \
19 - f(in_T, out_T, W_T, narrow, 896) \
20 - f(in_T, out_T, W_T, narrow, 1024) \
21 - f(in_T, out_T, W_T, narrow, 1152) \
22 - f(in_T, out_T, W_T, narrow, 1216) \
23 - f(in_T, out_T, W_T, narrow, 1280) \
24 - f(in_T, out_T, W_T, narrow, 1536) \
25 - f(in_T, out_T, W_T, narrow, 1664) \
26 - f(in_T, out_T, W_T, narrow, 1728) \
27 - f(in_T, out_T, W_T, narrow, 1792) \
28 - f(in_T, out_T, W_T, narrow, 2048) \
29 - f(in_T, out_T, W_T, narrow, 2240) \
30 - f(in_T, out_T, W_T, narrow, 2304) \
31 - f(in_T, out_T, W_T, narrow, 2368) \
32 - f(in_T, out_T, W_T, narrow, 2432) \
33 - f(in_T, out_T, W_T, narrow, 2560) \
34 - f(in_T, out_T, W_T, narrow, 2752) \
35 - f(in_T, out_T, W_T, narrow, 2816) \
36 - f(in_T, out_T, W_T, narrow, 3072) \
37 - f(in_T, out_T, W_T, narrow, 3328) \
38 - f(in_T, out_T, W_T, narrow, 3456) \
39 - f(in_T, out_T, W_T, narrow, 3584) \
40 - f(in_T, out_T, W_T, narrow, 3712) \
41 - f(in_T, out_T, W_T, narrow, 4096) \
42 - f(in_T, out_T, W_T, narrow, 4480) \
43 - f(in_T, out_T, W_T, narrow, 4608) \
44 - f(in_T, out_T, W_T, narrow, 4736) \
45 - f(in_T, out_T, W_T, narrow, 4864) \
46 - f(in_T, out_T, W_T, narrow, 5120) \
47 - f(in_T, out_T, W_T, narrow, 5504) \
48 - f(in_T, out_T, W_T, narrow, 5632) \
49 - f(in_T, out_T, W_T, narrow, 5888) \
50 - f(in_T, out_T, W_T, narrow, 6144) \
51 - f(in_T, out_T, W_T, narrow, 6400) \
52 - f(in_T, out_T, W_T, narrow, 6848) \
53 - f(in_T, out_T, W_T, narrow, 6912) \
54 - f(in_T, out_T, W_T, narrow, 7168) \
55 - f(in_T, out_T, W_T, narrow, 7424) \
56 - f(in_T, out_T, W_T, narrow, 8192) \
57 - f(in_T, out_T, W_T, narrow, 8960) \
58 - f(in_T, out_T, W_T, narrow, 9216) \
59 - f(in_T, out_T, W_T, narrow, 9472) \
60 - f(in_T, out_T, W_T, narrow, 10240) \
61 - f(in_T, out_T, W_T, narrow, 11008) \
62 - f(in_T, out_T, W_T, narrow, 11264) \
63 - f(in_T, out_T, W_T, narrow, 12288) \
64 - f(in_T, out_T, W_T, narrow, 13696) \
65 - f(in_T, out_T, W_T, narrow, 13824) \
66 - f(in_T, out_T, W_T, narrow, 14336) \
67 - f(in_T, out_T, W_T, narrow, 14784) \
68 - f(in_T, out_T, W_T, narrow, 14848) \
69 - f(in_T, out_T, W_T, narrow, 15360) \
70 - f(in_T, out_T, W_T, narrow, 16384) \
71 - f(in_T, out_T, W_T, narrow, 18944) \
72 - f(in_T, out_T, W_T, narrow, 20480) \
73 - f(in_T, out_T, W_T, narrow, 22016) \
74 - f(in_T, out_T, W_T, narrow, 22528) \
75 - f(in_T, out_T, W_T, narrow, 24576) \
76 - f(in_T, out_T, W_T, narrow, 27392) \
77 - f(in_T, out_T, W_T, narrow, 27648) \
78 - f(in_T, out_T, W_T, narrow, 28672) \
79 - f(in_T, out_T, W_T, narrow, 29568) \
80 - f(in_T, out_T, W_T, narrow, 29696) \
81 - f(in_T, out_T, W_T, narrow, 32000) \
82 - f(in_T, out_T, W_T, narrow, 32256) \
83 - f(in_T, out_T, W_T, narrow, 32512) \
84 - f(in_T, out_T, W_T, narrow, 32768) \
85 - f(in_T, out_T, W_T, narrow, 33024) \
86 - f(in_T, out_T, W_T, narrow, 36864) \
87 - f(in_T, out_T, W_T, narrow, 43264) \
88 - f(in_T, out_T, W_T, narrow, 49152) \
89 - f(in_T, out_T, W_T, narrow, 49408) \
90 - f(in_T, out_T, W_T, narrow, 60544) \
91 - f(in_T, out_T, W_T, narrow, 60672) \
92 - f(in_T, out_T, W_T, narrow, 64000) \
93 - f(in_T, out_T, W_T, narrow, 64256) \
94 - f(in_T, out_T, W_T, narrow, 64512) \
95 - f(in_T, out_T, W_T, narrow, 102400) \
96 - f(in_T, out_T, W_T, narrow, 102656) \
97 - f(in_T, out_T, W_T, narrow, 102912) \
98 - f(in_T, out_T, W_T, narrow, 128000) \
99 - f(in_T, out_T, W_T, narrow, 128256) \
100 - f(in_T, out_T, W_T, narrow, 128512) \
101 -
102 -
103 -// Keep above in sync with vllm/lora/layers::LogitsProcessorWithLoRA
104 -// and vllm/tests/lora/test_punica.py
105 -
106 -// Used for defining kernels going from the variety of
107 -// dim in to the narrow dim out
108 - // Using it for the fully sharded column
109 - // parallel LoRA A which splits the rank dim
110 -#define FOR_INST_BGMV_NARROW(f, in_T, out_T, W_T, narrow) \
111 - f(in_T, out_T, W_T, 128, narrow) \
112 - f(in_T, out_T, W_T, 256, narrow) \
113 - f(in_T, out_T, W_T, 512, narrow) \
114 - f(in_T, out_T, W_T, 640, narrow) \
115 - f(in_T, out_T, W_T, 768, narrow) \
116 - f(in_T, out_T, W_T, 896, narrow) \
117 - f(in_T, out_T, W_T, 1024, narrow) \
118 - f(in_T, out_T, W_T, 1152, narrow) \
119 - f(in_T, out_T, W_T, 1216, narrow) \
120 - f(in_T, out_T, W_T, 1280, narrow) \
121 - f(in_T, out_T, W_T, 1536, narrow) \
122 - f(in_T, out_T, W_T, 1664, narrow) \
123 - f(in_T, out_T, W_T, 1728, narrow) \
124 - f(in_T, out_T, W_T, 1792, narrow) \
125 - f(in_T, out_T, W_T, 2048, narrow) \
126 - f(in_T, out_T, W_T, 2240, narrow) \
127 - f(in_T, out_T, W_T, 2304, narrow) \
128 - f(in_T, out_T, W_T, 2368, narrow) \
129 - f(in_T, out_T, W_T, 2432, narrow) \
130 - f(in_T, out_T, W_T, 2560, narrow) \
131 - f(in_T, out_T, W_T, 2752, narrow) \
132 - f(in_T, out_T, W_T, 2816, narrow) \
133 - f(in_T, out_T, W_T, 3072, narrow) \
134 - f(in_T, out_T, W_T, 3328, narrow) \
135 - f(in_T, out_T, W_T, 3456, narrow) \
136 - f(in_T, out_T, W_T, 3584, narrow) \
137 - f(in_T, out_T, W_T, 3712, narrow) \
138 - f(in_T, out_T, W_T, 4096, narrow) \
139 - f(in_T, out_T, W_T, 4480, narrow) \
140 - f(in_T, out_T, W_T, 4608, narrow) \
141 - f(in_T, out_T, W_T, 4736, narrow) \
142 - f(in_T, out_T, W_T, 4864, narrow) \
143 - f(in_T, out_T, W_T, 5120, narrow) \
144 - f(in_T, out_T, W_T, 5504, narrow) \
145 - f(in_T, out_T, W_T, 5632, narrow) \
146 - f(in_T, out_T, W_T, 5888, narrow) \
147 - f(in_T, out_T, W_T, 6144, narrow) \
148 - f(in_T, out_T, W_T, 6400, narrow) \
149 - f(in_T, out_T, W_T, 6848, narrow) \
150 - f(in_T, out_T, W_T, 6912, narrow) \
151 - f(in_T, out_T, W_T, 7168, narrow) \
152 - f(in_T, out_T, W_T, 7424, narrow) \
153 - f(in_T, out_T, W_T, 8192, narrow) \
154 - f(in_T, out_T, W_T, 8960, narrow) \
155 - f(in_T, out_T, W_T, 9216, narrow) \
156 - f(in_T, out_T, W_T, 9472, narrow) \
157 - f(in_T, out_T, W_T, 10240, narrow) \
158 - f(in_T, out_T, W_T, 11008, narrow) \
159 - f(in_T, out_T, W_T, 11264, narrow) \
160 - f(in_T, out_T, W_T, 12288, narrow) \
161 - f(in_T, out_T, W_T, 13696, narrow) \
162 - f(in_T, out_T, W_T, 13824, narrow) \
163 - f(in_T, out_T, W_T, 14336, narrow) \
164 - f(in_T, out_T, W_T, 14784, narrow) \
165 - f(in_T, out_T, W_T, 14848, narrow) \
166 - f(in_T, out_T, W_T, 15360, narrow) \
167 - f(in_T, out_T, W_T, 16384, narrow) \
168 - f(in_T, out_T, W_T, 18944, narrow) \
169 - f(in_T, out_T, W_T, 20480, narrow) \
170 - f(in_T, out_T, W_T, 22016, narrow) \
171 - f(in_T, out_T, W_T, 22528, narrow) \
172 - f(in_T, out_T, W_T, 24576, narrow) \
173 - f(in_T, out_T, W_T, 27392, narrow) \
174 - f(in_T, out_T, W_T, 27648, narrow) \
175 - f(in_T, out_T, W_T, 28672, narrow) \
176 - f(in_T, out_T, W_T, 29568, narrow) \
177 - f(in_T, out_T, W_T, 29696, narrow) \
178 - f(in_T, out_T, W_T, 32000, narrow) \
179 - f(in_T, out_T, W_T, 32256, narrow) \
180 - f(in_T, out_T, W_T, 32512, narrow) \
181 - f(in_T, out_T, W_T, 32768, narrow) \
182 - f(in_T, out_T, W_T, 33024, narrow) \
183 - f(in_T, out_T, W_T, 36864, narrow) \
184 - f(in_T, out_T, W_T, 43264, narrow) \
185 - f(in_T, out_T, W_T, 49152, narrow) \
186 - f(in_T, out_T, W_T, 49408, narrow) \
187 - f(in_T, out_T, W_T, 60544, narrow) \
188 - f(in_T, out_T, W_T, 60672, narrow) \
189 - f(in_T, out_T, W_T, 64000, narrow) \
190 - f(in_T, out_T, W_T, 64256, narrow) \
191 - f(in_T, out_T, W_T, 64512, narrow) \
192 - f(in_T, out_T, W_T, 102400, narrow) \
193 - f(in_T, out_T, W_T, 102656, narrow) \
194 - f(in_T, out_T, W_T, 102912, narrow) \
195 - f(in_T, out_T, W_T, 128000, narrow) \
196 - f(in_T, out_T, W_T, 128256, narrow) \
197 - f(in_T, out_T, W_T, 128512, narrow) \
198 -// Keep above in sync with vllm/lora/layers::SamplerWithLoRA
199 -
200 -
201 -// Keep this in sync with vllm/config::LoRAConfig
202 -#define FOR_BGMV_WIDE_NARROW(f, in_T, out_T, W_T) \
203 - FOR_BGMV_WIDE(f, in_T, out_T, W_T, 8) \
204 - FOR_BGMV_WIDE(f, in_T, out_T, W_T, 16) \
205 - FOR_BGMV_WIDE(f, in_T, out_T, W_T, 32) \
206 - FOR_BGMV_WIDE(f, in_T, out_T, W_T, 64)
207 -
208 -
209 -#define FOR_INST_BGMV_WIDE_NARROW(f, in_T, out_T, W_T) \
210 - FOR_INST_BGMV_NARROW(f, in_T, out_T, W_T, 1) \
211 - FOR_INST_BGMV_NARROW(f, in_T, out_T, W_T, 2) \
212 - FOR_INST_BGMV_NARROW(f, in_T, out_T, W_T, 4) \
213 - f(in_T, out_T, W_T, 8, 64) \
214 - f(in_T, out_T, W_T, 16, 64) \
215 - f(in_T, out_T, W_T, 32, 64) \
216 - f(in_T, out_T, W_T, 64, 64)
217 -
218 -// clang-format on
csrc/punica/bgmv/bgmv_fp16_fp16_fp16.cu deleted
-5
@@ -1,5 +0,0 @@
1 -#include "bgmv_config.h"
2 -#include "bgmv_impl.cuh"
3 -
4 -FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_half, nv_half, nv_half)
5 -FOR_INST_BGMV_WIDE_NARROW(INST_BGMV_ONESIDE, nv_half, nv_half, nv_half)
csrc/punica/bgmv/bgmv_fp16_fp32_fp16.cu deleted
-5
@@ -1,5 +0,0 @@
1 -#include "bgmv_config.h"
2 -#include "bgmv_impl.cuh"
3 -
4 -FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_half, float, nv_half)
5 -FOR_INST_BGMV_WIDE_NARROW(INST_BGMV_ONESIDE, nv_half, float, nv_half)
csrc/punica/bgmv/bgmv_fp32_bf16_bf16.cu deleted
-5
@@ -1,5 +0,0 @@
1 -#include "bgmv_config.h"
2 -#include "bgmv_impl.cuh"
3 -
4 -FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, float, nv_bfloat16, nv_bfloat16)
5 -FOR_INST_BGMV_WIDE_NARROW(INST_BGMV_ONESIDE, float, nv_bfloat16, nv_bfloat16)
csrc/punica/bgmv/bgmv_fp32_fp16_fp16.cu deleted
-5
@@ -1,5 +0,0 @@
1 -#include "bgmv_config.h"
2 -#include "bgmv_impl.cuh"
3 -
4 -FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, float, nv_half, nv_half)
5 -FOR_INST_BGMV_WIDE_NARROW(INST_BGMV_ONESIDE, float, nv_half, nv_half)
csrc/punica/bgmv/bgmv_impl.cuh deleted
-451
@@ -1,451 +0,0 @@
1 -#pragma once
2 -
3 -#include <ATen/cuda/CUDAContext.h>
4 -#ifndef USE_ROCM
5 -#include <cooperative_groups.h>
6 -#else
7 -#include <hip/hip_cooperative_groups.h>
8 -#endif
9 -#ifndef USE_ROCM
10 -#include <cuda/pipeline>
11 -#endif
12 -#include <cuda_runtime.h>
13 -#include <iostream>
14 -#include <stdio.h>
15 -
16 -#include "vec_dtypes.cuh"
17 -
18 -namespace cg = cooperative_groups;
19 -
20 -#ifdef USE_ROCM
21 -template <size_t len>
22 -__host__ __device__
23 -inline void* memcpy_blocking(void *dst, const void *src) {
24 - // Does not handle the case of long datatypes
25 - char *d = reinterpret_cast<char *>(dst);
26 - const char *s = reinterpret_cast<const char *>(src);
27 - size_t i = 0;
28 -#pragma unroll
29 - for (i = 0; i < len; ++i) {
30 - d[i] = s[i];
31 - }
32 - return dst;
33 -}
34 -#endif
35 -
36 -#ifndef USE_ROCM
37 -
38 -// nthrs = (32, 4)
39 -template <int feat_in, int feat_out, size_t vec_size, size_t X_copy_size,
40 - size_t W_copy_size, int tx, int ty, int tz, typename in_T,
41 - typename out_T, typename W_T>
42 -__global__ void
43 -bgmv_shrink_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
44 - const W_T *__restrict__ W,
45 - const int64_t *__restrict__ indicies, int64_t y_offset,
46 - int64_t full_y_size, int64_t num_layers, int64_t layer_idx,
47 - float scale) {
48 - size_t batch_idx = blockIdx.y;
49 - int64_t idx = indicies[batch_idx] * num_layers + layer_idx;
50 - if (idx < 0) {
51 - return;
52 - }
53 -
54 - auto block = cg::this_thread_block();
55 - size_t j = blockIdx.x;
56 - constexpr size_t num_pipeline_stages = 2;
57 - constexpr size_t tile_size = tx * ty * vec_size;
58 - __shared__ W_T W_shared[num_pipeline_stages * tile_size];
59 - __shared__ in_T X_shared[num_pipeline_stages * tile_size];
60 - __shared__ float y_warpwise[ty];
61 -
62 - size_t W_shared_offset[num_pipeline_stages] = {0U, 1U * tile_size};
63 - size_t X_shared_offset[num_pipeline_stages] = {0U, 1U * tile_size};
64 - auto pipe = cuda::make_pipeline();
65 -
66 - // pipeline load W/X and compute WX;
67 - pipe.producer_acquire();
68 - cuda::memcpy_async(W_shared + (threadIdx.y * tx + threadIdx.x) * vec_size,
69 - W + (idx * feat_out + j) * feat_in +
70 - (threadIdx.y * tx + threadIdx.x) * vec_size,
71 - cuda::aligned_size_t<W_copy_size>(W_copy_size), pipe);
72 - cuda::memcpy_async(X_shared + (threadIdx.y * tx + threadIdx.x) * vec_size,
73 - X + (batch_idx * feat_in) +
74 - (threadIdx.y * tx + threadIdx.x) * vec_size,
75 - cuda::aligned_size_t<X_copy_size>(X_copy_size), pipe);
76 - pipe.producer_commit();
77 - size_t copy_idx, compute_idx;
78 - float y = 0.f;
79 - vec_t<in_T, vec_size> x_vec;
80 - vec_t<W_T, vec_size> w_vec;
81 - size_t tile_idx;
82 -
83 -#pragma unroll
84 - for (tile_idx = 1; tile_idx < (feat_in + tile_size - 1) / tile_size;
85 - ++tile_idx) {
86 - copy_idx = tile_idx % num_pipeline_stages;
87 - // pipeline stage: async copy W fragment
88 - pipe.producer_acquire();
89 - if (tile_idx * tile_size + threadIdx.y * tx * vec_size < feat_in) {
90 - cuda::memcpy_async(W_shared + W_shared_offset[copy_idx] +
91 - (threadIdx.y * tx + threadIdx.x) * vec_size,
92 - W + (idx * feat_out + j) * feat_in +
93 - tile_idx * tile_size +
94 - (threadIdx.y * tx + threadIdx.x) * vec_size,
95 - cuda::aligned_size_t<W_copy_size>(W_copy_size), pipe);
96 - cuda::memcpy_async(X_shared + X_shared_offset[copy_idx] +
97 - (threadIdx.y * tx + threadIdx.x) * vec_size,
98 - X + (batch_idx * feat_in) + tile_idx * tile_size +
99 - (threadIdx.y * tx + threadIdx.x) * vec_size,
100 - cuda::aligned_size_t<X_copy_size>(X_copy_size), pipe);
101 - }
102 - pipe.producer_commit();
103 -
104 - compute_idx = (tile_idx - 1) % num_pipeline_stages;
105 - // pipeline stage: compute WX
106 - pipe.consumer_wait();
107 - block.sync();
108 - x_vec.load(X_shared + X_shared_offset[compute_idx] +
109 - (threadIdx.y * tx + threadIdx.x) * vec_size);
110 - w_vec.load(W_shared + W_shared_offset[compute_idx] +
111 - (threadIdx.y * tx + threadIdx.x) * vec_size);
112 - float sum = 0.f;
113 -#pragma unroll
114 - for (size_t i = 0; i < vec_size; ++i) {
115 - sum += float(w_vec[i]) * float(x_vec[i]) * scale;
116 - }
117 -#pragma unroll
118 - for (size_t offset = tx / 2; offset > 0; offset /= 2) {
119 - sum += __shfl_down_sync(0xffffffff, sum, offset);
120 - }
121 - y_warpwise[threadIdx.y] = sum;
122 - block.sync();
123 -#pragma unroll
124 - for (size_t i = 0; i < ty; ++i) {
125 - y += y_warpwise[i];
126 - }
127 -
128 - block.sync();
129 - pipe.consumer_release();
130 - }
131 -
132 - compute_idx = (tile_idx - 1) % num_pipeline_stages;
133 - // final pipeline stage
134 - pipe.consumer_wait();
135 - block.sync();
136 - x_vec.load(X_shared + X_shared_offset[compute_idx] +
137 - (threadIdx.y * tx + threadIdx.x) * vec_size);
138 - w_vec.load(W_shared + W_shared_offset[compute_idx] +
139 - (threadIdx.y * tx + threadIdx.x) * vec_size);
140 - float sum = 0.f;
141 -#pragma unroll
142 - for (size_t i = 0; i < vec_size; ++i) {
143 - sum += float(w_vec[i]) * float(x_vec[i]) * scale;
144 - }
145 -#pragma unroll
146 - for (size_t offset = tx / 2; offset > 0; offset /= 2) {
147 - sum += __shfl_down_sync(0xffffffff, sum, offset);
148 - }
149 - y_warpwise[threadIdx.y] =
150 - ((tile_idx - 1) * tile_size + threadIdx.y * tx * vec_size < feat_in)
151 - ? sum
152 - : 0.f;
153 - block.sync();
154 -#pragma unroll
155 - for (size_t i = 0; i < ty; ++i) {
156 - y += y_warpwise[i];
157 - }
158 -
159 - block.sync();
160 - pipe.consumer_release();
161 -
162 - // write Y;
163 - if (block.thread_rank() == 0) {
164 - Y[batch_idx * full_y_size + y_offset + j] += static_cast<out_T>(y);
165 - }
166 -}
167 -
168 -#else
169 -
170 -template <int feat_in, int feat_out, size_t vec_size, size_t X_copy_size,
171 - size_t W_copy_size, int tx, int ty, int tz, typename in_T,
172 - typename out_T, typename W_T>
173 -__global__ void
174 -bgmv_shrink_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
175 - const W_T *__restrict__ W,
176 - const int64_t *__restrict__ indicies, int64_t y_offset,
177 - int64_t full_y_size, int64_t num_layers, int64_t layer_idx,
178 - float scale) {
179 - size_t batch_idx = blockIdx.y;
180 - int64_t idx = indicies[batch_idx] * num_layers + layer_idx;
181 - if (idx < 0) {
182 - return;
183 - }
184 -
185 - size_t j = blockIdx.x;
186 - constexpr size_t tile_size = tx * ty * vec_size;
187 - constexpr size_t num_tiles = (feat_in + tile_size - 1) / tile_size;
188 - __shared__ float y_warpwise[ty];
189 -
190 - float y = 0;
191 - vec_t<in_T, vec_size> x_vec;
192 - vec_t<W_T, vec_size> w_vec;
193 - size_t tile_idx;
194 -
195 -#pragma unroll
196 - for (tile_idx = 0; tile_idx < num_tiles; ++tile_idx) {
197 - if (tile_idx * tile_size + (threadIdx.y * tx + threadIdx.x + 1) * vec_size - 1 < feat_in) {
198 - x_vec.load(X + (batch_idx * feat_in) +
199 - tile_idx * tile_size +
200 - (threadIdx.y * tx + threadIdx.x) * vec_size);
201 - w_vec.load(W + (idx * feat_out + j) * feat_in +
202 - tile_idx * tile_size +
203 - (threadIdx.y * tx + threadIdx.x) * vec_size);
204 - }
205 -
206 - float sum = 0.f;
207 -#pragma unroll
208 - for (size_t i = 0; i < vec_size; ++i) {
209 - sum += convert_type<W_T, float>(w_vec[i]) * convert_type<in_T, float>(x_vec[i]) * scale;
210 - }
211 -#pragma unroll
212 - for (size_t offset = tx / 2; offset > 0; offset /= 2) {
213 - sum += VLLM_SHFL_DOWN_SYNC(sum, offset);
214 - }
215 -
216 - __syncthreads();
217 -
218 - if (tile_idx * tile_size + (threadIdx.y * tx + threadIdx.x + 1) * vec_size - 1 < feat_in) {
219 - y += sum;
220 - }
221 - }
222 -
223 - if (threadIdx.x == 0) {
224 - y_warpwise[threadIdx.y] = y;
225 - }
226 - __syncthreads();
227 -
228 - float y_write = 0.f;
229 -#pragma unroll
230 - for (size_t i = 0; i < ty; ++i) {
231 - y_write += y_warpwise[i];
232 - }
233 -
234 - // write Y;
235 - if (threadIdx.x == 0 && threadIdx.y == 0) {
236 - size_t y_idx = batch_idx * full_y_size + y_offset + j;
237 - Y[y_idx] = vllm_add<out_T>(Y[y_idx], convert_type<float, out_T>(y_write));
238 - }
239 -}
240 -
241 -#endif
242 -
243 -// nthrs = (2, 16, 4)
244 -template <int feat_in, int feat_out, size_t vec_size, int tx, int ty, int tz,
245 - typename in_T, typename out_T, typename W_T>
246 -__global__ void
247 -bgmv_expand_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
248 - const W_T *__restrict__ W,
249 - const int64_t *__restrict__ indicies, int64_t y_offset,
250 - int64_t full_y_size, int64_t num_layers, int64_t layer_idx,
251 - float scale) {
252 - size_t batch_idx = blockIdx.y;
253 - int64_t idx = indicies[batch_idx] * num_layers + layer_idx;
254 -
255 - if (idx < 0) {
256 - return;
257 - }
258 -
259 - auto block = cg::this_thread_block();
260 - size_t tile_idx = blockIdx.x;
261 -
262 - // load X;
263 - vec_t<in_T, vec_size> x_vec;
264 - x_vec.load(X + batch_idx * feat_in + threadIdx.x * vec_size);
265 -
266 - // load W;
267 - vec_t<W_T, vec_size> w_vec;
268 - w_vec.load(W + (idx * feat_out + tile_idx * tz * ty) * feat_in +
269 - block.thread_rank() * vec_size);
270 -
271 - float sum = 0.f;
272 -#pragma unroll
273 - for (size_t i = 0; i < vec_size; ++i) {
274 -#ifndef USE_ROCM
275 - sum += float(w_vec[i]) * float(x_vec[i]) * scale;
276 -#else
277 - sum += convert_type<W_T, float>(w_vec[i]) * convert_type<in_T, float>(x_vec[i]) * scale;
278 -#endif
279 - }
280 -
281 - cg::thread_block_tile g = cg::tiled_partition<tx>(block);
282 -#pragma unroll
283 - for (size_t offset = tx / 2; offset > 0; offset /= 2) {
284 - sum += g.shfl_down(sum, offset);
285 - }
286 - sum = g.shfl(sum, 0);
287 -
288 - if (threadIdx.x == 0) {
289 -#ifndef USE_ROCM
290 - Y[batch_idx * full_y_size + y_offset + tile_idx * (tz * ty) +
291 - threadIdx.z * ty + threadIdx.y] += static_cast<out_T>(sum);
292 -#else
293 - size_t y_idx = batch_idx * full_y_size + y_offset + tile_idx * (tz * ty) +
294 - threadIdx.z * ty + threadIdx.y;
295 - Y[y_idx] = vllm_add<out_T>(Y[y_idx], convert_type<float, out_T>(sum));
296 -#endif
297 - }
298 -}
299 -
300 -template <int feat_in, int feat_out, typename in_T, typename out_T,
301 - typename W_T>
302 -void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
303 - const W_T *__restrict__ W,
304 - const int64_t *__restrict__ indicies, int64_t y_offset,
305 - int64_t full_y_size, int64_t batch_size, int64_t num_layers,
306 - int64_t layer_idx, float scale) {
307 - constexpr size_t vec_size = 8;
308 - constexpr int tz = 4;
309 - const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
310 -
311 - if constexpr (feat_in <= feat_out) {
312 - static_assert(feat_in % vec_size == 0);
313 - constexpr int tx = feat_in / vec_size;
314 -
315 - static_assert((32 % tx == 0 && feat_out % (32 / tx * tz) == 0) ||
316 - (16 % tx == 0 && feat_out % (16 / tx * tz) == 0) ||
317 - (8 % tx == 0 && feat_out % (8 / tx * tz) == 0));
318 -
319 - if constexpr (32 % tx == 0 && feat_out % (32 / tx * tz) == 0) {
320 - constexpr int ty = 32 / tx;
321 - dim3 nblks(feat_out / (ty * tz), batch_size);
322 - dim3 nthrs(tx, ty, tz);
323 -
324 - bgmv_expand_kernel<feat_in, feat_out, vec_size, tx, ty, tz>
325 - <<<nblks, nthrs, 0, stream>>>(Y, X, W, indicies, y_offset,
326 - full_y_size, num_layers, layer_idx,
327 - scale);
328 - } else if (16 % tx == 0 && feat_out % (16 / tx * tz) == 0) {
329 - constexpr int ty = 16 / tx;
330 - dim3 nblks(feat_out / (ty * tz), batch_size);
331 - dim3 nthrs(tx, ty, tz);
332 -
333 - bgmv_expand_kernel<feat_in, feat_out, vec_size, tx, ty, tz>
334 - <<<nblks, nthrs, 0, stream>>>(Y, X, W, indicies, y_offset,
335 - full_y_size, num_layers, layer_idx,
336 - scale);
337 - } else {
338 - constexpr int ty = 8 / tx;
339 - dim3 nblks(feat_out / (ty * tz), batch_size);
340 - dim3 nthrs(tx, ty, tz);
341 -
342 - bgmv_expand_kernel<feat_in, feat_out, vec_size, tx, ty, tz>
343 - <<<nblks, nthrs, 0, stream>>>(Y, X, W, indicies, y_offset,
344 - full_y_size, num_layers, layer_idx,
345 - scale);
346 - }
347 - } else {
348 -#ifndef USE_ROCM
349 - static_assert(feat_in % (vec_size * 32) == 0 ||
350 - feat_in % (vec_size * 16) == 0 ||
351 - feat_in % (vec_size * 8) == 0);
352 -
353 - if constexpr (feat_in % (vec_size * 32) == 0) {
354 - constexpr int tx = 32;
355 - constexpr int ty = 4;
356 -
357 - dim3 nblks(feat_out, batch_size);
358 - dim3 nthrs(tx, ty);
359 -
360 - bgmv_shrink_kernel<feat_in, feat_out, vec_size, vec_size * sizeof(in_T),
361 - vec_size * sizeof(W_T), tx, ty, tz>
362 - <<<nblks, nthrs, 0, stream>>>(Y, X, W, indicies, y_offset,
363 - full_y_size, num_layers, layer_idx,
364 - scale);
365 - } else if constexpr (feat_in % (vec_size / 2 * 32) == 0) {
366 - constexpr int tx = 32;
367 - constexpr int ty = 4;
368 -
369 - dim3 nblks(feat_out, batch_size);
370 - dim3 nthrs(tx, ty);
371 -
372 - bgmv_shrink_kernel<feat_in, feat_out, vec_size / 2,
373 - vec_size * sizeof(in_T) / 2,
374 - vec_size * sizeof(W_T) / 2, tx, ty, tz>
375 - <<<nblks, nthrs, 0, stream>>>(Y, X, W, indicies, y_offset,
376 - full_y_size, num_layers, layer_idx,
377 - scale);
378 - } else if constexpr (feat_in % (vec_size / 2 * 16) == 0) {
379 - constexpr int tx = 16;
380 - constexpr int ty = 4;
381 -
382 - dim3 nblks(feat_out, batch_size);
383 - dim3 nthrs(tx, ty);
384 -
385 - bgmv_shrink_kernel<feat_in, feat_out, vec_size / 2,
386 - vec_size * sizeof(in_T) / 2,
387 - vec_size * sizeof(W_T) / 2, tx, ty, tz>
388 - <<<nblks, nthrs, 0, stream>>>(Y, X, W, indicies, y_offset,
389 - full_y_size, num_layers, layer_idx,
390 - scale);
391 - }
392 -#else
393 - constexpr size_t rocm_warp_size = warpSize;
394 -
395 -#define CHECK_INPUT_TILEABLE_BY(vec_size_) \
396 - feat_in % (rocm_warp_size * vec_size_) == 0
397 -
398 -#define LAUNCH_BGMV_SHRINK_KERNELS_ROCM(factor_, vec_size_, tx_, ty_) \
399 - if constexpr (CHECK_INPUT_TILEABLE_BY(factor_)) { \
400 - constexpr size_t vec_size_shrink = vec_size_; \
401 - constexpr int tx = tx_; \
402 - constexpr int ty = ty_; \
403 - dim3 nblks(feat_out, batch_size); \
404 - dim3 nthrs(tx, ty); \
405 - bgmv_shrink_kernel<feat_in, feat_out, vec_size_shrink, \
406 - vec_size_shrink * sizeof(in_T), \
407 - vec_size_shrink * sizeof(W_T), \
408 - tx, ty, tz> \
409 - <<<nblks, nthrs, 0, stream>>>(Y, X, W, indicies, y_offset, \
410 - full_y_size, num_layers, layer_idx, \
411 - scale); \
412 - }
413 -
414 - static_assert(CHECK_INPUT_TILEABLE_BY(32) ||
415 - CHECK_INPUT_TILEABLE_BY(16) ||
416 - CHECK_INPUT_TILEABLE_BY( 8) ||
417 - CHECK_INPUT_TILEABLE_BY( 4) ||
418 - CHECK_INPUT_TILEABLE_BY( 2) ||
419 - CHECK_INPUT_TILEABLE_BY( 1));
420 -
421 - LAUNCH_BGMV_SHRINK_KERNELS_ROCM(32, vec_size, rocm_warp_size, 32/vec_size)
422 - else
423 - LAUNCH_BGMV_SHRINK_KERNELS_ROCM(16, vec_size, rocm_warp_size, 16/vec_size)
424 - else
425 - LAUNCH_BGMV_SHRINK_KERNELS_ROCM( 8, vec_size, rocm_warp_size, 8/vec_size)
426 - else
427 - LAUNCH_BGMV_SHRINK_KERNELS_ROCM( 4, vec_size, rocm_warp_size/(vec_size/4), vec_size/4)
428 - else
429 - LAUNCH_BGMV_SHRINK_KERNELS_ROCM( 2, vec_size, rocm_warp_size/(vec_size/2), vec_size/2)
430 - else
431 - LAUNCH_BGMV_SHRINK_KERNELS_ROCM( 1, vec_size, rocm_warp_size/(vec_size/1), vec_size/1)
432 -
433 -#undef CHECK_INPUT_TILEABLE_BY
434 -#undef LAUNCH_BGMV_SHRINK_KERNELS_ROCM
435 -#endif
436 - }
437 -}
438 -
439 -#define INST_BGMV(feat_in, feat_out, in_T, out_T, W_T) \
440 - template void bgmv_kernel<feat_in, feat_out>( \
441 - out_T * __restrict__ Y, const in_T *__restrict__ X, \
442 - const W_T *__restrict__ W, const int64_t *__restrict__ indicies, \
443 - int64_t y_offset, int64_t full_y_size, int64_t batch_size, \
444 - int64_t num_layers, int64_t layer_idx, float scale);
445 -
446 -#define INST_BGMV_ONESIDE(in_T, out_T, W_T, feat_in, feat_out) \
447 - INST_BGMV(feat_in, feat_out, in_T, out_T, W_T)
448 -
449 -#define INST_BGMV_TWOSIDE(in_T, out_T, W_T, narrow, wide) \
450 - INST_BGMV(narrow, wide, in_T, out_T, W_T) \
451 - INST_BGMV(wide, narrow, in_T, out_T, W_T)
csrc/punica/bgmv/generator.py deleted
-48
@@ -1,48 +0,0 @@
1 -DTYPES = ["fp16", "bf16", "fp32"]
2 -DTYPE_MAP = {
3 - "fp16": "nv_half",
4 - "bf16": "nv_bfloat16",
5 - "fp32": "float",
6 -}
7 -
8 -TEMPLATE = """
9 -#include "bgmv_config.h"
10 -#include "bgmv_impl.cuh"
11 -
12 -FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, {input_dtype}, {output_dtype}, {weight_dtype})
13 -FOR_INST_BGMV_WIDE_NARROW(INST_BGMV_ONESIDE, {input_dtype}, {output_dtype}, {weight_dtype})
14 -""".lstrip() # noqa: E501
15 -
16 -for input_dtype in DTYPES:
17 - for output_dtype in DTYPES:
18 - for weight_dtype in DTYPES:
19 - if weight_dtype == "fp32":
20 - # FP32 weights are not supported.
21 - continue
22 - if output_dtype == "fp32":
23 - # LoRA A matrix.
24 - if input_dtype != weight_dtype:
25 - # NOTE(woosuk): While Punica supports the case where the
26 - # input and weight dtypes are different, we only generate
27 - # the kernels the same dtypes to reduce the binary size.
28 - continue
29 - elif input_dtype == "fp32":
30 - # LoRA B matrix.
31 - if output_dtype != weight_dtype:
32 - # NOTE(woosuk): While Punica supports the case where the
33 - # output and weight dtypes are different, we only generate
34 - # the kernels the same dtypes to reduce the binary size.
35 - continue
36 - elif not (input_dtype == output_dtype == weight_dtype):
37 - # NOTE(woosuk): While Punica supports mixed data types for
38 - # input, output, and weight, we only generate the kernels with
39 - # the same data types to reduce the binary size.
40 - continue
41 -
42 - kernel_definition = TEMPLATE.format(
43 - input_dtype=DTYPE_MAP[input_dtype],
44 - output_dtype=DTYPE_MAP[output_dtype],
45 - weight_dtype=DTYPE_MAP[weight_dtype])
46 - filename = f"bgmv_{input_dtype}_{output_dtype}_{weight_dtype}.cu"
47 - with open(filename, "w") as f:
48 - f.write(kernel_definition)
csrc/punica/bgmv/vec_dtypes.cuh deleted
-1325
@@ -1,1325 +0,0 @@
1 -#ifndef VEC_DTYPES_CUH_
2 -#define VEC_DTYPES_CUH_
3 -
4 -#ifdef FLASHINFER_USE_FP8
5 -#include <cuda_fp8.h>
6 -#endif
7 -#include <cuda_runtime.h>
8 -
9 -#include <type_traits>
10 -
11 -#include "../type_convert.h"
12 -#include "../../cuda_compat.h"
13 -
14 -#define FLASHINFER_INLINE \
15 - inline __attribute__((always_inline)) __device__ __host__
16 -
17 -template <typename float_t, size_t vec_size>
18 -struct vec_t {
19 - FLASHINFER_INLINE float_t &operator[](size_t i);
20 - FLASHINFER_INLINE const float_t &operator[](size_t i) const;
21 - FLASHINFER_INLINE void fill(float_t val);
22 - FLASHINFER_INLINE void load(const float_t *ptr);
23 - FLASHINFER_INLINE void store(float_t *ptr) const;
24 - template <typename T>
25 - FLASHINFER_INLINE void cast_from(const vec_t<T, vec_size> &src);
26 - template <typename T>
27 - FLASHINFER_INLINE void cast_load(const T *ptr);
28 - template <typename T>
29 - FLASHINFER_INLINE void cast_store(T *ptr) const;
30 - FLASHINFER_INLINE static void memcpy(float_t *dst, const float_t *src);
31 -};
32 -
33 -template <typename src_float_t, typename tgt_float_t, size_t vec_size>
34 -FLASHINFER_INLINE void cast_from_impl(const vec_t<src_float_t, vec_size> &src,
35 - vec_t<tgt_float_t, vec_size> &dst) {
36 -#pragma unroll
37 - for (size_t i = 0; i < vec_size; ++i) {
38 - dst[i] = tgt_float_t(src[i]);
39 - }
40 -}
41 -
42 -template <typename src_float_t, typename tgt_float_t, size_t vec_size>
43 -FLASHINFER_INLINE void cast_load_impl(const src_float_t *src_ptr,
44 - vec_t<tgt_float_t, vec_size> &dst) {
45 - if constexpr (std::is_same<src_float_t, tgt_float_t>::value) {
46 - dst.load(src_ptr);
47 - } else {
48 - vec_t<src_float_t, vec_size> tmp;
49 - tmp.load(src_ptr);
50 - dst.cast_from(tmp);
51 - }
52 -}
53 -
54 -template <typename src_float_t, typename tgt_float_t, size_t vec_size>
55 -FLASHINFER_INLINE void cast_store_impl(const vec_t<src_float_t, vec_size> &src,
56 - tgt_float_t *dst_ptr) {
57 - if constexpr (std::is_same<src_float_t, tgt_float_t>::value) {
58 - src.store(dst_ptr);
59 - } else {
60 - vec_t<tgt_float_t, vec_size> tmp;
61 - tmp.cast_from(src);
62 - tmp.store(dst_ptr);
63 - }
64 -}
65 -
66 -#ifdef FLASHINFER_USE_FP8
67 -/******************* vec_t<__nv_fp8_e4m3> *******************/
68 -
69 -// __nv_fp8_e4m3 x 1
70 -template <>
71 -struct vec_t<__nv_fp8_e4m3, 1> {
72 - __nv_fp8_e4m3 data;
73 -
74 - FLASHINFER_INLINE __nv_fp8_e4m3 &operator[](size_t i) {
75 - return ((__nv_fp8_e4m3 *)(&data))[i];
76 - }
77 - FLASHINFER_INLINE const __nv_fp8_e4m3 &operator[](size_t i) const {
78 - return ((const __nv_fp8_e4m3 *)(&data))[i];
79 - }
80 - FLASHINFER_INLINE void fill(__nv_fp8_e4m3 val);
81 - FLASHINFER_INLINE void load(const __nv_fp8_e4m3 *ptr);
82 - FLASHINFER_INLINE void store(__nv_fp8_e4m3 *ptr) const;
83 - template <typename T>
84 - FLASHINFER_INLINE void cast_from(const vec_t<T, 1> &src) {
85 - cast_from_impl(src, *this);
86 - }
87 - template <typename T>
88 - FLASHINFER_INLINE void cast_load(const T *ptr) {
89 - cast_load_impl(ptr, *this);
90 - }
91 - template <typename T>
92 - FLASHINFER_INLINE void cast_store(T *ptr) const {
93 - cast_store_impl(*this, ptr);
94 - }
95 -
96 - FLASHINFER_INLINE static void memcpy(__nv_fp8_e4m3 *dst,
97 - const __nv_fp8_e4m3 *src);
98 -};
99 -
100 -FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 1>::fill(__nv_fp8_e4m3 val) {
101 - data = val;
102 -}
103 -
104 -FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 1>::load(const __nv_fp8_e4m3 *ptr) {
105 - data = *ptr;
106 -}
107 -
108 -FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 1>::store(
109 - __nv_fp8_e4m3 *ptr) const {
110 - *ptr = data;
111 -}
112 -
113 -FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 1>::memcpy(
114 - __nv_fp8_e4m3 *dst, const __nv_fp8_e4m3 *src) {
115 - *dst = *src;
116 -}
117 -
118 -// __nv_fp8_e4m3 x 2
119 -template <>
120 -struct vec_t<__nv_fp8_e4m3, 2> {
121 - __nv_fp8x2_e4m3 data;
122 -
123 - FLASHINFER_INLINE __nv_fp8_e4m3 &operator[](size_t i) {
124 - return ((__nv_fp8_e4m3 *)(&data))[i];
125 - }
126 - FLASHINFER_INLINE const __nv_fp8_e4m3 &operator[](size_t i) const {
127 - return ((const __nv_fp8_e4m3 *)(&data))[i];
128 - }
129 - FLASHINFER_INLINE void fill(__nv_fp8_e4m3 val);
130 - FLASHINFER_INLINE void load(const __nv_fp8_e4m3 *ptr);
131 - FLASHINFER_INLINE void store(__nv_fp8_e4m3 *ptr) const;
132 - template <typename T>
133 - FLASHINFER_INLINE void cast_from(const vec_t<T, 2> &src) {
134 - cast_from_impl(src, *this);
135 - }
136 - template <typename T>
137 - FLASHINFER_INLINE void cast_load(const T *ptr) {
138 - cast_load_impl(ptr, *this);
139 - }
140 - template <typename T>
141 - FLASHINFER_INLINE void cast_store(T *ptr) const {
142 - cast_store_impl(*this, ptr);
143 - }
144 -
145 - FLASHINFER_INLINE static void memcpy(__nv_fp8_e4m3 *dst,
146 - const __nv_fp8_e4m3 *src);
147 -};
148 -
149 -FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 2>::fill(__nv_fp8_e4m3 val) {
150 - data.__x =
151 - (__nv_fp8x2_storage_t(val.__x) << 8) | __nv_fp8x2_storage_t(val.__x);
152 -}
153 -
154 -FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 2>::load(const __nv_fp8_e4m3 *ptr) {
155 - data = *((__nv_fp8x2_e4m3 *)ptr);
156 -}
157 -
158 -FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 2>::store(
159 - __nv_fp8_e4m3 *ptr) const {
160 - *((__nv_fp8x2_e4m3 *)ptr) = data;
161 -}
162 -
163 -FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 2>::memcpy(
164 - __nv_fp8_e4m3 *dst, const __nv_fp8_e4m3 *src) {
165 - *((__nv_fp8x2_e4m3 *)dst) = *((__nv_fp8x2_e4m3 *)src);
166 -}
167 -
168 -// __nv_fp8_e4m3 x 4
169 -
170 -template <>
171 -struct vec_t<__nv_fp8_e4m3, 4> {
172 - __nv_fp8x4_e4m3 data;
173 -
174 - FLASHINFER_INLINE __nv_fp8_e4m3 &operator[](size_t i) {
175 - return ((__nv_fp8_e4m3 *)(&data))[i];
176 - }
177 - FLASHINFER_INLINE const __nv_fp8_e4m3 &operator[](size_t i) const {
178 - return ((const __nv_fp8_e4m3 *)(&data))[i];
179 - }
180 - FLASHINFER_INLINE void fill(__nv_fp8_e4m3 val);
181 - FLASHINFER_INLINE void load(const __nv_fp8_e4m3 *ptr);
182 - FLASHINFER_INLINE void store(__nv_fp8_e4m3 *ptr) const;
183 - template <typename T>
184 - FLASHINFER_INLINE void cast_from(const vec_t<T, 4> &src) {
185 - cast_from_impl(src, *this);
186 - }
187 - template <typename T>
188 - FLASHINFER_INLINE void cast_load(const T *ptr) {
189 - cast_load_impl(ptr, *this);
190 - }
191 - template <typename T>
192 - FLASHINFER_INLINE void cast_store(T *ptr) const {
193 - cast_store_impl(*this, ptr);
194 - }
195 -
196 - FLASHINFER_INLINE static void memcpy(__nv_fp8_e4m3 *dst,
197 - const __nv_fp8_e4m3 *src);
198 -};
199 -
200 -FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 4>::fill(__nv_fp8_e4m3 val) {
201 - data.__x = (__nv_fp8x4_storage_t(val.__x) << 24) |
202 - (__nv_fp8x4_storage_t(val.__x) << 16) |
203 - (__nv_fp8x4_storage_t(val.__x) << 8) |
204 - __nv_fp8x4_storage_t(val.__x);
205 -}
206 -
207 -FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 4>::load(const __nv_fp8_e4m3 *ptr) {
208 - data = *((__nv_fp8x4_e4m3 *)ptr);
209 -}
210 -
211 -FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 4>::store(
212 - __nv_fp8_e4m3 *ptr) const {
213 - *((__nv_fp8x4_e4m3 *)ptr) = data;
214 -}
215 -
216 -FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 4>::memcpy(
217 - __nv_fp8_e4m3 *dst, const __nv_fp8_e4m3 *src) {
218 - *((__nv_fp8x4_e4m3 *)dst) = *((__nv_fp8x4_e4m3 *)src);
219 -}
220 -
221 -// __nv_fp8_e4m3 x 8
222 -
223 -template <>
224 -struct vec_t<__nv_fp8_e4m3, 8> {
225 - uint2 data;
226 -
227 - FLASHINFER_INLINE __nv_fp8_e4m3 &operator[](size_t i) {
228 - return ((__nv_fp8_e4m3 *)(&data))[i];
229 - }
230 - FLASHINFER_INLINE const __nv_fp8_e4m3 &operator[](size_t i) const {
231 - return ((const __nv_fp8_e4m3 *)(&data))[i];
232 - }
233 - FLASHINFER_INLINE void fill(__nv_fp8_e4m3 val);
234 - FLASHINFER_INLINE void load(const __nv_fp8_e4m3 *ptr);
235 - FLASHINFER_INLINE void store(__nv_fp8_e4m3 *ptr) const;
236 - template <typename T>
237 - FLASHINFER_INLINE void cast_from(const vec_t<T, 8> &src) {
238 - cast_from_impl(src, *this);
239 - }
240 - template <typename T>
241 - FLASHINFER_INLINE void cast_load(const T *ptr) {
242 - cast_load_impl(ptr, *this);
243 - }
244 - template <typename T>
245 - FLASHINFER_INLINE void cast_store(T *ptr) const {
246 - cast_store_impl(*this, ptr);
247 - }
248 -
249 - FLASHINFER_INLINE static void memcpy(__nv_fp8_e4m3 *dst,
250 - const __nv_fp8_e4m3 *src);
251 -};
252 -
253 -FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 8>::fill(__nv_fp8_e4m3 val) {
254 - ((__nv_fp8x4_e4m3 *)(&data.x))->__x = (__nv_fp8x4_storage_t(val.__x) << 24) |
255 - (__nv_fp8x4_storage_t(val.__x) << 16) |
256 - (__nv_fp8x4_storage_t(val.__x) << 8) |
257 - __nv_fp8x4_storage_t(val.__x);
258 - ((__nv_fp8x4_e4m3 *)(&data.y))->__x = (__nv_fp8x4_storage_t(val.__x) << 24) |
259 - (__nv_fp8x4_storage_t(val.__x) << 16) |
260 - (__nv_fp8x4_storage_t(val.__x) << 8) |
261 - __nv_fp8x4_storage_t(val.__x);
262 -}
263 -
264 -FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 8>::load(const __nv_fp8_e4m3 *ptr) {
265 - data = *((uint2 *)ptr);
266 -}
267 -
268 -FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 8>::store(
269 - __nv_fp8_e4m3 *ptr) const {
270 - *((uint2 *)ptr) = data;
271 -}
272 -
273 -FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 8>::memcpy(
274 - __nv_fp8_e4m3 *dst, const __nv_fp8_e4m3 *src) {
275 - *((__nv_fp8_e4m3 *)dst) = *((__nv_fp8_e4m3 *)src);
276 -}
277 -
278 -// __nv_fp8_e4m3 x 16 or more
279 -template <size_t vec_size>
280 -struct vec_t<__nv_fp8_e4m3, vec_size> {
281 - uint4 data[vec_size / 16];
282 -
283 - FLASHINFER_INLINE __nv_fp8_e4m3 &operator[](size_t i) {
284 - return ((__nv_fp8_e4m3 *)data)[i];
285 - }
286 - FLASHINFER_INLINE const __nv_fp8_e4m3 &operator[](size_t i) const {
287 - return ((const __nv_fp8_e4m3 *)data)[i];
288 - }
289 - FLASHINFER_INLINE void fill(__nv_fp8_e4m3 val) {
290 -#pragma unroll
291 - for (size_t i = 0; i < vec_size / 16; ++i) {
292 - ((__nv_fp8x4_e4m3 *)(&(data[i].x)))->__x =
293 - (__nv_fp8x4_storage_t(val.__x) << 24) |
294 - (__nv_fp8x4_storage_t(val.__x) << 16) |
295 - (__nv_fp8x4_storage_t(val.__x) << 8) | __nv_fp8x4_storage_t(val.__x);
296 - ((__nv_fp8x4_e4m3 *)(&(data[i].y)))->__x =
297 - (__nv_fp8x4_storage_t(val.__x) << 24) |
298 - (__nv_fp8x4_storage_t(val.__x) << 16) |
299 - (__nv_fp8x4_storage_t(val.__x) << 8) | __nv_fp8x4_storage_t(val.__x);
300 - ((__nv_fp8x4_e4m3 *)(&(data[i].z)))->__x =
301 - (__nv_fp8x4_storage_t(val.__x) << 24) |
302 - (__nv_fp8x4_storage_t(val.__x) << 16) |
303 - (__nv_fp8x4_storage_t(val.__x) << 8) | __nv_fp8x4_storage_t(val.__x);
304 - ((__nv_fp8x4_e4m3 *)(&(data[i].w)))->__x =
305 - (__nv_fp8x4_storage_t(val.__x) << 24) |
306 - (__nv_fp8x4_storage_t(val.__x) << 16) |
307 - (__nv_fp8x4_storage_t(val.__x) << 8) | __nv_fp8x4_storage_t(val.__x);
308 - }
309 - }
310 - FLASHINFER_INLINE void load(const __nv_fp8_e4m3 *ptr) {
311 -#pragma unroll
312 - for (size_t i = 0; i < vec_size / 16; ++i) {
313 - data[i] = ((uint4 *)ptr)[i];
314 - }
315 - }
316 - FLASHINFER_INLINE void store(__nv_fp8_e4m3 *ptr) const {
317 -#pragma unroll
318 - for (size_t i = 0; i < vec_size / 16; ++i) {
319 - ((uint4 *)ptr)[i] = data[i];
320 - }
321 - }
322 - template <typename T>
323 - FLASHINFER_INLINE void cast_from(const vec_t<T, vec_size> &src) {
324 - cast_from_impl(src, *this);
325 - }
326 - template <typename T>
327 - FLASHINFER_INLINE void cast_load(const T *ptr) {
328 - cast_load_impl(ptr, *this);
329 - }
330 - template <typename T>
331 - FLASHINFER_INLINE void cast_store(T *ptr) const {
332 - cast_store_impl(*this, ptr);
333 - }
334 -
335 - FLASHINFER_INLINE static void memcpy(__nv_fp8_e4m3 *dst,
336 - const __nv_fp8_e4m3 *src) {
337 -#pragma unroll
338 - for (size_t i = 0; i < vec_size / 16; ++i) {
339 - ((uint4 *)dst)[i] = ((uint4 *)src)[i];
340 - }
341 - }
342 -};
343 -
344 -/******************* vec_t<__nv_fp8_e5m2> *******************/
345 -
346 -// __nv_fp8_e5m2 x 1
347 -template <>
348 -struct vec_t<__nv_fp8_e5m2, 1> {
349 - __nv_fp8_e5m2 data;
350 -
351 - FLASHINFER_INLINE __nv_fp8_e5m2 &operator[](size_t i) {
352 - return ((__nv_fp8_e5m2 *)(&data))[i];
353 - }
354 - FLASHINFER_INLINE const __nv_fp8_e5m2 &operator[](size_t i) const {
355 - return ((const __nv_fp8_e5m2 *)(&data))[i];
356 - }
357 - FLASHINFER_INLINE void fill(__nv_fp8_e5m2 val);
358 - FLASHINFER_INLINE void load(const __nv_fp8_e5m2 *ptr);
359 - FLASHINFER_INLINE void store(__nv_fp8_e5m2 *ptr) const;
360 - template <typename T>
361 - FLASHINFER_INLINE void cast_from(const vec_t<T, 1> &src) {
362 - cast_from_impl(src, *this);
363 - }
364 - template <typename T>
365 - FLASHINFER_INLINE void cast_load(const T *ptr) {
366 - cast_load_impl(ptr, *this);
367 - }
368 - template <typename T>
369 - FLASHINFER_INLINE void cast_store(T *ptr) const {
370 - cast_store_impl(*this, ptr);
371 - }
372 -
373 - FLASHINFER_INLINE static void memcpy(__nv_fp8_e5m2 *dst,
374 - const __nv_fp8_e5m2 *src);
375 -};
376 -
377 -FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 1>::fill(__nv_fp8_e5m2 val) {
378 - data = val;
379 -}
380 -
381 -FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 1>::load(const __nv_fp8_e5m2 *ptr) {
382 - data = *ptr;
383 -}
384 -
385 -FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 1>::store(
386 - __nv_fp8_e5m2 *ptr) const {
387 - *ptr = data;
388 -}
389 -
390 -FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 1>::memcpy(
391 - __nv_fp8_e5m2 *dst, const __nv_fp8_e5m2 *src) {
392 - *dst = *src;
393 -}
394 -
395 -// __nv_fp8_e5m2 x 2
396 -template <>
397 -struct vec_t<__nv_fp8_e5m2, 2> {
398 - __nv_fp8x2_e5m2 data;
399 -
400 - FLASHINFER_INLINE __nv_fp8_e5m2 &operator[](size_t i) {
401 - return ((__nv_fp8_e5m2 *)(&data))[i];
402 - }
403 - FLASHINFER_INLINE const __nv_fp8_e5m2 &operator[](size_t i) const {
404 - return ((const __nv_fp8_e5m2 *)(&data))[i];
405 - }
406 - FLASHINFER_INLINE void fill(__nv_fp8_e5m2 val);
407 - FLASHINFER_INLINE void load(const __nv_fp8_e5m2 *ptr);
408 - FLASHINFER_INLINE void store(__nv_fp8_e5m2 *ptr) const;
409 - template <typename T>
410 - FLASHINFER_INLINE void cast_from(const vec_t<T, 2> &src) {
411 - cast_from_impl(src, *this);
412 - }
413 - template <typename T>
414 - FLASHINFER_INLINE void cast_load(const T *ptr) {
415 - cast_load_impl(ptr, *this);
416 - }
417 - template <typename T>
418 - FLASHINFER_INLINE void cast_store(T *ptr) const {
419 - cast_store_impl(*this, ptr);
420 - }
421 -
422 - FLASHINFER_INLINE static void memcpy(__nv_fp8_e5m2 *dst,
423 - const __nv_fp8_e5m2 *src);
424 -};
425 -
426 -FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 2>::fill(__nv_fp8_e5m2 val) {
427 - data.__x =
428 - (__nv_fp8x2_storage_t(val.__x) << 8) | __nv_fp8x2_storage_t(val.__x);
429 -}
430 -
431 -FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 2>::load(const __nv_fp8_e5m2 *ptr) {
432 - data = *((__nv_fp8x2_e5m2 *)ptr);
433 -}
434 -
435 -FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 2>::store(
436 - __nv_fp8_e5m2 *ptr) const {
437 - *((__nv_fp8x2_e5m2 *)ptr) = data;
438 -}
439 -
440 -FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 2>::memcpy(
441 - __nv_fp8_e5m2 *dst, const __nv_fp8_e5m2 *src) {
442 - *((__nv_fp8x2_e5m2 *)dst) = *((__nv_fp8x2_e5m2 *)src);
443 -}
444 -
445 -// __nv_fp8_e5m2 x 4
446 -
447 -template <>
448 -struct vec_t<__nv_fp8_e5m2, 4> {
449 - __nv_fp8x4_e5m2 data;
450 -
451 - FLASHINFER_INLINE __nv_fp8_e5m2 &operator[](size_t i) {
452 - return ((__nv_fp8_e5m2 *)(&data))[i];
453 - }
454 - FLASHINFER_INLINE const __nv_fp8_e5m2 &operator[](size_t i) const {
455 - return ((const __nv_fp8_e5m2 *)(&data))[i];
456 - }
457 - FLASHINFER_INLINE void fill(__nv_fp8_e5m2 val);
458 - FLASHINFER_INLINE void load(const __nv_fp8_e5m2 *ptr);
459 - FLASHINFER_INLINE void store(__nv_fp8_e5m2 *ptr) const;
460 - template <typename T>
461 - FLASHINFER_INLINE void cast_from(const vec_t<T, 4> &src) {
462 - cast_from_impl(src, *this);
463 - }
464 - template <typename T>
465 - FLASHINFER_INLINE void cast_load(const T *ptr) {
466 - cast_load_impl(ptr, *this);
467 - }
468 - template <typename T>
469 - FLASHINFER_INLINE void cast_store(T *ptr) const {
470 - cast_store_impl(*this, ptr);
471 - }
472 -
473 - FLASHINFER_INLINE static void memcpy(__nv_fp8_e5m2 *dst,
474 - const __nv_fp8_e5m2 *src);
475 -};
476 -
477 -FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 4>::fill(__nv_fp8_e5m2 val) {
478 - data.__x = (__nv_fp8x4_storage_t(val.__x) << 24) |
479 - (__nv_fp8x4_storage_t(val.__x) << 16) |
480 - (__nv_fp8x4_storage_t(val.__x) << 8) |
481 - __nv_fp8x4_storage_t(val.__x);
482 -}
483 -
484 -FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 4>::load(const __nv_fp8_e5m2 *ptr) {
485 - data = *((__nv_fp8x4_e5m2 *)ptr);
486 -}
487 -
488 -FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 4>::store(
489 - __nv_fp8_e5m2 *ptr) const {
490 - *((__nv_fp8x4_e5m2 *)ptr) = data;
491 -}
492 -
493 -FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 4>::memcpy(
494 - __nv_fp8_e5m2 *dst, const __nv_fp8_e5m2 *src) {
495 - *((__nv_fp8x4_e5m2 *)dst) = *((__nv_fp8x4_e5m2 *)src);
496 -}
497 -
498 -// __nv_fp8_e5m2 x 8
499 -
500 -template <>
501 -struct vec_t<__nv_fp8_e5m2, 8> {
502 - uint2 data;
503 -
504 - FLASHINFER_INLINE __nv_fp8_e5m2 &operator[](size_t i) {
505 - return ((__nv_fp8_e5m2 *)(&data))[i];
506 - }
507 - FLASHINFER_INLINE const __nv_fp8_e5m2 &operator[](size_t i) const {
508 - return ((const __nv_fp8_e5m2 *)(&data))[i];
509 - }
510 - FLASHINFER_INLINE void fill(__nv_fp8_e5m2 val);
511 - FLASHINFER_INLINE void load(const __nv_fp8_e5m2 *ptr);
512 - FLASHINFER_INLINE void store(__nv_fp8_e5m2 *ptr) const;
513 - template <typename T>
514 - FLASHINFER_INLINE void cast_from(const vec_t<T, 8> &src) {
515 - cast_from_impl(src, *this);
516 - }
517 - template <typename T>
518 - FLASHINFER_INLINE void cast_load(const T *ptr) {
519 - cast_load_impl(ptr, *this);
520 - }
521 - template <typename T>
522 - FLASHINFER_INLINE void cast_store(T *ptr) const {
523 - cast_store_impl(*this, ptr);
524 - }
525 -
526 - FLASHINFER_INLINE static void memcpy(__nv_fp8_e5m2 *dst,
527 - const __nv_fp8_e5m2 *src);
528 -};
529 -
530 -FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 8>::fill(__nv_fp8_e5m2 val) {
531 - ((__nv_fp8x4_e5m2 *)(&data.x))->__x = (__nv_fp8x4_storage_t(val.__x) << 24) |
532 - (__nv_fp8x4_storage_t(val.__x) << 16) |
533 - (__nv_fp8x4_storage_t(val.__x) << 8) |
534 - __nv_fp8x4_storage_t(val.__x);
535 - ((__nv_fp8x4_e5m2 *)(&data.y))->__x = (__nv_fp8x4_storage_t(val.__x) << 24) |
536 - (__nv_fp8x4_storage_t(val.__x) << 16) |
537 - (__nv_fp8x4_storage_t(val.__x) << 8) |
538 - __nv_fp8x4_storage_t(val.__x);
539 -}
540 -
541 -FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 8>::load(const __nv_fp8_e5m2 *ptr) {
542 - data = *((uint2 *)ptr);
543 -}
544 -
545 -FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 8>::store(
546 - __nv_fp8_e5m2 *ptr) const {
547 - *((uint2 *)ptr) = data;
548 -}
549 -
550 -FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 8>::memcpy(
551 - __nv_fp8_e5m2 *dst, const __nv_fp8_e5m2 *src) {
552 - *((__nv_fp8_e5m2 *)dst) = *((__nv_fp8_e5m2 *)src);
553 -}
554 -
555 -// __nv_fp8_e5m2 x 16 or more
556 -
557 -template <size_t vec_size>
558 -struct vec_t<__nv_fp8_e5m2, vec_size> {
559 - uint4 data[vec_size / 16];
560 -
561 - FLASHINFER_INLINE __nv_fp8_e5m2 &operator[](size_t i) {
562 - return ((__nv_fp8_e5m2 *)data)[i];
563 - }
564 - FLASHINFER_INLINE const __nv_fp8_e5m2 &operator[](size_t i) const {
565 - return ((const __nv_fp8_e5m2 *)data)[i];
566 - }
567 - FLASHINFER_INLINE void fill(__nv_fp8_e5m2 val) {
568 -#pragma unroll
569 - for (size_t i = 0; i < vec_size / 16; ++i) {
570 - ((__nv_fp8x4_e5m2 *)(&(data[i].x)))->__x =
571 - (__nv_fp8x4_storage_t(val.__x) << 24) |
572 - (__nv_fp8x4_storage_t(val.__x) << 16) |
573 - (__nv_fp8x4_storage_t(val.__x) << 8) | __nv_fp8x4_storage_t(val.__x);
574 - ((__nv_fp8x4_e5m2 *)(&(data[i].y)))->__x =
575 - (__nv_fp8x4_storage_t(val.__x) << 24) |
576 - (__nv_fp8x4_storage_t(val.__x) << 16) |
577 - (__nv_fp8x4_storage_t(val.__x) << 8) | __nv_fp8x4_storage_t(val.__x);
578 - ((__nv_fp8x4_e5m2 *)(&(data[i].z)))->__x =
579 - (__nv_fp8x4_storage_t(val.__x) << 24) |
580 - (__nv_fp8x4_storage_t(val.__x) << 16) |
581 - (__nv_fp8x4_storage_t(val.__x) << 8) | __nv_fp8x4_storage_t(val.__x);
582 - ((__nv_fp8x4_e5m2 *)(&(data[i].w)))->__x =
583 - (__nv_fp8x4_storage_t(val.__x) << 24) |
584 - (__nv_fp8x4_storage_t(val.__x) << 16) |
585 - (__nv_fp8x4_storage_t(val.__x) << 8) | __nv_fp8x4_storage_t(val.__x);
586 - }
587 - }
588 - FLASHINFER_INLINE void load(const __nv_fp8_e5m2 *ptr) {
589 -#pragma unroll
590 - for (size_t i = 0; i < vec_size / 16; ++i) {
591 - data[i] = ((uint4 *)ptr)[i];
592 - }
593 - }
594 - FLASHINFER_INLINE void store(__nv_fp8_e5m2 *ptr) const {
595 -#pragma unroll
596 - for (size_t i = 0; i < vec_size / 16; ++i) {
597 - ((uint4 *)ptr)[i] = data[i];
598 - }
599 - }
600 - template <typename T>
601 - FLASHINFER_INLINE void cast_from(const vec_t<T, vec_size> &src) {
602 - cast_from_impl(src, *this);
603 - }
604 - template <typename T>
605 - FLASHINFER_INLINE void cast_load(const T *ptr) {
606 - cast_load_impl(ptr, *this);
607 - }
608 - template <typename T>
609 - FLASHINFER_INLINE void cast_store(T *ptr) const {
610 - cast_store_impl(*this, ptr);
611 - }
612 -
613 - FLASHINFER_INLINE static void memcpy(__nv_fp8_e5m2 *dst,
614 - const __nv_fp8_e5m2 *src) {
615 -#pragma unroll
616 - for (size_t i = 0; i < vec_size / 16; ++i) {
617 - ((uint4 *)dst)[i] = ((uint4 *)src)[i];
618 - }
619 - }
620 -};
621 -#endif
622 -
623 -/******************* vec_t<half> *******************/
624 -
625 -// half x 1
626 -template <>
627 -struct vec_t<half, 1> {
628 - half data;
629 -
630 - FLASHINFER_INLINE half &operator[](size_t i) { return ((half *)(&data))[i]; }
631 - FLASHINFER_INLINE const half &operator[](size_t i) const {
632 - return ((const half *)(&data))[i];
633 - }
634 - FLASHINFER_INLINE void fill(half val);
635 - FLASHINFER_INLINE void load(const half *ptr);
636 - FLASHINFER_INLINE void store(half *ptr) const;
637 - template <typename T>
638 - FLASHINFER_INLINE void cast_from(const vec_t<T, 1> &src) {
639 - cast_from_impl(src, *this);
640 - }
641 - template <typename T>
642 - FLASHINFER_INLINE void cast_load(const T *ptr) {
643 - cast_load_impl(ptr, *this);
644 - }
645 - template <typename T>
646 - FLASHINFER_INLINE void cast_store(T *ptr) const {
647 - cast_store_impl(*this, ptr);
648 - }
649 -
650 - FLASHINFER_INLINE static void memcpy(half *dst, const half *src);
651 -};
652 -
653 -FLASHINFER_INLINE void vec_t<half, 1>::fill(half val) { data = val; }
654 -
655 -FLASHINFER_INLINE void vec_t<half, 1>::load(const half *ptr) { data = *ptr; }
656 -
657 -FLASHINFER_INLINE void vec_t<half, 1>::store(half *ptr) const { *ptr = data; }
658 -
659 -FLASHINFER_INLINE void vec_t<half, 1>::memcpy(half *dst, const half *src) {
660 - *dst = *src;
661 -}
662 -
663 -// half x 2
664 -template <>
665 -struct vec_t<half, 2> {
666 - half2 data;
667 -
668 - FLASHINFER_INLINE half &operator[](size_t i) { return ((half *)(&data))[i]; }
669 - FLASHINFER_INLINE const half &operator[](size_t i) const {
670 - return ((const half *)(&data))[i];
671 - }
672 - FLASHINFER_INLINE void fill(half val);
673 - FLASHINFER_INLINE void load(const half *ptr);
674 - FLASHINFER_INLINE void store(half *ptr) const;
675 - template <typename T>
676 - FLASHINFER_INLINE void cast_from(const vec_t<T, 2> &src) {
677 - cast_from_impl(src, *this);
678 - }
679 - template <typename T>
680 - FLASHINFER_INLINE void cast_load(const T *ptr) {
681 - cast_load_impl(ptr, *this);
682 - }
683 - template <typename T>
684 - FLASHINFER_INLINE void cast_store(T *ptr) const {
685 - cast_store_impl(*this, ptr);
686 - }
687 -
688 - FLASHINFER_INLINE static void memcpy(half *dst, const half *src);
689 -};
690 -
691 -FLASHINFER_INLINE void vec_t<half, 2>::fill(half val) {
692 - data = make_half2(val, val);
693 -}
694 -
695 -FLASHINFER_INLINE void vec_t<half, 2>::load(const half *ptr) {
696 - data = *((half2 *)ptr);
697 -}
698 -
699 -FLASHINFER_INLINE void vec_t<half, 2>::store(half *ptr) const {
700 - *((half2 *)ptr) = data;
701 -}
702 -
703 -FLASHINFER_INLINE void vec_t<half, 2>::memcpy(half *dst, const half *src) {
704 - *((half2 *)dst) = *((half2 *)src);
705 -}
706 -
707 -// half x 4
708 -
709 -template <>
710 -struct vec_t<half, 4> {
711 - uint2 data;
712 -
713 - FLASHINFER_INLINE half &operator[](size_t i) { return ((half *)(&data))[i]; }
714 - FLASHINFER_INLINE const half &operator[](size_t i) const {
715 - return ((const half *)(&data))[i];
716 - }
717 - FLASHINFER_INLINE void fill(half val);
718 - FLASHINFER_INLINE void load(const half *ptr);
719 - FLASHINFER_INLINE void store(half *ptr) const;
720 - template <typename T>
721 - FLASHINFER_INLINE void cast_from(const vec_t<T, 4> &src) {
722 - cast_from_impl(src, *this);
723 - }
724 - template <typename T>
725 - FLASHINFER_INLINE void cast_load(const T *ptr) {
726 - cast_load_impl(ptr, *this);
727 - }
728 - template <typename T>
729 - FLASHINFER_INLINE void cast_store(T *ptr) const {
730 - cast_store_impl(*this, ptr);
731 - }
732 -
733 - FLASHINFER_INLINE static void memcpy(half *dst, const half *src);
734 -};
735 -
736 -FLASHINFER_INLINE void vec_t<half, 4>::fill(half val) {
737 - *(half2 *)(&data.x) = make_half2(val, val);
738 - *(half2 *)(&data.y) = make_half2(val, val);
739 -}
740 -
741 -FLASHINFER_INLINE void vec_t<half, 4>::load(const half *ptr) {
742 - data = *((uint2 *)ptr);
743 -}
744 -
745 -FLASHINFER_INLINE void vec_t<half, 4>::store(half *ptr) const {
746 - *((uint2 *)ptr) = data;
747 -}
748 -
749 -FLASHINFER_INLINE void vec_t<half, 4>::memcpy(half *dst, const half *src) {
750 - *((uint2 *)dst) = *((uint2 *)src);
751 -}
752 -
753 -// half x 8 or more
754 -
755 -template <size_t vec_size>
756 -struct vec_t<half, vec_size> {
757 - uint4 data[vec_size / 8];
758 - FLASHINFER_INLINE half &operator[](size_t i) { return ((half *)data)[i]; }
759 - FLASHINFER_INLINE const half &operator[](size_t i) const {
760 - return ((const half *)data)[i];
761 - }
762 - FLASHINFER_INLINE void fill(half val) {
763 -#pragma unroll
764 - for (size_t i = 0; i < vec_size; ++i) {
765 - *(half2 *)(&(data[i].x)) = make_half2(val, val);
766 - *(half2 *)(&(data[i].y)) = make_half2(val, val);
767 - *(half2 *)(&(data[i].z)) = make_half2(val, val);
768 - *(half2 *)(&(data[i].w)) = make_half2(val, val);
769 - }
770 - }
771 - FLASHINFER_INLINE void load(const half *ptr) {
772 -#pragma unroll
773 - for (size_t i = 0; i < vec_size / 8; ++i) {
774 - data[i] = ((uint4 *)ptr)[i];
775 - }
776 - }
777 - FLASHINFER_INLINE void store(half *ptr) const {
778 -#pragma unroll
779 - for (size_t i = 0; i < vec_size / 8; ++i) {
780 - ((uint4 *)ptr)[i] = data[i];
781 - }
782 - }
783 - template <typename T>
784 - FLASHINFER_INLINE void cast_from(const vec_t<T, vec_size> &src) {
785 - cast_from_impl(src, *this);
786 - }
787 - template <typename T>
788 - FLASHINFER_INLINE void cast_load(const T *ptr) {
789 - cast_load_impl(ptr, *this);
790 - }
791 - template <typename T>
792 - FLASHINFER_INLINE void cast_store(T *ptr) const {
793 - cast_store_impl(*this, ptr);
794 - }
795 -
796 - FLASHINFER_INLINE static void memcpy(half *dst, const half *src) {
797 -#pragma unroll
798 - for (size_t i = 0; i < vec_size / 8; ++i) {
799 - ((uint4 *)dst)[i] = ((uint4 *)src)[i];
800 - }
801 - }
802 -};
803 -
804 -/******************* vec_t<nv_bfloat16> *******************/
805 -
806 -// nv_bfloat16 x 1
807 -template <>
808 -struct vec_t<nv_bfloat16, 1> {
809 - nv_bfloat16 data;
810 -
811 - FLASHINFER_INLINE nv_bfloat16 &operator[](size_t i) {
812 - return ((nv_bfloat16 *)(&data))[i];
813 - }
814 - FLASHINFER_INLINE const nv_bfloat16 &operator[](size_t i) const {
815 - return ((const nv_bfloat16 *)(&data))[i];
816 - }
817 - FLASHINFER_INLINE void fill(nv_bfloat16 val);
818 - FLASHINFER_INLINE void load(const nv_bfloat16 *ptr);
819 - FLASHINFER_INLINE void store(nv_bfloat16 *ptr) const;
820 - template <typename T>
821 - FLASHINFER_INLINE void cast_from(const vec_t<T, 1> &src) {
822 - cast_from_impl(src, *this);
823 - }
824 - template <typename T>
825 - FLASHINFER_INLINE void cast_load(const T *ptr) {
826 - cast_load_impl(ptr, *this);
827 - }
828 - template <typename T>
829 - FLASHINFER_INLINE void cast_store(T *ptr) const {
830 - cast_store_impl(*this, ptr);
831 - }
832 -
833 - FLASHINFER_INLINE static void memcpy(nv_bfloat16 *dst,
834 - const nv_bfloat16 *src);
835 -};
836 -
837 -FLASHINFER_INLINE void vec_t<nv_bfloat16, 1>::fill(nv_bfloat16 val) {
838 - data = val;
839 -}
840 -
841 -FLASHINFER_INLINE void vec_t<nv_bfloat16, 1>::load(const nv_bfloat16 *ptr) {
842 - data = *ptr;
843 -}
844 -
845 -FLASHINFER_INLINE void vec_t<nv_bfloat16, 1>::store(nv_bfloat16 *ptr) const {
846 - *ptr = data;
847 -}
848 -
849 -FLASHINFER_INLINE void vec_t<nv_bfloat16, 1>::memcpy(nv_bfloat16 *dst,
850 - const nv_bfloat16 *src) {
851 - *dst = *src;
852 -}
853 -
854 -// nv_bfloat16 x 2
855 -template <>
856 -struct vec_t<nv_bfloat16, 2> {
857 - nv_bfloat162 data;
858 -
859 - FLASHINFER_INLINE nv_bfloat16 &operator[](size_t i) {
860 - return ((nv_bfloat16 *)(&data))[i];
861 - }
862 - FLASHINFER_INLINE const nv_bfloat16 &operator[](size_t i) const {
863 - return ((const nv_bfloat16 *)(&data))[i];
864 - }
865 - FLASHINFER_INLINE void fill(nv_bfloat16 val);
866 - FLASHINFER_INLINE void load(const nv_bfloat16 *ptr);
867 - FLASHINFER_INLINE void store(nv_bfloat16 *ptr) const;
868 - template <typename T>
869 - FLASHINFER_INLINE void cast_from(const vec_t<T, 2> &src) {
870 - cast_from_impl(src, *this);
871 - }
872 - template <typename T>
873 - FLASHINFER_INLINE void cast_load(const T *ptr) {
874 - cast_load_impl(ptr, *this);
875 - }
876 - template <typename T>
877 - FLASHINFER_INLINE void cast_store(T *ptr) const {
878 - cast_store_impl(*this, ptr);
879 - }
880 -
881 - FLASHINFER_INLINE static void memcpy(nv_bfloat16 *dst,
882 - const nv_bfloat16 *src);
883 -};
884 -
885 -FLASHINFER_INLINE void vec_t<nv_bfloat16, 2>::fill(nv_bfloat16 val) {
886 - data = make_bfloat162(val, val);
887 -}
888 -
889 -FLASHINFER_INLINE void vec_t<nv_bfloat16, 2>::load(const nv_bfloat16 *ptr) {
890 - data = *((nv_bfloat162 *)ptr);
891 -}
892 -
893 -FLASHINFER_INLINE void vec_t<nv_bfloat16, 2>::store(nv_bfloat16 *ptr) const {
894 - *((nv_bfloat162 *)ptr) = data;
895 -}
896 -
897 -FLASHINFER_INLINE void vec_t<nv_bfloat16, 2>::memcpy(nv_bfloat16 *dst,
898 - const nv_bfloat16 *src) {
899 - *((nv_bfloat162 *)dst) = *((nv_bfloat162 *)src);
900 -}
901 -
902 -// nv_bfloat16 x 4
903 -
904 -template <>
905 -struct vec_t<nv_bfloat16, 4> {
906 - uint2 data;
907 -
908 - FLASHINFER_INLINE nv_bfloat16 &operator[](size_t i) {
909 - return ((nv_bfloat16 *)(&data))[i];
910 - }
911 - FLASHINFER_INLINE const nv_bfloat16 &operator[](size_t i) const {
912 - return ((const nv_bfloat16 *)(&data))[i];
913 - }
914 - FLASHINFER_INLINE void fill(nv_bfloat16 val);
915 - FLASHINFER_INLINE void load(const nv_bfloat16 *ptr);
916 - FLASHINFER_INLINE void store(nv_bfloat16 *ptr) const;
917 - template <typename T>
918 - FLASHINFER_INLINE void cast_from(const vec_t<T, 4> &src) {
919 - cast_from_impl(src, *this);
920 - }
921 - template <typename T>
922 - FLASHINFER_INLINE void cast_load(const T *ptr) {
923 - cast_load_impl(ptr, *this);
924 - }
925 - template <typename T>
926 - FLASHINFER_INLINE void cast_store(T *ptr) const {
927 - cast_store_impl(*this, ptr);
928 - }
929 -
930 - FLASHINFER_INLINE static void memcpy(nv_bfloat16 *dst,
931 - const nv_bfloat16 *src);
932 -};
933 -
934 -FLASHINFER_INLINE void vec_t<nv_bfloat16, 4>::fill(nv_bfloat16 val) {
935 - *(nv_bfloat162 *)(&data.x) = make_bfloat162(val, val);
936 - *(nv_bfloat162 *)(&data.y) = make_bfloat162(val, val);
937 -}
938 -
939 -FLASHINFER_INLINE void vec_t<nv_bfloat16, 4>::load(const nv_bfloat16 *ptr) {
940 - data = *((uint2 *)ptr);
941 -}
942 -
943 -FLASHINFER_INLINE void vec_t<nv_bfloat16, 4>::store(nv_bfloat16 *ptr) const {
944 - *((uint2 *)ptr) = data;
945 -}
946 -
947 -FLASHINFER_INLINE void vec_t<nv_bfloat16, 4>::memcpy(nv_bfloat16 *dst,
948 - const nv_bfloat16 *src) {
949 - *((uint2 *)dst) = *((uint2 *)src);
950 -}
951 -
952 -// nv_bfloat16 x 8 or more
953 -
954 -template <size_t vec_size>
955 -struct vec_t<nv_bfloat16, vec_size> {
956 - uint4 data[vec_size / 8];
957 -
958 - FLASHINFER_INLINE nv_bfloat16 &operator[](size_t i) {
959 - return ((nv_bfloat16 *)data)[i];
960 - }
961 - FLASHINFER_INLINE const nv_bfloat16 &operator[](size_t i) const {
962 - return ((const nv_bfloat16 *)data)[i];
963 - }
964 - FLASHINFER_INLINE void fill(nv_bfloat16 val) {
965 -#pragma unoll
966 - for (size_t i = 0; i < vec_size / 8; ++i) {
967 - *(nv_bfloat162 *)(&(data[i].x)) = make_bfloat162(val, val);
968 - *(nv_bfloat162 *)(&(data[i].y)) = make_bfloat162(val, val);
969 - *(nv_bfloat162 *)(&(data[i].z)) = make_bfloat162(val, val);
970 - *(nv_bfloat162 *)(&(data[i].w)) = make_bfloat162(val, val);
971 - }
972 - }
973 - FLASHINFER_INLINE void load(const nv_bfloat16 *ptr) {
974 -#pragma unoll
975 - for (size_t i = 0; i < vec_size / 8; ++i) {
976 - data[i] = ((uint4 *)ptr)[i];
977 - }
978 - }
979 - FLASHINFER_INLINE void store(nv_bfloat16 *ptr) const {
980 -#pragma unoll
981 - for (size_t i = 0; i < vec_size / 8; ++i) {
982 - ((uint4 *)ptr)[i] = data[i];
983 - }
984 - }
985 - template <typename T>
986 - FLASHINFER_INLINE void cast_from(const vec_t<T, vec_size> &src) {
987 - cast_from_impl(src, *this);
988 - }
989 - template <typename T>
990 - FLASHINFER_INLINE void cast_load(const T *ptr) {
991 - cast_load_impl(ptr, *this);
992 - }
993 - template <typename T>
994 - FLASHINFER_INLINE void cast_store(T *ptr) const {
995 - cast_store_impl(*this, ptr);
996 - }
997 -
998 - FLASHINFER_INLINE static void memcpy(nv_bfloat16 *dst,
999 - const nv_bfloat16 *src) {
1000 -#pragma unoll
1001 - for (size_t i = 0; i < vec_size / 8; ++i) {
1002 - ((uint4 *)dst)[i] = ((uint4 *)src)[i];
1003 - }
1004 - }
1005 -};
1006 -
1007 -/******************* vec_t<float> *******************/
1008 -
1009 -// float x 1
1010 -
1011 -template <>
1012 -struct vec_t<float, 1> {
1013 - float data;
1014 -
1015 - FLASHINFER_INLINE float &operator[](size_t i) {
1016 - return ((float *)(&data))[i];
1017 - }
1018 - FLASHINFER_INLINE const float &operator[](size_t i) const {
1019 - return ((const float *)(&data))[i];
1020 - }
1021 - FLASHINFER_INLINE void fill(float val);
1022 - FLASHINFER_INLINE void load(const float *ptr);
1023 - FLASHINFER_INLINE void store(float *ptr) const;
1024 - template <typename T>
1025 - FLASHINFER_INLINE void cast_from(const vec_t<T, 1> &src) {
1026 - cast_from_impl(src, *this);
1027 - }
1028 - template <typename T>
1029 - FLASHINFER_INLINE void cast_load(const T *ptr) {
1030 - cast_load_impl(ptr, *this);
1031 - }
1032 - template <typename T>
1033 - FLASHINFER_INLINE void cast_store(T *ptr) const {
1034 - cast_store_impl(*this, ptr);
1035 - }
1036 -
1037 - FLASHINFER_INLINE static void memcpy(float *dst, const float *src);
1038 -};
1039 -
1040 -FLASHINFER_INLINE void vec_t<float, 1>::fill(float val) { data = val; }
1041 -
1042 -FLASHINFER_INLINE void vec_t<float, 1>::load(const float *ptr) { data = *ptr; }
1043 -
1044 -FLASHINFER_INLINE void vec_t<float, 1>::store(float *ptr) const { *ptr = data; }
1045 -
1046 -FLASHINFER_INLINE void vec_t<float, 1>::memcpy(float *dst, const float *src) {
1047 - *dst = *src;
1048 -}
1049 -
1050 -// float x 2
1051 -
1052 -template <>
1053 -struct vec_t<float, 2> {
1054 - float2 data;
1055 -
1056 - FLASHINFER_INLINE float &operator[](size_t i) {
1057 - return ((float *)(&data))[i];
1058 - }
1059 - FLASHINFER_INLINE const float &operator[](size_t i) const {
1060 - return ((const float *)(&data))[i];
1061 - }
1062 - FLASHINFER_INLINE void fill(float val);
1063 - FLASHINFER_INLINE void load(const float *ptr);
1064 - FLASHINFER_INLINE void store(float *ptr) const;
1065 - template <typename T>
1066 - FLASHINFER_INLINE void cast_from(const vec_t<T, 2> &src) {
1067 - cast_from_impl(src, *this);
1068 - }
1069 - template <typename T>
1070 - FLASHINFER_INLINE void cast_load(const T *ptr) {
1071 - cast_load_impl(ptr, *this);
1072 - }
1073 - template <typename T>
1074 - FLASHINFER_INLINE void cast_store(T *ptr) const {
1075 - cast_store_impl(*this, ptr);
1076 - }
1077 - FLASHINFER_INLINE static void memcpy(float *dst, const float *src);
1078 -};
1079 -
1080 -FLASHINFER_INLINE void vec_t<float, 2>::fill(float val) {
1081 - data = make_float2(val, val);
1082 -}
1083 -
1084 -FLASHINFER_INLINE void vec_t<float, 2>::load(const float *ptr) {
1085 - data = *((float2 *)ptr);
1086 -}
1087 -
1088 -FLASHINFER_INLINE void vec_t<float, 2>::store(float *ptr) const {
1089 - *((float2 *)ptr) = data;
1090 -}
1091 -
1092 -FLASHINFER_INLINE void vec_t<float, 2>::memcpy(float *dst, const float *src) {
1093 - *((float2 *)dst) = *((float2 *)src);
1094 -}
1095 -
1096 -// float x 4 or more
1097 -template <size_t vec_size>
1098 -struct vec_t<float, vec_size> {
1099 - float4 data[vec_size / 4];
1100 -
1101 - FLASHINFER_INLINE float &operator[](size_t i) { return ((float *)(data))[i]; }
1102 - FLASHINFER_INLINE const float &operator[](size_t i) const {
1103 - return ((const float *)(data))[i];
1104 - }
1105 - FLASHINFER_INLINE void fill(float val) {
1106 -#pragma unroll
1107 - for (size_t i = 0; i < vec_size / 4; ++i) {
1108 - data[i] = make_float4(val, val, val, val);
1109 - }
1110 - }
1111 - FLASHINFER_INLINE void load(const float *ptr) {
1112 -#pragma unroll
1113 - for (size_t i = 0; i < vec_size / 4; ++i) {
1114 - data[i] = ((float4 *)ptr)[i];
1115 - }
1116 - }
1117 - FLASHINFER_INLINE void store(float *ptr) const {
1118 -#pragma unroll
1119 - for (size_t i = 0; i < vec_size / 4; ++i) {
1120 - ((float4 *)ptr)[i] = data[i];
1121 - }
1122 - }
1123 - template <typename T>
1124 - FLASHINFER_INLINE void cast_from(const vec_t<T, vec_size> &src) {
1125 - cast_from_impl(src, *this);
1126 - }
1127 - template <typename T>
1128 - FLASHINFER_INLINE void cast_load(const T *ptr) {
1129 - cast_load_impl(ptr, *this);
1130 - }
1131 - template <typename T>
1132 - FLASHINFER_INLINE void cast_store(T *ptr) const {
1133 - cast_store_impl(*this, ptr);
1134 - }
1135 - FLASHINFER_INLINE static void memcpy(float *dst, const float *src) {
1136 -#pragma unroll
1137 - for (size_t i = 0; i < vec_size / 4; ++i) {
1138 - ((float4 *)dst)[i] = ((float4 *)src)[i];
1139 - }
1140 - }
1141 -};
1142 -
1143 -/******************* vec_t type cast *******************/
1144 -
1145 -template <size_t vec_size>
1146 -FLASHINFER_INLINE void cast_from_impl(const vec_t<half, vec_size> &src,
1147 - vec_t<float, vec_size> &dst) {
1148 - if constexpr (vec_size == 1) {
1149 - dst.data = float(src.data);
1150 - } else {
1151 -#pragma unroll
1152 - for (size_t i = 0; i < vec_size / 2; ++i) {
1153 - ((float2 *)(&dst.data))[i] = __half22float2(((half2 *)(&src.data))[i]);
1154 - }
1155 - }
1156 -}
1157 -
1158 -template <size_t vec_size>
1159 -FLASHINFER_INLINE void cast_from_impl(const vec_t<float, vec_size> &src,
1160 - vec_t<half, vec_size> &dst) {
1161 - if constexpr (vec_size == 1) {
1162 - dst.data = half(src.data);
1163 - } else {
1164 -#pragma unroll
1165 - for (size_t i = 0; i < vec_size / 2; ++i) {
1166 - ((half2 *)(&dst.data))[i] = __float22half2_rn(((float2 *)(&src.data))[i]);
1167 - }
1168 - }
1169 -}
1170 -
1171 -template <size_t vec_size>
1172 -FLASHINFER_INLINE void cast_from_impl(const vec_t<nv_bfloat16, vec_size> &src,
1173 - vec_t<float, vec_size> &dst) {
1174 - if constexpr (vec_size == 1) {
1175 - dst.data = float(src.data);
1176 - } else {
1177 -#pragma unroll
1178 - for (size_t i = 0; i < vec_size / 2; ++i) {
1179 - ((float2 *)(&dst.data))[i] =
1180 - __bfloat1622float2(((nv_bfloat162 *)(&src.data))[i]);
1181 - }
1182 - }
1183 -}
1184 -
1185 -template <size_t vec_size>
1186 -FLASHINFER_INLINE void cast_from_impl(const vec_t<float, vec_size> &src,
1187 - vec_t<nv_bfloat16, vec_size> &dst) {
1188 - if constexpr (vec_size == 1) {
1189 - dst.data = nv_bfloat16(src.data);
1190 - } else {
1191 -#pragma unroll
1192 - for (size_t i = 0; i < vec_size / 2; ++i) {
1193 - ((nv_bfloat162 *)(&dst.data))[i] =
1194 - __float22bfloat162_rn(((float2 *)(&src.data))[i]);
1195 - }
1196 - }
1197 -}
1198 -
1199 -#ifdef FLASHINFER_USE_FP8
1200 -
1201 -template <size_t vec_size>
1202 -FLASHINFER_INLINE void cast_from_impl(const vec_t<__nv_fp8_e4m3, vec_size> &src,
1203 - vec_t<float, vec_size> &dst) {
1204 - if constexpr (vec_size == 1) {
1205 - dst.data = float(src.data);
1206 - } else if constexpr (vec_size == 2) {
1207 - *(float2 *)(&dst.data) = float2(*(__nv_fp8x2_e4m3 *)(&src.data));
1208 - } else {
1209 -#pragma unroll
1210 - for (size_t i = 0; i < vec_size / 4; ++i) {
1211 - ((float4 *)(&dst.data))[i] = float4(((__nv_fp8x4_e4m3 *)(&src.data))[i]);
1212 - }
1213 - }
1214 -}
1215 -
1216 -template <size_t vec_size>
1217 -FLASHINFER_INLINE void cast_from_impl(const vec_t<__nv_fp8_e4m3, vec_size> &src,
1218 - vec_t<half, vec_size> &dst) {
1219 - if constexpr (vec_size == 1) {
1220 - dst.data = float(src.data);
1221 - } else {
1222 -#pragma unroll
1223 - for (size_t i = 0; i < vec_size / 2; ++i) {
1224 - ((half2 *)(&dst.data))[i] = half2(((__nv_fp8x2_e4m3 *)(&src.data))[i]);
1225 - }
1226 - }
1227 -}
1228 -
1229 -template <size_t vec_size>
1230 -FLASHINFER_INLINE void cast_from_impl(const vec_t<float, vec_size> &src,
1231 - vec_t<__nv_fp8_e4m3, vec_size> &dst) {
1232 - if constexpr (vec_size == 1) {
1233 - dst.data = __nv_fp8_e4m3(src.data);
1234 - } else if constexpr (vec_size == 2) {
1235 - *(__nv_fp8x2_e4m3 *)(&dst.data) = __nv_fp8x2_e4m3(*(float2 *)(&src.data));
1236 - } else {
1237 -#pragma unroll
1238 - for (size_t i = 0; i < vec_size / 4; ++i) {
1239 - ((__nv_fp8x4_e4m3 *)(&dst.data))[i] =
1240 - __nv_fp8x4_e4m3(((float4 *)(&src.data))[i]);
1241 - }
1242 - }
1243 -}
1244 -
1245 -template <size_t vec_size>
1246 -FLASHINFER_INLINE void cast_from_impl(const vec_t<half, vec_size> &src,
1247 - vec_t<__nv_fp8_e4m3, vec_size> &dst) {
1248 - if constexpr (vec_size == 1) {
1249 - dst.data = __nv_fp8_e4m3(src.data);
1250 - } else if constexpr (vec_size == 2) {
1251 - *(__nv_fp8x2_e4m3 *)(&dst.data) = __nv_fp8x2_e4m3(*(half2 *)(&src.data));
1252 - } else {
1253 -#pragma unroll
1254 - for (size_t i = 0; i < vec_size / 4; ++i) {
1255 - // NOTE(Zihao): need to double check if we properly handle flo and fhi
1256 - ((__nv_fp8x4_e4m3 *)(&dst.data))[i] = __nv_fp8x4_e4m3(
1257 - ((half2 *)(&src.data))[i * 2], ((half2 *)(&src.data))[i * 2 + 1]);
1258 - }
1259 - }
1260 -}
1261 -
1262 -template <size_t vec_size>
1263 -FLASHINFER_INLINE void cast_from_impl(const vec_t<__nv_fp8_e5m2, vec_size> &src,
1264 - vec_t<float, vec_size> &dst) {
1265 - if constexpr (vec_size == 1) {
1266 - dst.data = float(src.data);
1267 - } else if constexpr (vec_size == 2) {
1268 - *(float2 *)(&dst.data) = float2(*(__nv_fp8x2_e5m2 *)(&src.data));
1269 - } else {
1270 -#pragma unroll
1271 - for (size_t i = 0; i < vec_size / 4; ++i) {
1272 - ((float4 *)(&dst.data))[i] = float4(((__nv_fp8x4_e5m2 *)(&src.data))[i]);
1273 - }
1274 - }
1275 -}
1276 -
1277 -template <size_t vec_size>
1278 -FLASHINFER_INLINE void cast_from_impl(const vec_t<__nv_fp8_e5m2, vec_size> &src,
1279 - vec_t<half, vec_size> &dst) {
1280 - if constexpr (vec_size == 1) {
1281 - dst.data = float(src.data);
1282 - } else {
1283 -#pragma unroll
1284 - for (size_t i = 0; i < vec_size / 2; ++i) {
1285 - ((half2 *)(&dst.data))[i] = half2(((__nv_fp8x2_e5m2 *)(&src.data))[i]);
1286 - }
1287 - }
1288 -}
1289 -
1290 -template <size_t vec_size>
1291 -FLASHINFER_INLINE void cast_from_impl(const vec_t<float, vec_size> &src,
1292 - vec_t<__nv_fp8_e5m2, vec_size> &dst) {
1293 - if constexpr (vec_size == 1) {
1294 - dst.data = __nv_fp8_e5m2(src.data);
1295 - } else if constexpr (vec_size == 2) {
1296 - *(__nv_fp8x2_e5m2 *)(&dst.data) = __nv_fp8x2_e5m2(*(float2 *)(&src.data));
1297 - } else {
1298 -#pragma unroll
1299 - for (size_t i = 0; i < vec_size / 4; ++i) {
1300 - ((__nv_fp8x4_e5m2 *)(&dst.data))[i] =
1301 - __nv_fp8x4_e5m2(((float4 *)(&src.data))[i]);
1302 - }
1303 - }
1304 -}
1305 -
1306 -template <size_t vec_size>
1307 -FLASHINFER_INLINE void cast_from_impl(const vec_t<half, vec_size> &src,
1308 - vec_t<__nv_fp8_e5m2, vec_size> &dst) {
1309 - if constexpr (vec_size == 1) {
1310 - dst.data = __nv_fp8_e4m3(src.data);
1311 - } else if constexpr (vec_size == 2) {
1312 - *(__nv_fp8x2_e5m2 *)(&dst.data) = __nv_fp8x2_e5m2(*(half2 *)(&src.data));
1313 - } else {
1314 -#pragma unroll
1315 - for (size_t i = 0; i < vec_size / 4; ++i) {
1316 - // NOTE(Zihao): need to double check if we properly handle flo and fhi
1317 - ((__nv_fp8x4_e5m2 *)(&dst.data))[i] = __nv_fp8x4_e5m2(
1318 - ((half2 *)(&src.data))[i * 2], ((half2 *)(&src.data))[i * 2 + 1]);
1319 - }
1320 - }
1321 -}
1322 -
1323 -#endif // FLASHINFER_USE_FP8
1324 -
1325 -#endif // VEC_DTYPES_CUH_
csrc/punica/punica_ops.cu deleted
-569
@@ -1,569 +0,0 @@
1 -#include <torch/all.h>
2 -#include <c10/cuda/CUDAGuard.h>
3 -#include <cstdint>
4 -
5 -#include "type_convert.h"
6 -#include "../cuda_compat.h"
7 -#include "bgmv/bgmv_config.h"
8 -
9 -
10 -//====== utils ======
11 -
12 -inline void check_shape(const torch::Tensor &a, const torch::Tensor &b,
13 - const char *a_name, const char *b_name) {
14 - TORCH_CHECK(a.dim() == b.dim(), a_name, ".dim() != ", b_name, ".dim(). ",
15 - a.dim(), " vs ", b.dim());
16 - for (int i = 0; i < a.dim(); ++i) {
17 - TORCH_CHECK(a.size(i) == b.size(i), a_name, ".size(", i, ") != ", b_name,
18 - ".size(", i, ")");
19 - }
20 -}
21 -
22 -inline constexpr uint64_t pack_u32(uint32_t a, uint32_t b) {
23 - return (uint64_t(a) << 32) | uint64_t(b);
24 -}
25 -
26 -#define CHECK_CUDA(x) TORCH_CHECK(x.is_cuda(), #x " must be a CUDA tensor")
27 -
28 -#define CHECK_CONTIGUOUS(x) \
29 - TORCH_CHECK(x.is_contiguous(), #x " must be contiguous")
30 -
31 -#define CHECK_INPUT(x) \
32 - CHECK_CUDA(x); \
33 - CHECK_CONTIGUOUS(x)
34 -
35 -#define CHECK_DIM(d, x) \
36 - TORCH_CHECK(x.dim() == d, #x " must be a " #d "D tensor")
37 -
38 -#define CHECK_SHAPE(a, b) check_shape(a, b, #a, #b)
39 -
40 -#define CHECK_EQ(a, b) \
41 - TORCH_CHECK(a == b, "CHECK_EQ(" #a ", " #b ") failed. ", a, " vs ", b)
42 -
43 -//====== bgmv ======
44 -
45 -template <typename in_T, typename out_T, typename W_T>
46 -inline bool launch_bgmv_kernel(out_T *Y, const in_T *X, const W_T *W,
47 - const int64_t *lora_indices,
48 - uint32_t in_features, uint32_t out_features,
49 - int64_t y_offset, int64_t full_y_size,
50 - int64_t batch_size, int64_t num_layers,
51 - int64_t layer_idx, float scale) {
52 - // NOTE(woosuk): While Punica supports various combinations of input/output
53 - // data types, we limit the supported data types to reduce the binary size.
54 - constexpr bool is_input_float = std::is_same<in_T, float>::value;
55 - constexpr bool is_output_float = std::is_same<out_T, float>::value;
56 - if (is_input_float) {
57 - if (!std::is_same<out_T, W_T>::value) {
58 - return false;
59 - }
60 - } else if (is_output_float) {
61 - if (!std::is_same<in_T, W_T>::value) {
62 - return false;
63 - }
64 - } else if (!(std::is_same<in_T, W_T>::value &&
65 - std::is_same<out_T, W_T>::value)) {
66 - return false;
67 - }
68 -
69 - switch (pack_u32(in_features, out_features)) {
70 -#define CASE_ONESIDE(_in_T, _out_T, _W_T, feat_in, feat_out) \
71 - case pack_u32(feat_in, feat_out): \
72 - bgmv_kernel<feat_in, feat_out>(Y, X, W, lora_indices, y_offset, \
73 - full_y_size, batch_size, num_layers, \
74 - layer_idx, scale); \
75 - break;
76 -#define CASE(_in_T, _out_T, _W_T, narrow, wide) \
77 - CASE_ONESIDE(in_T, out_T, W_T, narrow, wide) \
78 - CASE_ONESIDE(in_T, out_T, W_T, wide, narrow)
79 -
80 - FOR_BGMV_WIDE_NARROW(CASE, _, _, _)
81 - FOR_INST_BGMV_WIDE_NARROW(CASE_ONESIDE, _, _, _)
82 -#undef CASE
83 -#undef CASE_ONESIDE
84 - default:
85 - return false;
86 - }
87 - return true;
88 -}
89 -
90 -void dispatch_bgmv(torch::Tensor y, torch::Tensor x, torch::Tensor w,
91 - torch::Tensor indicies, int64_t layer_idx, double scale) {
92 - CHECK_INPUT(y);
93 - CHECK_INPUT(x);
94 - CHECK_INPUT(w);
95 - CHECK_INPUT(indicies);
96 -
97 - CHECK_DIM(2, y);
98 - CHECK_DIM(2, x);
99 - CHECK_DIM(4, w);
100 - CHECK_DIM(1, indicies);
101 -
102 - int64_t B = x.size(0);
103 - int64_t h_in = x.size(1);
104 - int64_t h_out = y.size(1);
105 - int64_t num_layers = w.size(1);
106 - CHECK_EQ(w.size(3), h_in);
107 - CHECK_EQ(w.size(2), h_out);
108 - CHECK_EQ(indicies.size(0), x.size(0));
109 - CHECK_EQ(y.size(0), x.size(0));
110 - const at::cuda::OptionalCUDAGuard device_guard(device_of(x));
111 - bool ok = false;
112 - if (h_in <= 128512 && h_out <= 128512) {
113 - // TODO: See if we can get rid of this massive nested switch
114 - switch (x.scalar_type()) {
115 - case at::ScalarType::Half:
116 - switch (y.scalar_type()) {
117 - case at::ScalarType::Half:
118 - switch (w.scalar_type()) {
119 - case at::ScalarType::Half:
120 - ok = launch_bgmv_kernel(static_cast<nv_half *>(y.data_ptr()),
121 - static_cast<nv_half *>(x.data_ptr()),
122 - static_cast<nv_half *>(w.data_ptr()),
123 - indicies.data_ptr<int64_t>(), h_in, h_out, 0,
124 - h_out, B, num_layers, layer_idx, scale);
125 - break;
126 - case at::ScalarType::BFloat16:
127 - ok = launch_bgmv_kernel(static_cast<nv_half *>(y.data_ptr()),
128 - static_cast<nv_half *>(x.data_ptr()),
129 - static_cast<nv_bfloat16 *>(w.data_ptr()),
130 - indicies.data_ptr<int64_t>(), h_in, h_out, 0,
131 - h_out, B, num_layers, layer_idx, scale);
132 - break;
133 - default:
134 - break;
135 - }
136 - break;
137 - case at::ScalarType::BFloat16:
138 - switch (w.scalar_type()) {
139 - case at::ScalarType::Half:
140 - ok = launch_bgmv_kernel(static_cast<nv_bfloat16 *>(y.data_ptr()),
141 - static_cast<nv_half *>(x.data_ptr()),
142 - static_cast<nv_half *>(w.data_ptr()),
143 - indicies.data_ptr<int64_t>(), h_in, h_out, 0,
144 - h_out, B, num_layers, layer_idx, scale);
145 - break;
146 - case at::ScalarType::BFloat16:
147 - ok = launch_bgmv_kernel(static_cast<nv_bfloat16 *>(y.data_ptr()),
148 - static_cast<nv_half *>(x.data_ptr()),
149 - static_cast<nv_bfloat16 *>(w.data_ptr()),
150 - indicies.data_ptr<int64_t>(), h_in, h_out, 0,
151 - h_out, B, num_layers, layer_idx, scale);
152 - break;
153 - default:
154 - break;
155 - }
156 - break;
157 - case at::ScalarType::Float:
158 - switch (w.scalar_type()) {
159 - case at::ScalarType::Half:
160 - ok = launch_bgmv_kernel(static_cast<float *>(y.data_ptr()),
161 - static_cast<nv_half *>(x.data_ptr()),
162 - static_cast<nv_half *>(w.data_ptr()),
163 - indicies.data_ptr<int64_t>(), h_in, h_out, 0,
164 - h_out, B, num_layers, layer_idx, scale);
165 - break;
166 - case at::ScalarType::BFloat16:
167 - ok = launch_bgmv_kernel(static_cast<float *>(y.data_ptr()),
168 - static_cast<nv_half *>(x.data_ptr()),
169 - static_cast<nv_bfloat16 *>(w.data_ptr()),
170 - indicies.data_ptr<int64_t>(), h_in, h_out, 0,
171 - h_out, B, num_layers, layer_idx, scale);
172 - break;
173 - default:
174 - break;
175 - }
176 - break;
177 - default:
178 - break;
179 - }
180 - break;
181 - case at::ScalarType::BFloat16:
182 - switch (y.scalar_type()) {
183 - case at::ScalarType::Half:
184 - switch (w.scalar_type()) {
185 - case at::ScalarType::Half:
186 - ok = launch_bgmv_kernel(static_cast<nv_half *>(y.data_ptr()),
187 - static_cast<nv_bfloat16 *>(x.data_ptr()),
188 - static_cast<nv_half *>(w.data_ptr()),
189 - indicies.data_ptr<int64_t>(), h_in, h_out, 0,
190 - h_out, B, num_layers, layer_idx, scale);
191 - break;
192 - case at::ScalarType::BFloat16:
193 - ok = launch_bgmv_kernel(static_cast<nv_half *>(y.data_ptr()),
194 - static_cast<nv_bfloat16 *>(x.data_ptr()),
195 - static_cast<nv_bfloat16 *>(w.data_ptr()),
196 - indicies.data_ptr<int64_t>(), h_in, h_out, 0,
197 - h_out, B, num_layers, layer_idx, scale);
198 - break;
199 - default:
200 - break;
201 - }
202 - break;
203 - case at::ScalarType::BFloat16:
204 - switch (w.scalar_type()) {
205 - case at::ScalarType::Half:
206 - ok = launch_bgmv_kernel(static_cast<nv_bfloat16 *>(y.data_ptr()),
207 - static_cast<nv_bfloat16 *>(x.data_ptr()),
208 - static_cast<nv_half *>(w.data_ptr()),
209 - indicies.data_ptr<int64_t>(), h_in, h_out, 0,
210 - h_out, B, num_layers, layer_idx, scale);
211 - break;
212 - case at::ScalarType::BFloat16:
213 - ok = launch_bgmv_kernel(static_cast<nv_bfloat16 *>(y.data_ptr()),
214 - static_cast<nv_bfloat16 *>(x.data_ptr()),
215 - static_cast<nv_bfloat16 *>(w.data_ptr()),
216 - indicies.data_ptr<int64_t>(), h_in, h_out, 0,
217 - h_out, B, num_layers, layer_idx, scale);
218 - break;
219 - default:
220 - break;
221 - }
222 - break;
223 - case at::ScalarType::Float:
224 - switch (w.scalar_type()) {
225 - case at::ScalarType::Half:
226 - ok = launch_bgmv_kernel(static_cast<float *>(y.data_ptr()),
227 - static_cast<nv_bfloat16 *>(x.data_ptr()),
228 - static_cast<nv_half *>(w.data_ptr()),
229 - indicies.data_ptr<int64_t>(), h_in, h_out, 0,
230 - h_out, B, num_layers, layer_idx, scale);
231 - break;
232 - case at::ScalarType::BFloat16:
233 - ok = launch_bgmv_kernel(static_cast<float *>(y.data_ptr()),
234 - static_cast<nv_bfloat16 *>(x.data_ptr()),
235 - static_cast<nv_bfloat16 *>(w.data_ptr()),
236 - indicies.data_ptr<int64_t>(), h_in, h_out, 0,
237 - h_out, B, num_layers, layer_idx, scale);
238 - break;
239 - default:
240 - break;
241 - }
242 - break;
243 - default:
244 - break;
245 - }
246 - break;
247 - case at::ScalarType::Float:
248 - switch (y.scalar_type()) {
249 - case at::ScalarType::Half:
250 - switch (w.scalar_type()) {
251 - case at::ScalarType::Half:
252 - ok = launch_bgmv_kernel(static_cast<nv_half *>(y.data_ptr()),
253 - static_cast<float *>(x.data_ptr()),
254 - static_cast<nv_half *>(w.data_ptr()),
255 - indicies.data_ptr<int64_t>(), h_in, h_out, 0,
256 - h_out, B, num_layers, layer_idx, scale);
257 - break;
258 - case at::ScalarType::BFloat16:
259 - ok = launch_bgmv_kernel(static_cast<nv_half *>(y.data_ptr()),
260 - static_cast<float *>(x.data_ptr()),
261 - static_cast<nv_bfloat16 *>(w.data_ptr()),
262 - indicies.data_ptr<int64_t>(), h_in, h_out, 0,
263 - h_out, B, num_layers, layer_idx, scale);
264 - break;
265 - default:
266 - break;
267 - }
268 - break;
269 - case at::ScalarType::BFloat16:
270 - switch (w.scalar_type()) {
271 - case at::ScalarType::Half:
272 - ok = launch_bgmv_kernel(static_cast<nv_bfloat16 *>(y.data_ptr()),
273 - static_cast<float *>(x.data_ptr()),
274 - static_cast<nv_half *>(w.data_ptr()),
275 - indicies.data_ptr<int64_t>(), h_in, h_out, 0,
276 - h_out, B, num_layers, layer_idx, scale);
277 - break;
278 - case at::ScalarType::BFloat16:
279 - ok = launch_bgmv_kernel(static_cast<nv_bfloat16 *>(y.data_ptr()),
280 - static_cast<float *>(x.data_ptr()),
281 - static_cast<nv_bfloat16 *>(w.data_ptr()),
282 - indicies.data_ptr<int64_t>(), h_in, h_out, 0,
283 - h_out, B, num_layers, layer_idx, scale);
284 - break;
285 - default:
286 - break;
287 - }
288 - break;
289 - case at::ScalarType::Float:
290 - switch (w.scalar_type()) {
291 - case at::ScalarType::Half:
292 - ok = launch_bgmv_kernel(static_cast<float *>(y.data_ptr()),
293 - static_cast<float *>(x.data_ptr()),
294 - static_cast<nv_half *>(w.data_ptr()),
295 - indicies.data_ptr<int64_t>(), h_in, h_out, 0,
296 - h_out, B, num_layers, layer_idx, scale);
297 - break;
298 - case at::ScalarType::BFloat16:
299 - ok = launch_bgmv_kernel(static_cast<float *>(y.data_ptr()),
300 - static_cast<float *>(x.data_ptr()),
301 - static_cast<nv_bfloat16 *>(w.data_ptr()),
302 - indicies.data_ptr<int64_t>(), h_in, h_out, 0,
303 - h_out, B, num_layers, layer_idx, scale);
304 - break;
305 - default:
306 - break;
307 - }
308 - break;
309 - default:
310 - break;
311 - }
312 - break;
313 - default:
314 - break;
315 - }
316 - }
317 - TORCH_CHECK(ok, "No suitable kernel.", " h_in=", h_in, " h_out=", h_out,
318 - " dtype=", x.scalar_type(), " out_dtype=", y.scalar_type());
319 -}
320 -
321 -void dispatch_bgmv_low_level(torch::Tensor y, torch::Tensor x, torch::Tensor w,
322 - torch::Tensor indicies, int64_t layer_idx,
323 - double scale, int64_t h_in, int64_t h_out,
324 - int64_t y_offset) {
325 - CHECK_INPUT(y);
326 - CHECK_INPUT(x);
327 - CHECK_INPUT(w);
328 - CHECK_INPUT(indicies);
329 -
330 - CHECK_DIM(2, y);
331 - CHECK_DIM(2, x);
332 - CHECK_DIM(4, w);
333 - CHECK_DIM(1, indicies);
334 -
335 - int64_t B = x.size(0);
336 - int64_t num_layers = w.size(1);
337 - int64_t full_y_size = y.size(1);
338 - CHECK_EQ(w.size(3), h_in);
339 - CHECK_EQ(w.size(2), h_out);
340 - CHECK_EQ(indicies.size(0), x.size(0));
341 - CHECK_EQ(y.size(0), x.size(0));
342 - const at::cuda::OptionalCUDAGuard device_guard(device_of(x));
343 - bool ok = false;
344 - if (h_in <= 128512 && h_out <= 128512) {
345 - // TODO: See if we can get rid of this massive nested switch
346 - switch (x.scalar_type()) {
347 - case at::ScalarType::Half:
348 - switch (y.scalar_type()) {
349 - case at::ScalarType::Half:
350 - switch (w.scalar_type()) {
351 - case at::ScalarType::Half:
352 - ok = launch_bgmv_kernel(static_cast<nv_half *>(y.data_ptr()),
353 - static_cast<nv_half *>(x.data_ptr()),
354 - static_cast<nv_half *>(w.data_ptr()),
355 - indicies.data_ptr<int64_t>(), h_in, h_out,
356 - y_offset, full_y_size, B, num_layers,
357 - layer_idx, scale);
358 - break;
359 - case at::ScalarType::BFloat16:
360 - ok = launch_bgmv_kernel(static_cast<nv_half *>(y.data_ptr()),
361 - static_cast<nv_half *>(x.data_ptr()),
362 - static_cast<nv_bfloat16 *>(w.data_ptr()),
363 - indicies.data_ptr<int64_t>(), h_in, h_out,
364 - y_offset, full_y_size, B, num_layers,
365 - layer_idx, scale);
366 - break;
367 - default:
368 - break;
369 - }
370 - break;
371 - case at::ScalarType::BFloat16:
372 - switch (w.scalar_type()) {
373 - case at::ScalarType::Half:
374 - ok = launch_bgmv_kernel(static_cast<nv_bfloat16 *>(y.data_ptr()),
375 - static_cast<nv_half *>(x.data_ptr()),
376 - static_cast<nv_half *>(w.data_ptr()),
377 - indicies.data_ptr<int64_t>(), h_in, h_out,
378 - y_offset, full_y_size, B, num_layers,
379 - layer_idx, scale);
380 - break;
381 - case at::ScalarType::BFloat16:
382 - ok = launch_bgmv_kernel(static_cast<nv_bfloat16 *>(y.data_ptr()),
383 - static_cast<nv_half *>(x.data_ptr()),
384 - static_cast<nv_bfloat16 *>(w.data_ptr()),
385 - indicies.data_ptr<int64_t>(), h_in, h_out,
386 - y_offset, full_y_size, B, num_layers,
387 - layer_idx, scale);
388 - break;
389 - default:
390 - break;
391 - }
392 - break;
393 - case at::ScalarType::Float:
394 - switch (w.scalar_type()) {
395 - case at::ScalarType::Half:
396 - ok = launch_bgmv_kernel(static_cast<float *>(y.data_ptr()),
397 - static_cast<nv_half *>(x.data_ptr()),
398 - static_cast<nv_half *>(w.data_ptr()),
399 - indicies.data_ptr<int64_t>(), h_in, h_out,
400 - y_offset, full_y_size, B, num_layers,
401 - layer_idx, scale);
402 - break;
403 - case at::ScalarType::BFloat16:
404 - ok = launch_bgmv_kernel(static_cast<float *>(y.data_ptr()),
405 - static_cast<nv_half *>(x.data_ptr()),
406 - static_cast<nv_bfloat16 *>(w.data_ptr()),
407 - indicies.data_ptr<int64_t>(), h_in, h_out,
408 - y_offset, full_y_size, B, num_layers,
409 - layer_idx, scale);
410 - break;
411 - default:
412 - break;
413 - }
414 - break;
415 - default:
416 - break;
417 - }
418 - break;
419 - case at::ScalarType::BFloat16:
420 - switch (y.scalar_type()) {
421 - case at::ScalarType::Half:
422 - switch (w.scalar_type()) {
423 - case at::ScalarType::Half:
424 - ok = launch_bgmv_kernel(static_cast<nv_half *>(y.data_ptr()),
425 - static_cast<nv_bfloat16 *>(x.data_ptr()),
426 - static_cast<nv_half *>(w.data_ptr()),
427 - indicies.data_ptr<int64_t>(), h_in, h_out,
428 - y_offset, full_y_size, B, num_layers,
429 - layer_idx, scale);
430 - break;
431 - case at::ScalarType::BFloat16:
432 - ok = launch_bgmv_kernel(static_cast<nv_half *>(y.data_ptr()),
433 - static_cast<nv_bfloat16 *>(x.data_ptr()),
434 - static_cast<nv_bfloat16 *>(w.data_ptr()),
435 - indicies.data_ptr<int64_t>(), h_in, h_out,
436 - y_offset, full_y_size, B, num_layers,
437 - layer_idx, scale);
438 - break;
439 - default:
440 - break;
441 - }
442 - break;
443 - case at::ScalarType::BFloat16:
444 - switch (w.scalar_type()) {
445 - case at::ScalarType::Half:
446 - ok = launch_bgmv_kernel(static_cast<nv_bfloat16 *>(y.data_ptr()),
447 - static_cast<nv_bfloat16 *>(x.data_ptr()),
448 - static_cast<nv_half *>(w.data_ptr()),
449 - indicies.data_ptr<int64_t>(), h_in, h_out,
450 - y_offset, full_y_size, B, num_layers,
451 - layer_idx, scale);
452 - break;
453 - case at::ScalarType::BFloat16:
454 - ok = launch_bgmv_kernel(static_cast<nv_bfloat16 *>(y.data_ptr()),
455 - static_cast<nv_bfloat16 *>(x.data_ptr()),
456 - static_cast<nv_bfloat16 *>(w.data_ptr()),
457 - indicies.data_ptr<int64_t>(), h_in, h_out,
458 - y_offset, full_y_size, B, num_layers,
459 - layer_idx, scale);
460 - break;
461 - default:
462 - break;
463 - }
464 - break;
465 - case at::ScalarType::Float:
466 - switch (w.scalar_type()) {
467 - case at::ScalarType::Half:
468 - ok = launch_bgmv_kernel(static_cast<float *>(y.data_ptr()),
469 - static_cast<nv_bfloat16 *>(x.data_ptr()),
470 - static_cast<nv_half *>(w.data_ptr()),
471 - indicies.data_ptr<int64_t>(), h_in, h_out,
472 - y_offset, full_y_size, B, num_layers,
473 - layer_idx, scale);
474 - break;
475 - case at::ScalarType::BFloat16:
476 - ok = launch_bgmv_kernel(static_cast<float *>(y.data_ptr()),
477 - static_cast<nv_bfloat16 *>(x.data_ptr()),
478 - static_cast<nv_bfloat16 *>(w.data_ptr()),
479 - indicies.data_ptr<int64_t>(), h_in, h_out,
480 - y_offset, full_y_size, B, num_layers,
481 - layer_idx, scale);
482 - break;
483 - default:
484 - break;
485 - }
486 - break;
487 - default:
488 - break;
489 - }
490 - break;
491 - case at::ScalarType::Float:
492 - switch (y.scalar_type()) {
493 - case at::ScalarType::Half:
494 - switch (w.scalar_type()) {
495 - case at::ScalarType::Half:
496 - ok = launch_bgmv_kernel(static_cast<nv_half *>(y.data_ptr()),
497 - static_cast<float *>(x.data_ptr()),
498 - static_cast<nv_half *>(w.data_ptr()),
499 - indicies.data_ptr<int64_t>(), h_in, h_out,
500 - y_offset, full_y_size, B, num_layers,
501 - layer_idx, scale);
502 - break;
503 - case at::ScalarType::BFloat16:
504 - ok = launch_bgmv_kernel(static_cast<nv_half *>(y.data_ptr()),
505 - static_cast<float *>(x.data_ptr()),
506 - static_cast<nv_bfloat16 *>(w.data_ptr()),
507 - indicies.data_ptr<int64_t>(), h_in, h_out,
508 - y_offset, full_y_size, B, num_layers,
509 - layer_idx, scale);
510 - break;
511 - default:
512 - break;
513 - }
514 - break;
515 - case at::ScalarType::BFloat16:
516 - switch (w.scalar_type()) {
517 - case at::ScalarType::Half:
518 - ok = launch_bgmv_kernel(static_cast<nv_bfloat16 *>(y.data_ptr()),
519 - static_cast<float *>(x.data_ptr()),
520 - static_cast<nv_half *>(w.data_ptr()),
521 - indicies.data_ptr<int64_t>(), h_in, h_out,
522 - y_offset, full_y_size, B, num_layers,
523 - layer_idx, scale);
524 - break;
525 - case at::ScalarType::BFloat16:
526 - ok = launch_bgmv_kernel(static_cast<nv_bfloat16 *>(y.data_ptr()),
527 - static_cast<float *>(x.data_ptr()),
528 - static_cast<nv_bfloat16 *>(w.data_ptr()),
529 - indicies.data_ptr<int64_t>(), h_in, h_out,
530 - y_offset, full_y_size, B, num_layers,
531 - layer_idx, scale);
532 - break;
533 - default:
534 - break;
535 - }
536 - break;
537 - case at::ScalarType::Float:
538 - switch (w.scalar_type()) {
539 - case at::ScalarType::Half:
540 - ok = launch_bgmv_kernel(static_cast<float *>(y.data_ptr()),
541 - static_cast<float *>(x.data_ptr()),
542 - static_cast<nv_half *>(w.data_ptr()),
543 - indicies.data_ptr<int64_t>(), h_in, h_out,
544 - y_offset, full_y_size, B, num_layers,
545 - layer_idx, scale);
546 - break;
547 - case at::ScalarType::BFloat16:
548 - ok = launch_bgmv_kernel(static_cast<float *>(y.data_ptr()),
549 - static_cast<float *>(x.data_ptr()),
550 - static_cast<nv_bfloat16 *>(w.data_ptr()),
551 - indicies.data_ptr<int64_t>(), h_in, h_out,
552 - y_offset, full_y_size, B, num_layers,
553 - layer_idx, scale);
554 - break;
555 - default:
556 - break;
557 - }
558 - break;
559 - default:
560 - break;
561 - }
562 - break;
563 - default:
564 - break;
565 - }
566 - }
567 - TORCH_CHECK(ok, "No suitable kernel.", " h_in=", h_in, " h_out=", h_out,
568 - " dtype=", x.scalar_type(), " out_dtype=", y.scalar_type());
569 -}
csrc/punica/punica_ops.h deleted
-11
@@ -1,11 +0,0 @@
1 -#pragma once
2 -
3 -#include <torch/all.h>
4 -
5 -void dispatch_bgmv(torch::Tensor y, torch::Tensor x, torch::Tensor w,
6 - torch::Tensor indicies, int64_t layer_idx, double scale);
7 -
8 -void dispatch_bgmv_low_level(torch::Tensor y, torch::Tensor x, torch::Tensor w,
9 - torch::Tensor indicies, int64_t layer_idx,
10 - double scale, int64_t h_in, int64_t h_out,
11 - int64_t y_offset);
csrc/punica/torch_bindings.cpp deleted
-18
@@ -1,18 +0,0 @@
1 -#include "registration.h"
2 -#include "punica_ops.h"
3 -
4 -TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) {
5 - m.def(
6 - "dispatch_bgmv(Tensor! y, Tensor x, Tensor w, Tensor indicies, int "
7 - "layer_idx, float scale) -> ()");
8 - m.impl("dispatch_bgmv", torch::kCUDA, &dispatch_bgmv);
9 -
10 - m.def(
11 - "dispatch_bgmv_low_level(Tensor! y, Tensor x, Tensor w,"
12 - "Tensor indicies, int layer_idx,"
13 - "float scale, int h_in, int h_out,"
14 - "int y_offset) -> ()");
15 - m.impl("dispatch_bgmv_low_level", torch::kCUDA, &dispatch_bgmv_low_level);
16 -}
17 -
18 -REGISTER_EXTENSION(TORCH_EXTENSION_NAME)
csrc/punica/type_convert.h deleted
-82
@@ -1,82 +0,0 @@
1 -#ifndef CSRC__PUNICA__TYPE_CONVERT_H__
2 -#define CSRC__PUNICA__TYPE_CONVERT_H__
3 -
4 -#ifndef USE_ROCM
5 -
6 -#include <cuda_bf16.h>
7 -#include <cuda_fp16.h>
8 -
9 -#else
10 -
11 -#include <hip/hip_bf16.h>
12 -#include <hip/hip_fp16.h>
13 -
14 -#define __TYPE_CONVERT__HOST_DEVICE__ __host__ __device__
15 -
16 -typedef __half nv_half;
17 -typedef __hip_bfloat16 nv_bfloat16;
18 -typedef __hip_bfloat162 nv_bfloat162;
19 -
20 -__TYPE_CONVERT__HOST_DEVICE__
21 -inline __hip_bfloat162 make_bfloat162(__hip_bfloat16 val) {
22 - return __hip_bfloat162{val, val};
23 -}
24 -
25 -__TYPE_CONVERT__HOST_DEVICE__
26 -inline __hip_bfloat162 make_bfloat162(__hip_bfloat16 vall, __hip_bfloat16 valr) {
27 - return __hip_bfloat162{vall, valr};
28 -}
29 -
30 -template <typename T_src, typename T_dst>
31 -__TYPE_CONVERT__HOST_DEVICE__
32 -inline T_dst convert_type(T_src val) {
33 - return static_cast<T_dst>(val);
34 -}
35 -
36 -template <>
37 -__TYPE_CONVERT__HOST_DEVICE__
38 -inline float convert_type<__half, float>(__half val) {
39 - return __half2float(val);
40 -}
41 -
42 -template <>
43 -__TYPE_CONVERT__HOST_DEVICE__
44 -inline __half convert_type<float, __half>(float val) {
45 - return __float2half(val);
46 -}
47 -
48 -template <>
49 -__TYPE_CONVERT__HOST_DEVICE__
50 -inline float convert_type<__hip_bfloat16, float>(__hip_bfloat16 val) {
51 - return __bfloat162float(val);
52 -}
53 -
54 -template <>
55 -__TYPE_CONVERT__HOST_DEVICE__
56 -inline __hip_bfloat16 convert_type<float, __hip_bfloat16>(float val) {
57 - return __float2bfloat16(val);
58 -}
59 -
60 -template <typename T>
61 -__TYPE_CONVERT__HOST_DEVICE__
62 -inline T vllm_add(T a, T b) {
63 - return a + b;
64 -}
65 -
66 -template <>
67 -__TYPE_CONVERT__HOST_DEVICE__
68 -inline __half vllm_add<__half>(__half a, __half b) {
69 - return __hadd(a, b);
70 -}
71 -
72 -template <>
73 -__TYPE_CONVERT__HOST_DEVICE__
74 -inline __hip_bfloat16 vllm_add<__hip_bfloat16>(__hip_bfloat16 a, __hip_bfloat16 b) {
75 - return __hadd(a, b);
76 -}
77 -
78 -#undef __TYPE_CONVERT__HOST_DEVICE__
79 -
80 -#endif // USE_ROCM
81 -
82 -#endif // CSRC__PUNICA__TYPE_CONVERT_H__
docs/source/getting_started/installation.rst
-1
@@ -66,7 +66,6 @@ You can also build and install vLLM from source:
66
67 $ git clone https://github.com/vllm-project/vllm.git
68 $ cd vllm
69 - $ # export VLLM_INSTALL_PUNICA_KERNELS=1 # optionally build for multi-LoRA capability
69 $ pip install -e . # This may take 5-10 minutes.
70
71 .. tip::
setup.py
-10
@@ -181,9 +181,6 @@ class cmake_build_ext(build_ext):
181 # match.
182 cmake_args += ['-DVLLM_PYTHON_EXECUTABLE={}'.format(sys.executable)]
183
184 - if _install_punica():
185 - cmake_args += ['-DVLLM_INSTALL_PUNICA_KERNELS=ON']
186 -
184 #
185 # Setup parallelism and build tool
186 #
@@ -274,10 +271,6 @@ def _build_custom_ops() -> bool:
271 return _is_cuda() or _is_hip() or _is_cpu()
272
273
277 -def _install_punica() -> bool:
278 - return envs.VLLM_INSTALL_PUNICA_KERNELS
279 -
280 -
274 def get_hipcc_rocm_version():
275 # Run the hipcc --version command
276 result = subprocess.run(['hipcc', '--version'],
@@ -446,9 +439,6 @@ if _is_cuda() or _is_hip():
439 if _build_custom_ops():
440 ext_modules.append(CMakeExtension(name="vllm._C"))
441
449 - if _install_punica():
450 - ext_modules.append(CMakeExtension(name="vllm._punica_C"))
451 -
442 package_data = {
443 "vllm": ["py.typed", "model_executor/layers/fused_moe/configs/*.json"]
444 }
tests/kernels/test_sampler.py
+30 -18
@@ -1,14 +1,17 @@
1 import gc
2 +from unittest.mock import patch
3
4 import pytest
5 import torch
6 import triton
7 import triton.language as tl
8
8 -from vllm.model_executor.layers.ops.sample import (_uniform_to_exponential,
9 +from vllm.model_executor.layers.ops.sample import (_sample_triton,
10 + _uniform_to_exponential,
11 sample)
12 from vllm.model_executor.sampling_metadata import SamplingTensors
13 from vllm.model_executor.utils import set_random_seed
14 +from vllm.triton_utils.libentry import LibEntry
15 from vllm.triton_utils.sample import (MAX_TRITON_N_COLS,
16 get_num_triton_sampler_splits)
17
@@ -76,15 +79,20 @@ def test_sample_decoding_only(random_sampling, max_best_of,
79 seeds = torch.randint(1,
80 torch.iinfo(torch.long).max, (n_splits, bs),
81 device="cuda").mul_(random_sampling_mask)
79 - sampled_tokens, sampled_logprobs, sampled_modified_probs = sample(
80 - probs=probs,
81 - logprobs=logprobs,
82 - sample_indices=sample_indices,
83 - seeds=seeds,
84 - max_best_of=max_best_of,
85 - modify_greedy_probs=modify_greedy_probs,
86 - save_logprobs=save_logprobs,
87 - _save_modified_probs=True)
82 + #The current _sample_triton does not utilize the
83 + # libentry decoration. The purpose of adding this patch is to test
84 + # the correctness of libentry.
85 + with patch("vllm.model_executor.layers.ops.sample._sample_triton",
86 + LibEntry(_sample_triton)):
87 + sampled_tokens, sampled_logprobs, sampled_modified_probs = sample(
88 + probs=probs,
89 + logprobs=logprobs,
90 + sample_indices=sample_indices,
91 + seeds=seeds,
92 + max_best_of=max_best_of,
93 + modify_greedy_probs=modify_greedy_probs,
94 + save_logprobs=save_logprobs,
95 + _save_modified_probs=True)
96 assert sampled_tokens.shape == (bs, max_best_of)
97 for i in range(bs):
98 assert torch.all(sampled_tokens[i] == i * (vocab_size // bs))
@@ -130,6 +138,7 @@ def test_sample_decoding_only(random_sampling, max_best_of,
138 [SINGLE_SPLIT_VOCAB_SIZE, MULTI_SPLIT_VOCAB_SIZE])
139 def test_sample_prompt_logprobs(random_sampling, max_best_of,
140 modify_greedy_probs, seed, vocab_size):
141 +
142 set_random_seed(seed)
143 prompt_sizes = [16, 32, 64, 128] * 2
144 samples = 8
@@ -157,14 +166,17 @@ def test_sample_prompt_logprobs(random_sampling, max_best_of,
166 seeds = torch.randint(1,
167 torch.iinfo(torch.long).max, (n_splits, samples),
168 device="cuda").mul_(random_sampling_mask)
160 - sampled_tokens, sampled_logprobs, _ = sample(
161 - probs=probs,
162 - logprobs=logprobs,
163 - sample_indices=sample_indices,
164 - seeds=seeds,
165 - max_best_of=max_best_of,
166 - modify_greedy_probs=modify_greedy_probs,
167 - save_logprobs=True)
169 + #ditto
170 + with patch("vllm.model_executor.layers.ops.sample._sample_triton",
171 + LibEntry(_sample_triton)):
172 + sampled_tokens, sampled_logprobs, _ = sample(
173 + probs=probs,
174 + logprobs=logprobs,
175 + sample_indices=sample_indices,
176 + seeds=seeds,
177 + max_best_of=max_best_of,
178 + modify_greedy_probs=modify_greedy_probs,
179 + save_logprobs=True)
180 assert sampled_tokens.shape == (samples, max_best_of)
181 assert sampled_logprobs.shape == (samples, max_best_of)
182 for i, t in enumerate(sample_indices):
tests/lora/test_gemma.py
+1 -1
@@ -37,7 +37,7 @@ def test_gemma_lora(gemma_lora_files):
37 expected_lora_output = [
38 "more important than knowledge.\nAuthor: Albert Einstein\n",
39 "everyone else is already taken.\nAuthor: Oscar Wilde\n",
40 - "so little time\nAuthor: Frank Zappa\n",
40 + "so little time.\nAuthor: Frank Zappa\n",
41 ]
42
43 output1 = do_sample(llm, gemma_lora_files, lora_id=1)
tests/lora/test_layers.py
+81 -59
@@ -26,7 +26,8 @@ from vllm.lora.layers import (BaseLayerWithLoRA, ColumnParallelLinearWithLoRA,
26 VocabParallelEmbeddingWithLoRA)
27 # yapf: enable
28 from vllm.lora.models import (LongContextLoRAContext, LoRALayerWeights,
29 - PackedLoRALayerWeights, convert_mapping)
29 + PackedLoRALayerWeights)
30 +from vllm.lora.punica import PunicaWrapper
31 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
32 MergedColumnParallelLinear,
33 QKVParallelLinear,
@@ -47,6 +48,9 @@ TOLERANCES = {
48 CUDA_DEVICES = [
49 f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
50 ]
51 +# We will launch different triton kernels between the prefill and decode
52 +# stages, so we need to verify this. prefill stage(True) or decode stage(False)
53 +STAGES = [True, False]
54
55
56 def get_random_id_to_index(num_loras: int,
@@ -182,10 +186,12 @@ def create_random_inputs(
186 @pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
187 @pytest.mark.parametrize("device", CUDA_DEVICES)
188 @pytest.mark.parametrize("vocab_size", [512, 32000, 64000, 128000])
185 -def test_embeddings(dist_init, num_loras, device, vocab_size) -> None:
189 +@pytest.mark.parametrize("stage", STAGES)
190 +def test_embeddings(dist_init, num_loras, device, vocab_size, stage) -> None:
191
192 torch.set_default_device(device)
193 max_loras = 8
194 + punica_wrapper = PunicaWrapper(8192, 256, device)
195 lora_config = LoRAConfig(max_loras=max_loras,
196 max_lora_rank=8,
197 lora_dtype=torch.float16)
@@ -204,7 +210,7 @@ def test_embeddings(dist_init, num_loras, device, vocab_size) -> None:
210
211 id_to_index = get_random_id_to_index(num_loras, max_loras)
212 embedding, lora_embedding = create_random_embedding_layer()
207 -
213 + lora_embedding.set_mapping(punica_wrapper)
214 lora_dict, _ = populate_loras(
215 id_to_index,
216 layer=lora_embedding,
@@ -217,12 +223,12 @@ def test_embeddings(dist_init, num_loras, device, vocab_size) -> None:
223 input_size=(200, ),
224 input_range=(1, vocab_size),
225 )
220 - lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
221 -
222 - mapping_info = convert_mapping(lora_mapping, id_to_index, max_loras,
226 + lora_mapping = LoRAMapping(index_mapping,
227 + prompt_mapping,
228 + is_prefill=stage)
229 + punica_wrapper.update_metadata(lora_mapping, id_to_index, max_loras,
230 vocab_size,
231 lora_config.lora_extra_vocab_size)
225 - lora_embedding.set_mapping(*mapping_info)
232
233 lora_result = lora_embedding(torch.cat(inputs))
234
@@ -255,12 +261,12 @@ def test_embeddings(dist_init, num_loras, device, vocab_size) -> None:
261 input_size=(200, ),
262 input_range=(1, vocab_size),
263 )
258 - lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
259 -
260 - mapping_info = convert_mapping(lora_mapping, id_to_index, max_loras,
264 + lora_mapping = LoRAMapping(index_mapping,
265 + prompt_mapping,
266 + is_prefill=stage)
267 + punica_wrapper.update_metadata(lora_mapping, id_to_index, max_loras,
268 vocab_size,
269 lora_config.lora_extra_vocab_size)
263 - lora_embedding.set_mapping(*mapping_info, )
270
271 lora_result = lora_embedding(torch.cat(inputs))
272 expected_result = embedding(torch.cat(inputs))
@@ -278,11 +284,13 @@ def test_embeddings(dist_init, num_loras, device, vocab_size) -> None:
284 @pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
285 @pytest.mark.parametrize("device", CUDA_DEVICES)
286 @pytest.mark.parametrize("vocab_size", [512, 32000, 64000, 128000])
287 +@pytest.mark.parametrize("stage", STAGES)
288 def test_embeddings_with_new_embeddings(dist_init, num_loras, device,
282 - vocab_size) -> None:
289 + vocab_size, stage) -> None:
290
291 torch.set_default_device(device)
292 max_loras = 8
293 + punica_wrapper = PunicaWrapper(8192, 256, device)
294 lora_config = LoRAConfig(max_loras=max_loras,
295 max_lora_rank=8,
296 lora_dtype=torch.float16)
@@ -318,6 +326,7 @@ def test_embeddings_with_new_embeddings(dist_init, num_loras, device,
326 generate_embeddings_tensor=256,
327 )
328
329 + lora_embedding.set_mapping(punica_wrapper)
330 # All embeddings tensors have the same shape.
331 embeddings_tensors = [
332 lora_dict[id].embeddings_tensor for id in sorted(lora_dict.keys())
@@ -334,8 +343,12 @@ def test_embeddings_with_new_embeddings(dist_init, num_loras, device,
343 input_size=(200, ),
344 input_range=(1, vocab_size),
345 )
337 - lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
338 -
346 + lora_mapping = LoRAMapping(index_mapping,
347 + prompt_mapping,
348 + is_prefill=stage)
349 + punica_wrapper.update_metadata(lora_mapping, id_to_index, max_loras,
350 + vocab_size,
351 + lora_config.lora_extra_vocab_size)
352 original_inputs = deepcopy(inputs)
353
354 # Force some of the inputs to be in the extended embeddings range
@@ -349,11 +362,6 @@ def test_embeddings_with_new_embeddings(dist_init, num_loras, device,
362 (embedding_id + 1) * embeddings_tensor_len - 1)
363 original_input_[-2] = vocab_size + embeddings_tensor_len - 1
364
352 - mapping_info = convert_mapping(lora_mapping, id_to_index, max_loras,
353 - vocab_size,
354 - lora_config.lora_extra_vocab_size)
355 - lora_embedding.set_mapping(*mapping_info, )
356 -
365 expanded_embedding.weight[vocab_size:vocab_size +
366 (embeddings_tensor_len *
367 max_loras)] = torch.cat(embeddings_tensors)
@@ -390,15 +398,13 @@ def test_embeddings_with_new_embeddings(dist_init, num_loras, device,
398 input_size=(200, ),
399 input_range=(1, vocab_size),
400 )
393 - lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
394 -
401 original_inputs = deepcopy(inputs)
396 -
397 - mapping_info = convert_mapping(lora_mapping, id_to_index, max_loras,
402 + lora_mapping = LoRAMapping(index_mapping,
403 + prompt_mapping,
404 + is_prefill=stage)
405 + punica_wrapper.update_metadata(lora_mapping, id_to_index, max_loras,
406 vocab_size,
407 lora_config.lora_extra_vocab_size)
400 - lora_embedding.set_mapping(*mapping_info, )
401 -
408 lora_result = lora_embedding(torch.cat(original_inputs))
409 expected_result = expanded_embedding(torch.cat(inputs))
410
@@ -413,11 +419,13 @@ def test_embeddings_with_new_embeddings(dist_init, num_loras, device,
419 @pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
420 @pytest.mark.parametrize("device", CUDA_DEVICES)
421 @pytest.mark.parametrize("vocab_size", [512, 32000, 64000, 128000])
416 -def test_lm_head_logits_processor(dist_init, num_loras, device,
417 - vocab_size) -> None:
422 +@pytest.mark.parametrize("stage", STAGES)
423 +def test_lm_head_logits_processor(dist_init, num_loras, device, vocab_size,
424 + stage) -> None:
425
426 torch.set_default_device(device)
427 max_loras = 8
428 + punica_wrapper = PunicaWrapper(8192, 256, device)
429 lora_config = LoRAConfig(max_loras=max_loras,
430 max_lora_rank=8,
431 lora_dtype=torch.float16)
@@ -443,7 +451,7 @@ def test_lm_head_logits_processor(dist_init, num_loras, device,
451
452 id_to_index = get_random_id_to_index(num_loras, max_loras)
453 linear, logits_processor, lora_logits_processor = _pretest()
446 -
454 + lora_logits_processor.set_mapping(punica_wrapper)
455 # NOTE: all the generated loras share the same embeddings tensor.
456 lora_dict, _ = populate_loras(
457 id_to_index,
@@ -461,17 +469,17 @@ def test_lm_head_logits_processor(dist_init, num_loras, device,
469 input_range=(0, 1),
470 input_type=torch.float16,
471 )
464 - lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
465 -
466 - input_ = torch.rand(20, 1024)
467 - mapping_info = convert_mapping(
472 + lora_mapping = LoRAMapping(index_mapping,
473 + prompt_mapping,
474 + is_prefill=stage)
475 + punica_wrapper.update_metadata(
476 lora_mapping,
477 id_to_index,
478 max_loras,
479 vocab_size,
480 lora_config.lora_extra_vocab_size,
481 )
474 - lora_logits_processor.set_mapping(*mapping_info, )
482 + input_ = torch.rand(20, 1024)
483
484 lora_result = lora_logits_processor._get_logits(
485 hidden_states=torch.cat(inputs),
@@ -510,12 +518,16 @@ def test_lm_head_logits_processor(dist_init, num_loras, device,
518 input_range=(0, 1),
519 input_type=torch.float16,
520 )
513 - lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
514 -
515 - mapping_info = convert_mapping(lora_mapping, id_to_index, max_loras,
516 - vocab_size,
517 - lora_config.lora_extra_vocab_size)
518 - lora_logits_processor.set_mapping(*mapping_info, )
521 + lora_mapping = LoRAMapping(index_mapping,
522 + prompt_mapping,
523 + is_prefill=stage)
524 + punica_wrapper.update_metadata(
525 + lora_mapping,
526 + id_to_index,
527 + max_loras,
528 + vocab_size,
529 + lora_config.lora_extra_vocab_size,
530 + )
531
532 lora_result = lora_logits_processor._get_logits(
533 hidden_states=torch.cat(inputs),
@@ -538,10 +550,12 @@ def test_lm_head_logits_processor(dist_init, num_loras, device,
550 @pytest.mark.parametrize("orientation", ["row", "column"])
551 @pytest.mark.parametrize("fully_shard", [True, False])
552 @pytest.mark.parametrize("device", CUDA_DEVICES)
553 +@pytest.mark.parametrize("stage", STAGES)
554 def test_linear_parallel(dist_init, num_loras, orientation, fully_shard,
542 - device) -> None:
555 + device, stage) -> None:
556
557 torch.set_default_device(device)
558 + punica_wrapper = PunicaWrapper(8192, 256, device)
559 max_loras = 8
560 lora_config = LoRAConfig(max_loras=max_loras,
561 max_lora_rank=8,
@@ -575,7 +589,7 @@ def test_linear_parallel(dist_init, num_loras, orientation, fully_shard,
589
590 id_to_index = get_random_id_to_index(num_loras, max_loras)
591 linear, lora_linear = create_random_linear_parallel_layer()
578 -
592 + lora_linear.set_mapping(punica_wrapper)
593 lora_dict, _ = populate_loras(
594 id_to_index,
595 layer=lora_linear,
@@ -589,16 +603,16 @@ def test_linear_parallel(dist_init, num_loras, orientation, fully_shard,
603 input_range=(0, 1),
604 input_type=torch.float16,
605 )
592 - lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
593 -
594 - mapping_info = convert_mapping(
606 + lora_mapping = LoRAMapping(index_mapping,
607 + prompt_mapping,
608 + is_prefill=stage)
609 + punica_wrapper.update_metadata(
610 lora_mapping,
611 id_to_index,
612 max_loras,
613 512,
614 lora_config.lora_extra_vocab_size,
615 )
601 - lora_linear.set_mapping(*mapping_info, )
616
617 lora_result = lora_linear(torch.cat(inputs))[0]
618
@@ -628,11 +642,12 @@ def test_linear_parallel(dist_init, num_loras, orientation, fully_shard,
642 input_range=(0, 1),
643 input_type=torch.float16,
644 )
631 - lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
645 + lora_mapping = LoRAMapping(index_mapping,
646 + prompt_mapping,
647 + is_prefill=stage)
648
633 - mapping_info = convert_mapping(lora_mapping, id_to_index, max_loras,
649 + punica_wrapper.update_metadata(lora_mapping, id_to_index, max_loras,
650 512, lora_config.lora_extra_vocab_size)
635 - lora_linear.set_mapping(*mapping_info, )
651
652 lora_result = lora_linear(torch.cat(inputs))[0]
653 expected_result = linear(torch.cat(inputs))[0]
@@ -649,10 +664,12 @@ def test_linear_parallel(dist_init, num_loras, orientation, fully_shard,
664 @pytest.mark.parametrize("repeats", [1, 2, 3])
665 @pytest.mark.parametrize("fully_shard", [True, False])
666 @pytest.mark.parametrize("device", CUDA_DEVICES)
667 +@pytest.mark.parametrize("stage", STAGES)
668 def test_column_parallel_packed(dist_init, num_loras, repeats, fully_shard,
653 - device) -> None:
669 + device, stage) -> None:
670
671 torch.set_default_device(device)
672 + punica_wrapper = PunicaWrapper(8192, 256, device)
673 max_loras = 8
674 lora_config = LoRAConfig(max_loras=max_loras,
675 max_lora_rank=8,
@@ -707,7 +724,7 @@ def test_column_parallel_packed(dist_init, num_loras, repeats, fully_shard,
724 id_to_index = get_random_id_to_index(num_loras, max_loras)
725
726 linear, lora_linear = create_column_parallel_packed_layer()
710 -
727 + lora_linear.set_mapping(punica_wrapper)
728 lora_dict, sublora_dict = populate_loras(
729 id_to_index,
730 layer=lora_linear,
@@ -722,16 +739,17 @@ def test_column_parallel_packed(dist_init, num_loras, repeats, fully_shard,
739 input_range=(0, 1),
740 input_type=torch.float16,
741 )
725 - lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
742 + lora_mapping = LoRAMapping(index_mapping,
743 + prompt_mapping,
744 + is_prefill=stage)
745
727 - mapping_info = convert_mapping(
746 + punica_wrapper.update_metadata(
747 lora_mapping,
748 id_to_index,
749 max_loras,
750 512,
751 lora_config.lora_extra_vocab_size,
752 )
734 - lora_linear.set_mapping(*mapping_info)
753
754 lora_result = lora_linear(torch.cat(inputs))[0]
755
@@ -762,16 +780,18 @@ def test_column_parallel_packed(dist_init, num_loras, repeats, fully_shard,
780 input_range=(0, 1),
781 input_type=torch.float16,
782 )
765 - lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
783 + lora_mapping = LoRAMapping(index_mapping,
784 + prompt_mapping,
785 + is_prefill=stage)
786
767 - mapping_info = convert_mapping(
787 + punica_wrapper.update_metadata(
788 lora_mapping,
789 id_to_index,
790 max_loras,
791 512,
792 lora_config.lora_extra_vocab_size,
793 )
774 - lora_linear.set_mapping(*mapping_info)
794 + # lora_linear.set_mapping(*mapping_info)
795
796 lora_result = lora_linear(torch.cat(inputs))[0]
797 expected_result = linear(torch.cat(inputs))[0]
@@ -803,7 +823,7 @@ def test_rotary_embedding_long_context(dist_init, num_loras, device,
823 if torch.cuda.is_available():
824 torch.cuda.manual_seed(seed)
825 torch.set_default_device(device)
806 -
826 + punica_wrapper = PunicaWrapper(8192, 256, device)
827 max_loras = 8
828 lora_config = LoRAConfig(max_loras=max_loras,
829 max_lora_rank=8,
@@ -825,6 +845,7 @@ def test_rotary_embedding_long_context(dist_init, num_loras, device,
845 is_neox_style,
846 )
847 lora_rope = LinearScalingRotaryEmbeddingWithLora(rope)
848 + lora_rope.set_mapping(punica_wrapper)
849 lora_rope.create_lora_weights(max_loras, lora_config)
850 linear_rope = get_rope(head_size, rotary_dim, max_position, base,
851 is_neox_style, {
@@ -840,6 +861,7 @@ def test_rotary_embedding_long_context(dist_init, num_loras, device,
861 input_range=(0, lora_config.lora_extra_vocab_size),
862 input_type=torch.float16,
863 )
864 +
865 lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
866 long_lora_context = LongContextLoRAContext(list(scaling_factors),
867 rotary_dim)
@@ -854,7 +876,7 @@ def test_rotary_embedding_long_context(dist_init, num_loras, device,
876 for i in range(len(scaling_factors)):
877 long_lora_context.offsets_by_lora_id[i] = scaling_factor_to_offset.get(
878 scaling_factors[i], 0)
857 - mapping_info = convert_mapping(
879 + punica_wrapper.update_metadata(
880 lora_mapping,
881 id_to_index,
882 max_loras,
@@ -862,7 +884,7 @@ def test_rotary_embedding_long_context(dist_init, num_loras, device,
884 lora_config.lora_extra_vocab_size,
885 long_lora_context=long_lora_context,
886 )
865 - lora_rope.set_mapping(*mapping_info)
887 + # lora_rope.set_mapping(*mapping_info)
888
889 positions = torch.randint(0, max_position, (batch_size, seq_len))
890 query = torch.randn(batch_size,
tests/lora/test_lora.py deleted
-224
@@ -1,224 +0,0 @@
1 -import pytest
2 -import torch
3 -
4 -from vllm.lora.layers import _apply_lora, _apply_lora_packed_nslice
5 -
6 -from .utils import DummyLoRAManager
7 -
8 -TENSOR_SIZES = [128, 1024, 2048, 4096, 8192, 11008, 11008 // 2, 11008 // 4]
9 -QKV_TENSOR_SIZES = [
10 - (8192, 1024, 1024),
11 - (8192 // 8, 1024 // 8, 1024 // 8),
12 - (4096, 4096, 4096),
13 - (4096 // 2, 4096 // 2, 4096 // 2),
14 -]
15 -BATCH_SIZES = [8, 32, 256]
16 -RANKS = [8]
17 -DTYPES = [torch.float16]
18 -TOLERANCES = {
19 - torch.float16: (5e-3, 5e-3),
20 - torch.bfloat16: (3e-2, 2e-2),
21 -}
22 -
23 -
24 -@pytest.mark.parametrize("m", TENSOR_SIZES)
25 -@pytest.mark.parametrize("n", TENSOR_SIZES)
26 -@pytest.mark.parametrize("k", BATCH_SIZES)
27 -@pytest.mark.parametrize("rank", RANKS)
28 -@pytest.mark.parametrize("dtype", DTYPES)
29 -def test_apply_lora(m, n, k, rank, dtype) -> None:
30 - manager = DummyLoRAManager()
31 -
32 - module_name = "module"
33 - weight = torch.rand([m, n], device="cuda", dtype=dtype)
34 -
35 - manager.init_random_lora(module_name, weight, rank=rank)
36 - lora = manager.get_module_lora(module_name)
37 -
38 - input = torch.rand(k, n, device="cuda", dtype=dtype)
39 - expected = input @ lora.lora_a @ lora.lora_b * lora.scaling
40 -
41 - lora_a_stack = torch.zeros(8,
42 - 1,
43 - lora.lora_a.shape[1],
44 - lora.lora_a.shape[0],
45 - device="cuda",
46 - dtype=dtype)
47 - lora_b_stack = torch.zeros(8,
48 - 1,
49 - lora.lora_b.shape[1],
50 - lora.lora_b.shape[0],
51 - device="cuda",
52 - dtype=dtype)
53 - for i in range(lora_a_stack.shape[0]):
54 - lora_a_stack[i][0] = lora.lora_a.T
55 - lora_b_stack[i][0] = (lora.lora_b * lora.scaling).T
56 -
57 - output = torch.zeros(k, m, device="cuda", dtype=dtype)
58 - _apply_lora(
59 - input, lora_a_stack, lora_b_stack,
60 - torch.randint(0, lora_a_stack.shape[0], (len(input), ), device="cuda"),
61 - output)
62 -
63 - rtol, atol = TOLERANCES[dtype]
64 - assert torch.allclose(expected, output, rtol=rtol, atol=atol)
65 -
66 - output[:] = 0
67 - _apply_lora(input, lora_a_stack, lora_b_stack,
68 - torch.full((len(input), ), -1, device="cuda"), output)
69 - assert torch.allclose(torch.zeros_like(output), output)
70 -
71 - manager.reset_lora()
72 -
73 -
74 -@pytest.mark.parametrize("m", TENSOR_SIZES)
75 -@pytest.mark.parametrize("n", TENSOR_SIZES)
76 -@pytest.mark.parametrize("k", BATCH_SIZES)
77 -@pytest.mark.parametrize("rank", RANKS)
78 -@pytest.mark.parametrize("dtype", DTYPES)
79 -def test_apply_lora_packed_2slice(m, n, k, rank, dtype) -> None:
80 - if m % 2 != 0:
81 - pytest.skip("m must be divisible by 2")
82 - if m // 2 not in TENSOR_SIZES:
83 - pytest.skip("m//2 must be in TENSOR_SIZES")
84 -
85 - manager = DummyLoRAManager()
86 -
87 - module_name = "module"
88 - weight = torch.rand([m // 2, n], device="cuda", dtype=dtype)
89 -
90 - manager.init_random_lora(module_name + "1", weight, rank=rank)
91 - lora_1 = manager.get_module_lora(module_name + "1")
92 - manager.init_random_lora(module_name + "2", weight, rank=rank)
93 - lora_2 = manager.get_module_lora(module_name + "2")
94 -
95 - input = torch.rand(k, n, device="cuda", dtype=dtype)
96 - expected = torch.cat([
97 - input @ lora_1.lora_a @ lora_1.lora_b * lora_1.scaling,
98 - input @ lora_2.lora_a @ lora_2.lora_b * lora_2.scaling
99 - ],
100 - dim=1)
101 -
102 - lora_a_stacks = [
103 - torch.zeros(8,
104 - 1,
105 - lora_1.lora_a.shape[1],
106 - lora_1.lora_a.shape[0],
107 - device="cuda",
108 - dtype=dtype) for i in range(2)
109 - ]
110 - lora_b_stacks = [
111 - torch.zeros(8,
112 - 1,
113 - lora_1.lora_b.shape[1],
114 - lora_1.lora_b.shape[0],
115 - device="cuda",
116 - dtype=dtype) for i in range(2)
117 - ]
118 - for i in range(lora_a_stacks[0].shape[0]):
119 - lora_a_stacks[0][i][0] = lora_1.lora_a.T
120 - lora_b_stacks[0][i][0] = (lora_1.lora_b * lora_1.scaling).T
121 - lora_a_stacks[1][i][0] = lora_2.lora_a.T
122 - lora_b_stacks[1][i][0] = (lora_2.lora_b * lora_2.scaling).T
123 -
124 - output = torch.zeros(k, m, device="cuda", dtype=dtype)
125 - _apply_lora_packed_nslice(
126 - input, lora_a_stacks, lora_b_stacks,
127 - torch.randint(0,
128 - lora_a_stacks[0].shape[0], (len(input), ),
129 - device="cuda"), output, (m // 2, m // 2))
130 -
131 - rtol, atol = TOLERANCES[dtype]
132 - assert torch.allclose(expected, output, rtol=rtol, atol=atol)
133 -
134 - output[:] = 0
135 - _apply_lora_packed_nslice(input, lora_a_stacks, lora_b_stacks,
136 - torch.full((len(input), ), -1, device="cuda"),
137 - output, (m // 2, m // 2))
138 - assert torch.allclose(torch.zeros_like(output), output)
139 -
140 - manager.reset_lora()
141 -
142 -
143 -@pytest.mark.parametrize("qkv", QKV_TENSOR_SIZES)
144 -@pytest.mark.parametrize("n", TENSOR_SIZES)
145 -@pytest.mark.parametrize("k", BATCH_SIZES)
146 -@pytest.mark.parametrize("rank", RANKS)
147 -@pytest.mark.parametrize("dtype", DTYPES)
148 -def test_apply_lora_packed_3slice(qkv, n, k, rank, dtype) -> None:
149 - manager = DummyLoRAManager()
150 -
151 - module_name = "module"
152 - weight_q = torch.empty(qkv[0], n, device="cuda", dtype=dtype)
153 - weight_kv = torch.empty(qkv[1], n, device="cuda", dtype=dtype)
154 -
155 - manager.init_random_lora(module_name + "q", weight_q, rank=rank)
156 - lora_q = manager.get_module_lora(module_name + "q")
157 - manager.init_random_lora(module_name + "k", weight_kv, rank=rank)
158 - lora_k = manager.get_module_lora(module_name + "k")
159 - manager.init_random_lora(module_name + "v", weight_kv, rank=rank)
160 - lora_v = manager.get_module_lora(module_name + "v")
161 -
162 - input = torch.rand(k, n, device="cuda", dtype=dtype)
163 - expected = torch.cat([
164 - input @ lora_q.lora_a @ lora_q.lora_b * lora_q.scaling,
165 - input @ lora_k.lora_a @ lora_k.lora_b * lora_k.scaling,
166 - input @ lora_v.lora_a @ lora_v.lora_b * lora_v.scaling
167 - ],
168 - dim=1)
169 -
170 - lora_a_stacks = [
171 - torch.zeros(8,
172 - 1,
173 - lora_q.lora_a.shape[1],
174 - lora_q.lora_a.shape[0],
175 - device="cuda",
176 - dtype=dtype)
177 - ] + [
178 - torch.zeros(8,
179 - 1,
180 - lora_k.lora_a.shape[1],
181 - lora_k.lora_a.shape[0],
182 - device="cuda",
183 - dtype=dtype) for i in range(2)
184 - ]
185 - lora_b_stacks = [
186 - torch.zeros(8,
187 - 1,
188 - lora_q.lora_b.shape[1],
189 - lora_q.lora_b.shape[0],
190 - device="cuda",
191 - dtype=dtype)
192 - ] + [
193 - torch.zeros(8,
194 - 1,
195 - lora_k.lora_b.shape[1],
196 - lora_k.lora_b.shape[0],
197 - device="cuda",
198 - dtype=dtype) for i in range(2)
199 - ]
200 - for i in range(lora_a_stacks[0].shape[0]):
201 - lora_a_stacks[0][i][0] = lora_q.lora_a.T
202 - lora_b_stacks[0][i][0] = (lora_q.lora_b * lora_q.scaling).T
203 - lora_a_stacks[1][i][0] = lora_k.lora_a.T
204 - lora_b_stacks[1][i][0] = (lora_k.lora_b * lora_k.scaling).T
205 - lora_a_stacks[2][i][0] = lora_v.lora_a.T
206 - lora_b_stacks[2][i][0] = (lora_v.lora_b * lora_v.scaling).T
207 -
208 - output = torch.zeros(k, sum(qkv), device="cuda", dtype=dtype)
209 - _apply_lora_packed_nslice(
210 - input, lora_a_stacks, lora_b_stacks,
211 - torch.randint(0,
212 - lora_a_stacks[0].shape[0], (len(input), ),
213 - device="cuda"), output, (qkv[0], qkv[1], qkv[2]))
214 -
215 - rtol, atol = TOLERANCES[dtype]
216 - assert torch.allclose(expected, output, rtol=rtol, atol=atol)
217 -
218 - output[:] = 0
219 - _apply_lora_packed_nslice(input, lora_a_stacks, lora_b_stacks,
220 - torch.full((len(input), ), -1, device="cuda"),
221 - output, (qkv[0], qkv[1], qkv[2]))
222 - assert torch.allclose(torch.zeros_like(output), output)
223 -
224 - manager.reset_lora()
tests/lora/test_punica.py deleted
-258
@@ -1,258 +0,0 @@
1 -# Based on code from https://github.com/punica-ai/punica
2 -
3 -import pytest
4 -import torch
5 -
6 -import vllm.lora.punica as punica
7 -
8 -
9 -def assert_close(a, b):
10 - rtol, atol = {
11 - torch.float16: (5e-3, 5e-3),
12 - torch.bfloat16: (3e-2, 2e-2),
13 - torch.float32: (None, None),
14 - }[a.dtype]
15 - torch.testing.assert_close(a, b, rtol=rtol, atol=atol)
16 -
17 -
18 -def _lora_ref_impl(
19 - y_final: torch.Tensor,
20 - x: torch.Tensor,
21 - wa_T_all: torch.Tensor,
22 - wb_T_all: torch.Tensor,
23 - indicies: torch.LongTensor,
24 - layer_idx: int,
25 - scale: float,
26 -):
27 - y_stage_1 = torch.empty(
28 - (x.size(0), wa_T_all.size(-2)),
29 - dtype=torch.float32,
30 - device=x.device,
31 - )
32 - bs = x.shape[0]
33 - s = torch.tensor(scale, dtype=torch.float32, device=x.device)
34 - for i, lora_idx in zip(range(bs), indicies.cpu().tolist()):
35 - xi = x[i].unsqueeze(0).to(torch.float32)
36 - wa = wa_T_all[lora_idx, layer_idx].transpose(-1, -2).to(torch.float32)
37 - if wb_T_all is not None:
38 - wb = wb_T_all[lora_idx, layer_idx].transpose(-1,
39 - -2).to(torch.float32)
40 -
41 - tmp = xi @ wa
42 - y_stage_1[i] = tmp.squeeze(0)
43 - y_final[i] += ((tmp @ wb).squeeze(0) *
44 - s if wb_T_all is not None else y_stage_1[i])
45 - return y_final, y_stage_1
46 -
47 -
48 -H1 = H2 = [
49 - 128,
50 - 256,
51 - 512,
52 - 896,
53 - 1024,
54 - 1152,
55 - 1216,
56 - 1280,
57 - 1536,
58 - 1664,
59 - 2048,
60 - 2240,
61 - 2304,
62 - 2368,
63 - 2432,
64 - 2560,
65 - 2752,
66 - 3072,
67 - 3328,
68 - 3456,
69 - 3584,
70 - 3712,
71 - 4096,
72 - 4480,
73 - 4608,
74 - 4736,
75 - 4864,
76 - 5120,
77 - 5504,
78 - 5632,
79 - 5888,
80 - 6144,
81 - 6400,
82 - 6848,
83 - 6912,
84 - 7168,
85 - 7424,
86 - 8192,
87 - 8960,
88 - 9216,
89 - 9472,
90 - 10240,
91 - 11008,
92 - 11264,
93 - 13824,
94 - 14336,
95 - 14784,
96 - 14848,
97 - 15360,
98 - 18944,
99 - 22016,
100 - 22528,
101 - 24576,
102 - 27392,
103 - 27648,
104 - 29568,
105 - 29696,
106 - 32000,
107 - 32256,
108 - 32512,
109 - 32768,
110 - 33024,
111 - 36864,
112 - 43264,
113 - 49152,
114 - 49408,
115 - 60544,
116 - 60672,
117 - 64000,
118 - 64256,
119 - 102400,
120 - 102656,
121 - 128000,
122 - 128256,
123 -]
124 -H2 = [64] + H2
125 -R = [1, 2, 4]
126 -SEED = [0xabcdabcd987]
127 -CUDA_DEVICES = [
128 - f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
129 -]
130 -
131 -
132 -@pytest.mark.parametrize("dtype_str", ["float16", "bfloat16"])
133 -@pytest.mark.parametrize("h1", H1)
134 -@pytest.mark.parametrize("r", R)
135 -@pytest.mark.parametrize("seed", SEED)
136 -@torch.inference_mode()
137 -def test_lora_a_extra_shapes(dtype_str, h1, r, seed):
138 - torch.manual_seed(seed)
139 - num_loras = 4
140 - num_layers = 1
141 - bs = 32
142 - dtype = getattr(torch, dtype_str)
143 - device = torch.device("cuda")
144 -
145 - wa_T_all = torch.randn(num_loras,
146 - num_layers,
147 - r,
148 - h1,
149 - dtype=dtype,
150 - device=device)
151 - indices = torch.randint(num_loras, (bs, ), dtype=torch.long, device=device)
152 -
153 - for layer_idx in range(num_layers):
154 - x = torch.randn(bs, h1, dtype=dtype, device=device)
155 - y = torch.randn(bs, r, dtype=dtype, device=device)
156 -
157 - y_ref = y.clone()
158 - _lora_ref_impl(
159 - y_ref,
160 - x,
161 - wa_T_all,
162 - None,
163 - indices,
164 - layer_idx,
165 - 1.0,
166 - )
167 -
168 - y_our = y.clone()
169 - punica.bgmv(y_our, x, wa_T_all, indices, layer_idx, 1.0)
170 -
171 - assert_close(y_ref, y_our)
172 -
173 -
174 -@pytest.mark.parametrize("dtype_str", ["float16", "bfloat16"])
175 -@pytest.mark.parametrize("h1", H1)
176 -@pytest.mark.parametrize("h2", H2)
177 -@pytest.mark.parametrize("seed", SEED)
178 -@pytest.mark.parametrize("device", CUDA_DEVICES)
179 -@torch.inference_mode()
180 -def test_lora_correctness(dtype_str, h1, h2, seed, device):
181 - torch.manual_seed(seed)
182 - num_loras = 4
183 - num_layers = 1
184 - r = 8
185 - bs = 32
186 - scale = 0.123
187 - dtype = getattr(torch, dtype_str)
188 - torch.set_default_device(device)
189 -
190 - wa_T_all = torch.randn(num_loras, num_layers, r, h1, dtype=dtype)
191 - wb_T_all = torch.randn(num_loras, num_layers, h2, r, dtype=dtype)
192 - indices = torch.randint(num_loras, (bs, ), dtype=torch.long)
193 -
194 - for layer_idx in range(num_layers):
195 - x = torch.randn(bs, h1, dtype=dtype)
196 - y = torch.randn(bs, h2, dtype=dtype)
197 -
198 - y_ref = y.clone()
199 - _lora_ref_impl(y_ref, x, wa_T_all, wb_T_all, indices, layer_idx, scale)
200 -
201 - y_our = y.clone()
202 - punica.add_lora(y_our, x, wa_T_all, wb_T_all, indices, layer_idx,
203 - scale)
204 -
205 - assert_close(y_ref, y_our)
206 -
207 -
208 -@pytest.mark.parametrize("dtype_str", ["float16", "bfloat16"])
209 -@pytest.mark.parametrize("h1", H1)
210 -@pytest.mark.parametrize("h2", H2)
211 -@pytest.mark.parametrize("seed", SEED)
212 -@pytest.mark.parametrize("device", CUDA_DEVICES)
213 -@torch.inference_mode()
214 -def test_lora_correctness_slice(dtype_str, h1, h2, seed, device):
215 - if h2 % 3 != 0 or h2 // 3 not in H1:
216 - pytest.skip("h2 must be divisible by 3 and in supported shapes")
217 - torch.manual_seed(seed)
218 - num_loras = 4
219 - num_layers = 1
220 - r = 8
221 - bs = 32
222 - scale = 0.123
223 - dtype = getattr(torch, dtype_str)
224 - torch.set_default_device(device)
225 -
226 - wa_T_all_0 = torch.randn(num_loras, num_layers, r, h1, dtype=dtype)
227 - wa_T_all_1 = torch.randn(num_loras, num_layers, r, h1, dtype=dtype)
228 - wa_T_all_2 = torch.randn(num_loras, num_layers, r, h1, dtype=dtype)
229 - wb_T_all_0 = torch.randn(num_loras, num_layers, h2 // 3, r, dtype=dtype)
230 - wb_T_all_1 = torch.randn(num_loras, num_layers, h2 // 3, r, dtype=dtype)
231 - wb_T_all_2 = torch.randn(num_loras, num_layers, h2 // 3, r, dtype=dtype)
232 -
233 - indices = torch.randint(num_loras, (bs, ), dtype=torch.long)
234 -
235 - for layer_idx in range(num_layers):
236 - x = torch.randn(bs, h1, dtype=dtype)
237 - y = torch.randn(bs, h2, dtype=dtype)
238 - s = h2 // 3
239 -
240 - y_ref = y.clone()
241 - _lora_ref_impl(y_ref[:, :s], x, wa_T_all_0, wb_T_all_0, indices,
242 - layer_idx, scale)
243 - _lora_ref_impl(y_ref[:, s:s * 2], x, wa_T_all_1, wb_T_all_1, indices,
244 - layer_idx, scale)
245 - _lora_ref_impl(y_ref[:, s * 2:], x, wa_T_all_2, wb_T_all_2, indices,
246 - layer_idx, scale)
247 -
248 - y_our = y.clone()
249 - punica.add_lora_slice(y_our, x, wa_T_all_0, wb_T_all_0, indices,
250 - layer_idx, scale, 0, s)
251 - punica.add_lora_slice(y_our, x, wa_T_all_1, wb_T_all_1, indices,
252 - layer_idx, scale, s, s)
253 - punica.add_lora_slice(y_our, x, wa_T_all_2, wb_T_all_2, indices,
254 - layer_idx, scale, s * 2, s)
255 -
256 - assert_close(y_ref[:, :s], y_our[:, :s])
257 - assert_close(y_ref[:, s:s * 2], y_our[:, s:s * 2])
258 - assert_close(y_ref[:, s * 2:], y_our[:, s * 2:])
tests/lora/test_punica_sizes.py new
+408
@@ -0,0 +1,408 @@
1 +"""
2 +This script is mainly used to tests various hidden_sizes. We have collected the
3 +hidden_sizes included in the LoRA models currently supported by vLLM. It tests
4 +whether the corresponding Triton kernel can run normally when tensor parallelism
5 +is set to [1, 2, 4, 8, 16, 32, 64].
6 +"""
7 +import random
8 +from unittest.mock import patch
9 +
10 +import pytest
11 +import torch
12 +
13 +from vllm.lora.ops.bgmv_expand import bgmv_expand
14 +from vllm.lora.ops.bgmv_expand_slice import bgmv_expand_slice
15 +from vllm.lora.ops.bgmv_shrink import bgmv_shrink
16 +from vllm.lora.ops.sgmv_expand import sgmv_expand
17 +from vllm.lora.ops.sgmv_expand_slice import sgmv_expand_slice
18 +from vllm.lora.ops.sgmv_shrink import sgmv_shrink
19 +from vllm.triton_utils.libentry import LibEntry
20 +
21 +from .utils import (generate_data, generate_data_for_expand_nslices,
22 + ref_torch_groupgemm)
23 +
24 +HIDDEN_SIZES = [
25 + 128,
26 + 256,
27 + 512,
28 + 896,
29 + 1024,
30 + 1152,
31 + 1216,
32 + 1280,
33 + 1536,
34 + 1664,
35 + 2048,
36 + 2240,
37 + 2304,
38 + 2368,
39 + 2432,
40 + 2560,
41 + 2752,
42 + 3072,
43 + 3328,
44 + 3456,
45 + 3584,
46 + 3712,
47 + 4096,
48 + 4480,
49 + 4608,
50 + 4736,
51 + 4864,
52 + 5120,
53 + 5504,
54 + 5632,
55 + 5888,
56 + 6144,
57 + 6400,
58 + 6848,
59 + 6912,
60 + 7168,
61 + 7424,
62 + 8192,
63 + 8960,
64 + 9216,
65 + 9472,
66 + 10240,
67 + 11008,
68 + 11264,
69 + 13824,
70 + 14336,
71 + 14784,
72 + 14848,
73 + 15360,
74 + 18944,
75 + 22016,
76 + 22528,
77 + 24576,
78 + 27392,
79 + 27648,
80 + 29568,
81 + 29696,
82 + 32000,
83 + 32256,
84 + 32512,
85 + 32768,
86 + 33024,
87 + 36864,
88 + 43264,
89 + 49152,
90 + 49408,
91 + 60544,
92 + 60672,
93 + 64000,
94 + 64256,
95 + 102400,
96 + 102656,
97 + 128000,
98 + 128256,
99 +]
100 +#The size of TP
101 +divisibility = [1, 2, 4, 8, 16, 32, 64]
102 +
103 +all_hidden_size = []
104 +for div in divisibility:
105 + for hidden_size in HIDDEN_SIZES:
106 + all_hidden_size.append(hidden_size // div)
107 +
108 +HIDDEN_SIZES = list(set(all_hidden_size))
109 +
110 +BATCHES = [4]
111 +NUM_LORA = [4]
112 +DTYPES = [torch.float16, torch.bfloat16]
113 +MAX_RANKS = [32]
114 +SCALES = [0.5]
115 +SEED = [0]
116 +CUDA_DEVICES = [f"cuda:{0}"]
117 +
118 +
119 +def assert_close(a, b):
120 + rtol, atol = {
121 + torch.float16: (6e-2, 6e-2),
122 + torch.bfloat16: (6e-2, 6e-2),
123 + torch.float32: (1e-2, 1e-2),
124 + }[a.dtype]
125 + torch.testing.assert_close(a, b, rtol=rtol, atol=atol)
126 +
127 +
128 +@pytest.mark.parametrize("batches", BATCHES)
129 +@pytest.mark.parametrize("num_loras", NUM_LORA)
130 +@pytest.mark.parametrize("rank", MAX_RANKS)
131 +@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
132 +@pytest.mark.parametrize("scaling", SCALES)
133 +@pytest.mark.parametrize("dtype", DTYPES)
134 +@pytest.mark.parametrize("op_type", ["shrink", "expand"])
135 +@pytest.mark.parametrize("seed", SEED)
136 +@pytest.mark.parametrize("device", CUDA_DEVICES)
137 +def test_punica_sgmv(
138 + batches: int,
139 + num_loras: int,
140 + rank: int,
141 + hidden_size: int,
142 + scaling: float,
143 + dtype: torch.dtype,
144 + op_type: str,
145 + seed: int,
146 + device: str,
147 +):
148 + random.seed(seed)
149 + torch.set_default_device(device)
150 + torch.random.manual_seed(seed)
151 + if torch.cuda.is_available():
152 + torch.cuda.manual_seed(seed)
153 +
154 + seq_length = 128
155 + (
156 + inputs_tensor,
157 + lora_weights,
158 + our_out_tensor,
159 + ref_out_tensor,
160 + b_seq_start_loc,
161 + lora_indices_tensor,
162 + seq_len_tensor,
163 + indices,
164 + ) = generate_data(
165 + batches,
166 + hidden_size,
167 + num_loras,
168 + rank,
169 + seq_length,
170 + dtype,
171 + op_type,
172 + device,
173 + )
174 + max_seq_length = seq_len_tensor.max()
175 + if isinstance(max_seq_length, tuple):
176 + max_seq_length = max_seq_length[0].item()
177 + else:
178 + max_seq_length = max_seq_length.item()
179 + if op_type == "shrink":
180 + sgmv_shrink(
181 + inputs_tensor,
182 + lora_weights,
183 + our_out_tensor,
184 + b_seq_start_loc,
185 + seq_len_tensor,
186 + lora_indices_tensor,
187 + batches,
188 + max_seq_length,
189 + scaling,
190 + )
191 + else:
192 + sgmv_expand(
193 + inputs_tensor,
194 + lora_weights,
195 + our_out_tensor,
196 + b_seq_start_loc,
197 + seq_len_tensor,
198 + lora_indices_tensor,
199 + batches,
200 + max_seq_length,
201 + add_inputs=True,
202 + )
203 + ref_torch_groupgemm(
204 + ref_out_tensor,
205 + inputs_tensor,
206 + lora_weights,
207 + lora_indices_tensor,
208 + seq_len_tensor,
209 + batches,
210 + scaling if op_type == "shrink" else 1.0,
211 + op_type,
212 + )
213 + if op_type == "shrink":
214 + ref_out_tensor = ref_out_tensor.to(torch.float32)
215 + assert_close(our_out_tensor, ref_out_tensor)
216 +
217 +
218 +@pytest.mark.parametrize("batches", BATCHES)
219 +@pytest.mark.parametrize("num_loras", NUM_LORA)
220 +@pytest.mark.parametrize("rank", MAX_RANKS)
221 +@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
222 +@pytest.mark.parametrize("scaling", SCALES)
223 +@pytest.mark.parametrize("dtype", DTYPES)
224 +@pytest.mark.parametrize("op_type", ["shrink", "expand"])
225 +@pytest.mark.parametrize("seed", SEED)
226 +@pytest.mark.parametrize("device", CUDA_DEVICES)
227 +def test_punica_bgmv(
228 + batches: int,
229 + num_loras: int,
230 + rank: int,
231 + hidden_size: int,
232 + scaling: float,
233 + dtype: torch.dtype,
234 + op_type: str,
235 + seed: int,
236 + device: str,
237 +):
238 + from vllm.lora.ops.bgmv_expand import _bgmv_expand_kernel
239 + from vllm.lora.ops.bgmv_shrink import _bgmv_shrink_kernel
240 +
241 + random.seed(seed)
242 + torch.set_default_device(device)
243 + torch.random.manual_seed(seed)
244 + if torch.cuda.is_available():
245 + torch.cuda.manual_seed(seed)
246 +
247 + seq_length = 1
248 + (
249 + inputs_tensor,
250 + lora_weights,
251 + our_out_tensor,
252 + ref_out_tensor,
253 + b_seq_start_loc,
254 + lora_indices_tensor,
255 + seq_len_tensor,
256 + indices,
257 + ) = generate_data(
258 + batches,
259 + hidden_size,
260 + num_loras,
261 + rank,
262 + seq_length,
263 + dtype,
264 + op_type,
265 + device,
266 + )
267 + if op_type == "shrink":
268 + # The current _bgmv_shrink_kernel does not require the libentry
269 + # decoration. The purpose of adding this patch is to test the
270 + # correctness of libentry.
271 + with patch(
272 + "vllm.lora.ops.bgmv_shrink._bgmv_shrink_kernel",
273 + LibEntry(_bgmv_shrink_kernel),
274 + ):
275 + bgmv_shrink(
276 + inputs_tensor,
277 + lora_weights,
278 + our_out_tensor,
279 + indices,
280 + scaling,
281 + )
282 + else:
283 + # ditto
284 + with patch(
285 + "vllm.lora.ops.bgmv_expand._bgmv_expand_kernel",
286 + LibEntry(_bgmv_expand_kernel),
287 + ):
288 + bgmv_expand(
289 + inputs_tensor,
290 + lora_weights,
291 + our_out_tensor,
292 + indices,
293 + add_inputs=True,
294 + )
295 + ref_torch_groupgemm(
296 + ref_out_tensor,
297 + inputs_tensor,
298 + lora_weights,
299 + lora_indices_tensor,
300 + seq_len_tensor,
301 + batches,
302 + scaling if op_type == "shrink" else 1.0,
303 + op_type,
304 + )
305 + if op_type == "shrink":
306 + ref_out_tensor = ref_out_tensor.to(torch.float32)
307 + assert_close(our_out_tensor, ref_out_tensor)
308 +
309 +
310 +@pytest.mark.parametrize("batches", BATCHES)
311 +@pytest.mark.parametrize("num_loras", NUM_LORA)
312 +@pytest.mark.parametrize("rank", MAX_RANKS)
313 +@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
314 +@pytest.mark.parametrize("nslices", [2, 3])
315 +@pytest.mark.parametrize("dtype", DTYPES)
316 +@pytest.mark.parametrize("op_type", ["sgmv", "bgmv"])
317 +@pytest.mark.parametrize("seed", SEED)
318 +@pytest.mark.parametrize("device", CUDA_DEVICES)
319 +def test_punica_expand_nslices(
320 + batches: int,
321 + num_loras: int,
322 + rank: int,
323 + hidden_size: int,
324 + nslices: int,
325 + dtype: torch.dtype,
326 + op_type: str,
327 + seed: int,
328 + device: str,
329 +):
330 + from vllm.lora.ops.bgmv_expand_slice import _bgmv_expand_slice_kernel
331 +
332 + random.seed(seed)
333 + torch.set_default_device(device)
334 + torch.random.manual_seed(seed)
335 + if torch.cuda.is_available():
336 + torch.cuda.manual_seed(seed)
337 + seq_length = 128 if op_type == "sgmv" else 1
338 + (
339 + inputs_tensor,
340 + lora_weights_lst,
341 + our_outputs,
342 + ref_outputs,
343 + b_seq_start_loc,
344 + lora_indices_tensor,
345 + seq_len_tensor,
346 + indices,
347 + ) = generate_data_for_expand_nslices(
348 + batches,
349 + hidden_size,
350 + num_loras,
351 + rank,
352 + seq_length,
353 + dtype,
354 + nslices,
355 + device,
356 + )
357 + max_seq_length = seq_len_tensor.max()
358 + if isinstance(max_seq_length, tuple):
359 + max_seq_length = max_seq_length[0].item()
360 + else:
361 + max_seq_length = max_seq_length.item()
362 + slice_offset = 0
363 + for index in range(nslices):
364 + lora_weights = lora_weights_lst[index]
365 + if op_type == "sgmv":
366 + sgmv_expand_slice(
367 + inputs_tensor,
368 + lora_weights,
369 + our_outputs,
370 + b_seq_start_loc,
371 + seq_len_tensor,
372 + lora_indices_tensor,
373 + batches,
374 + max_seq_length,
375 + slice_offset,
376 + hidden_size,
377 + add_inputs=True,
378 + )
379 + else:
380 + # The current _bgmv_expand_slice_kernel does not require the
381 + # libentry decoration. The purpose of adding this patch is to test
382 + # the correctness of libentry.
383 + with patch(
384 + "vllm.lora.ops.bgmv_expand_slice._bgmv_expand_slice_kernel",
385 + LibEntry(_bgmv_expand_slice_kernel),
386 + ):
387 + bgmv_expand_slice(
388 + inputs_tensor,
389 + lora_weights,
390 + our_outputs,
391 + indices,
392 + slice_offset,
393 + slice_size=hidden_size,
394 + add_inputs=True,
395 + )
396 + ref_torch_groupgemm(
397 + ref_outputs[:, slice_offset:slice_offset + hidden_size],
398 + inputs_tensor,
399 + lora_weights,
400 + lora_indices_tensor,
401 + seq_len_tensor,
402 + batches,
403 + 1.0,
404 + op_type="expand",
405 + )
406 +
407 + slice_offset += hidden_size
408 + assert_close(our_outputs, ref_outputs)
tests/lora/test_punica_variation.py new
+342
@@ -0,0 +1,342 @@
1 +"""
2 +This script is mainly used to test whether trtion kernels can run normally
3 +under different conditions, including various batches, numbers of LoRA , and
4 +maximum ranks.
5 +"""
6 +import random
7 +from unittest.mock import patch
8 +
9 +import pytest
10 +import torch
11 +
12 +from vllm.lora.ops.bgmv_expand import bgmv_expand
13 +from vllm.lora.ops.bgmv_expand_slice import bgmv_expand_slice
14 +from vllm.lora.ops.bgmv_shrink import bgmv_shrink
15 +from vllm.lora.ops.sgmv_expand import sgmv_expand
16 +from vllm.lora.ops.sgmv_expand_slice import sgmv_expand_slice
17 +from vllm.lora.ops.sgmv_shrink import sgmv_shrink
18 +from vllm.triton_utils.libentry import LibEntry
19 +
20 +from .utils import (generate_data, generate_data_for_expand_nslices,
21 + ref_torch_groupgemm)
22 +
23 +HIDDEN_SIZES = [3424, 4096, 4097]
24 +
25 +BATCHES = [1, 4, 16, 32]
26 +NUM_LORA = [1, 4, 8, 16, 32, 64, 128]
27 +DTYPES = [torch.float16, torch.bfloat16]
28 +MAX_RANKS = [1, 4, 8, 16, 32, 64, 128]
29 +SCALES = [0.5]
30 +SEED = [0]
31 +CUDA_DEVICES = [f"cuda:{0}"]
32 +
33 +
34 +def assert_close(a, b):
35 + rtol, atol = {
36 + torch.float16: (6e-2, 6e-2),
37 + torch.bfloat16: (6e-2, 6e-2),
38 + torch.float32: (1e-2, 1e-2),
39 + }[a.dtype]
40 + torch.testing.assert_close(a, b, rtol=rtol, atol=atol)
41 +
42 +
43 +@pytest.mark.parametrize("batches", BATCHES)
44 +@pytest.mark.parametrize("num_loras", NUM_LORA)
45 +@pytest.mark.parametrize("rank", MAX_RANKS)
46 +@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
47 +@pytest.mark.parametrize("scaling", SCALES)
48 +@pytest.mark.parametrize("dtype", DTYPES)
49 +@pytest.mark.parametrize("op_type", ["shrink", "expand"])
50 +@pytest.mark.parametrize("seed", SEED)
51 +@pytest.mark.parametrize("device", CUDA_DEVICES)
52 +def test_punica_sgmv(
53 + batches: int,
54 + num_loras: int,
55 + rank: int,
56 + hidden_size: int,
57 + scaling: float,
58 + dtype: torch.dtype,
59 + op_type: str,
60 + seed: int,
61 + device: str,
62 +):
63 + random.seed(seed)
64 + torch.set_default_device(device)
65 + torch.random.manual_seed(seed)
66 + if torch.cuda.is_available():
67 + torch.cuda.manual_seed(seed)
68 +
69 + seq_length = 128
70 + (
71 + inputs_tensor,
72 + lora_weights,
73 + our_out_tensor,
74 + ref_out_tensor,
75 + b_seq_start_loc,
76 + lora_indices_tensor,
77 + seq_len_tensor,
78 + indices,
79 + ) = generate_data(
80 + batches,
81 + hidden_size,
82 + num_loras,
83 + rank,
84 + seq_length,
85 + dtype,
86 + op_type,
87 + device,
88 + )
89 + max_seq_length = seq_len_tensor.max()
90 + if isinstance(max_seq_length, tuple):
91 + max_seq_length = max_seq_length[0].item()
92 + else:
93 + max_seq_length = max_seq_length.item()
94 + if op_type == "shrink":
95 + sgmv_shrink(
96 + inputs_tensor,
97 + lora_weights,
98 + our_out_tensor,
99 + b_seq_start_loc,
100 + seq_len_tensor,
101 + lora_indices_tensor,
102 + batches,
103 + max_seq_length,
104 + scaling,
105 + )
106 + else:
107 + sgmv_expand(
108 + inputs_tensor,
109 + lora_weights,
110 + our_out_tensor,
111 + b_seq_start_loc,
112 + seq_len_tensor,
113 + lora_indices_tensor,
114 + batches,
115 + max_seq_length,
116 + add_inputs=True,
117 + )
118 + ref_torch_groupgemm(
119 + ref_out_tensor,
120 + inputs_tensor,
121 + lora_weights,
122 + lora_indices_tensor,
123 + seq_len_tensor,
124 + batches,
125 + scaling if op_type == "shrink" else 1.0,
126 + op_type,
127 + )
128 + if op_type == "shrink":
129 + ref_out_tensor = ref_out_tensor.to(torch.float32)
130 + assert_close(our_out_tensor, ref_out_tensor)
131 +
132 +
133 +@pytest.mark.parametrize("batches", BATCHES)
134 +@pytest.mark.parametrize("num_loras", NUM_LORA)
135 +@pytest.mark.parametrize("rank", MAX_RANKS)
136 +@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
137 +@pytest.mark.parametrize("scaling", SCALES)
138 +@pytest.mark.parametrize("dtype", DTYPES)
139 +@pytest.mark.parametrize("op_type", ["shrink", "expand"])
140 +@pytest.mark.parametrize("seed", SEED)
141 +@pytest.mark.parametrize("device", CUDA_DEVICES)
142 +def test_punica_bgmv(
143 + batches: int,
144 + num_loras: int,
145 + rank: int,
146 + hidden_size: int,
147 + scaling: float,
148 + dtype: torch.dtype,
149 + op_type: str,
150 + seed: int,
151 + device: str,
152 +):
153 + from vllm.lora.ops.bgmv_expand import _bgmv_expand_kernel
154 + from vllm.lora.ops.bgmv_shrink import _bgmv_shrink_kernel
155 +
156 + random.seed(seed)
157 + torch.set_default_device(device)
158 + torch.random.manual_seed(seed)
159 + if torch.cuda.is_available():
160 + torch.cuda.manual_seed(seed)
161 +
162 + seq_length = 1
163 + (
164 + inputs_tensor,
165 + lora_weights,
166 + our_out_tensor,
167 + ref_out_tensor,
168 + b_seq_start_loc,
169 + lora_indices_tensor,
170 + seq_len_tensor,
171 + indices,
172 + ) = generate_data(
173 + batches,
174 + hidden_size,
175 + num_loras,
176 + rank,
177 + seq_length,
178 + dtype,
179 + op_type,
180 + device,
181 + )
182 + if op_type == "shrink":
183 + # The current _bgmv_shrink_kernel does not require the libentry
184 + # decoration. The purpose of adding this patch is to test the
185 + # correctness of libentry.
186 + with patch(
187 + "vllm.lora.ops.bgmv_shrink._bgmv_shrink_kernel",
188 + LibEntry(_bgmv_shrink_kernel),
189 + ):
190 + bgmv_shrink(
191 + inputs_tensor,
192 + lora_weights,
193 + our_out_tensor,
194 + indices,
195 + scaling,
196 + )
197 + else:
198 + # ditto
199 + with patch(
200 + "vllm.lora.ops.bgmv_expand._bgmv_expand_kernel",
201 + LibEntry(_bgmv_expand_kernel),
202 + ):
203 + bgmv_expand(
204 + inputs_tensor,
205 + lora_weights,
206 + our_out_tensor,
207 + indices,
208 + add_inputs=True,
209 + )
210 + ref_torch_groupgemm(
211 + ref_out_tensor,
212 + inputs_tensor,
213 + lora_weights,
214 + lora_indices_tensor,
215 + seq_len_tensor,
216 + batches,
217 + scaling if op_type == "shrink" else 1.0,
218 + op_type,
219 + )
220 + if op_type == "shrink":
221 + ref_out_tensor = ref_out_tensor.to(torch.float32)
222 + assert_close(our_out_tensor, ref_out_tensor)
223 +
224 +
225 +@pytest.mark.parametrize("batches", BATCHES)
226 +@pytest.mark.parametrize("num_loras", NUM_LORA)
227 +@pytest.mark.parametrize("rank", MAX_RANKS)
228 +@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
229 +@pytest.mark.parametrize("nslices", [2, 3])
230 +@pytest.mark.parametrize("dtype", DTYPES)
231 +@pytest.mark.parametrize("op_type", ["sgmv", "bgmv"])
232 +@pytest.mark.parametrize("seed", SEED)
233 +@pytest.mark.parametrize("device", CUDA_DEVICES)
234 +def test_punica_expand_nslices(
235 + batches: int,
236 + num_loras: int,
237 + rank: int,
238 + hidden_size: int,
239 + nslices: int,
240 + dtype: torch.dtype,
241 + op_type: str,
242 + seed: int,
243 + device: str,
244 +):
245 + from vllm.lora.ops.bgmv_expand_slice import _bgmv_expand_slice_kernel
246 +
247 + random.seed(seed)
248 + torch.set_default_device(device)
249 + torch.random.manual_seed(seed)
250 + if torch.cuda.is_available():
251 + torch.cuda.manual_seed(seed)
252 + seq_length = 128 if op_type == "sgmv" else 1
253 + (
254 + inputs_tensor,
255 + lora_weights_lst,
256 + our_outputs,
257 + ref_outputs,
258 + b_seq_start_loc,
259 + lora_indices_tensor,
260 + seq_len_tensor,
261 + indices,
262 + ) = generate_data_for_expand_nslices(
263 + batches,
264 + hidden_size,
265 + num_loras,
266 + rank,
267 + seq_length,
268 + dtype,
269 + nslices,
270 + device,
271 + )
272 + max_seq_length = seq_len_tensor.max()
273 + if isinstance(max_seq_length, tuple):
274 + max_seq_length = max_seq_length[0].item()
275 + else:
276 + max_seq_length = max_seq_length.item()
277 + slice_offset = 0
278 + for index in range(nslices):
279 + lora_weights = lora_weights_lst[index]
280 + if op_type == "sgmv":
281 + sgmv_expand_slice(
282 + inputs_tensor,
283 + lora_weights,
284 + our_outputs,
285 + b_seq_start_loc,
286 + seq_len_tensor,
287 + lora_indices_tensor,
288 + batches,
289 + max_seq_length,
290 + slice_offset,
291 + hidden_size,
292 + add_inputs=True,
293 + )
294 + else:
295 + # The current _bgmv_expand_slice_kernel does not require the
296 + # libentry decoration. The purpose of adding this patch is to test
297 + # the correctness of libentry.
298 + with patch(
299 + "vllm.lora.ops.bgmv_expand_slice._bgmv_expand_slice_kernel",
300 + LibEntry(_bgmv_expand_slice_kernel),
301 + ):
302 + bgmv_expand_slice(
303 + inputs_tensor,
304 + lora_weights,
305 + our_outputs,
306 + indices,
307 + slice_offset,
308 + slice_size=hidden_size,
309 + add_inputs=True,
310 + )
311 + ref_torch_groupgemm(
312 + ref_outputs[:, slice_offset:slice_offset + hidden_size],
313 + inputs_tensor,
314 + lora_weights,
315 + lora_indices_tensor,
316 + seq_len_tensor,
317 + batches,
318 + 1.0,
319 + op_type="expand",
320 + )
321 +
322 + slice_offset += hidden_size
323 + assert_close(our_outputs, ref_outputs)
324 +
325 +
326 +if __name__ == "__main__":
327 + from itertools import product
328 +
329 + lst = list(
330 + product(
331 + BATCHES,
332 + NUM_LORA,
333 + MAX_RANKS,
334 + [1.0],
335 + [torch.float16],
336 + ["expand"],
337 + SEED,
338 + CUDA_DEVICES,
339 + ))
340 + for ele in lst:
341 + test_punica_bgmv(*ele)
342 + print(f"{ele},pass")
tests/lora/test_quant_model.py
+27 -21
@@ -64,14 +64,16 @@ def test_quant_model_lora(tinyllama_lora_files, model, tp_size):
64 # if torch.cuda.device_count() < tp_size:
65 # pytest.skip(f"Not enough GPUs for tensor parallelism {tp_size}")
66
67 - llm = vllm.LLM(model=model.model_path,
68 - enable_lora=True,
69 - max_num_seqs=16,
70 - max_loras=4,
71 - max_model_len=400,
72 - tensor_parallel_size=tp_size,
73 - quantization=model.quantization,
74 - trust_remote_code=True)
67 + llm = vllm.LLM(
68 + model=model.model_path,
69 + enable_lora=True,
70 + max_num_seqs=16,
71 + max_loras=4,
72 + max_model_len=400,
73 + tensor_parallel_size=tp_size,
74 + gpu_memory_utilization=0.2, #avoid OOM
75 + quantization=model.quantization,
76 + trust_remote_code=True)
77
78 if model.quantization is None:
79 expected_no_lora_output = [
@@ -156,24 +158,28 @@ def test_quant_model_tp_equality(tinyllama_lora_files, model):
158 # if torch.cuda.device_count() < 2:
159 # pytest.skip(f"Not enough GPUs for tensor parallelism {2}")
160
159 - llm_tp1 = vllm.LLM(model=model.model_path,
160 - enable_lora=True,
161 - max_num_seqs=16,
162 - max_loras=4,
163 - tensor_parallel_size=1,
164 - quantization=model.quantization,
165 - trust_remote_code=True)
161 + llm_tp1 = vllm.LLM(
162 + model=model.model_path,
163 + enable_lora=True,
164 + max_num_seqs=16,
165 + max_loras=4,
166 + tensor_parallel_size=1,
167 + gpu_memory_utilization=0.2, #avoid OOM
168 + quantization=model.quantization,
169 + trust_remote_code=True)
170 output_tp1 = do_sample(llm_tp1, tinyllama_lora_files, lora_id=1)
171
172 del llm_tp1
173 cleanup()
174
171 - llm_tp2 = vllm.LLM(model=model.model_path,
172 - enable_lora=True,
173 - max_num_seqs=16,
174 - max_loras=4,
175 - tensor_parallel_size=2,
176 - quantization=model.quantization)
175 + llm_tp2 = vllm.LLM(
176 + model=model.model_path,
177 + enable_lora=True,
178 + max_num_seqs=16,
179 + max_loras=4,
180 + tensor_parallel_size=2,
181 + gpu_memory_utilization=0.2, #avoid OOM
182 + quantization=model.quantization)
183 output_tp2 = do_sample(llm_tp2, tinyllama_lora_files, lora_id=1)
184
185 del llm_tp2
tests/lora/utils.py
+148
@@ -86,3 +86,151 @@ class DummyLoRAManager:
86 packed_lora = PackedLoRALayerWeights.pack(base_loras)
87 self.set_module_lora(module_name, packed_lora)
88 return packed_lora
89 +
90 +
91 +def assert_close(a, b):
92 + rtol, atol = {
93 + torch.float16: (6e-2, 6e-2),
94 + torch.bfloat16: (6e-2, 6e-2),
95 + torch.float32: (1e-2, 1e-2),
96 + }[a.dtype]
97 + torch.testing.assert_close(a, b, rtol=rtol, atol=atol)
98 +
99 +
100 +def ref_torch_groupgemm(
101 + out_tensor,
102 + inputs,
103 + lora_weights,
104 + lora_indices_tensor,
105 + seq_len_tensor,
106 + batches,
107 + scaling,
108 + op_type,
109 +) -> torch.Tensor:
110 + out_list = []
111 + current_offset = 0
112 + for lora_index, b_length in zip(range(batches), seq_len_tensor):
113 + input_weight = inputs[current_offset:b_length + current_offset, :]
114 + current_offset += b_length
115 + lora_weight = lora_weights[lora_indices_tensor[lora_index]]
116 + result = torch.nn.functional.linear(input_weight, lora_weight)
117 + result *= scaling
118 + out_list.append(result)
119 + cat_result = torch.cat(out_list, dim=0)
120 + if op_type == "expand":
121 + out_tensor += cat_result
122 + else:
123 + out_tensor.copy_(cat_result)
124 + return
125 +
126 +
127 +def generate_data(batches, hidden_size, lora_nums, max_rank, seq_length, dtype,
128 + op_type, device):
129 + seq_len_tensor = torch.randint(seq_length, seq_length + 1,
130 + (batches, )).to(device)
131 + b_seq_start_loc = torch.cumsum(
132 + torch.tensor([0] + seq_len_tensor[:-1].tolist(), dtype=torch.long),
133 + dim=0,
134 + ).to(device)
135 + total_tokens = seq_len_tensor.sum()
136 + if op_type == "shrink":
137 + inputs_tensor = torch.rand((total_tokens, hidden_size),
138 + dtype=dtype).to(device)
139 + lora_weights = torch.rand(
140 + (lora_nums, max_rank, hidden_size), # col-major
141 + dtype=dtype,
142 + ).to(device)
143 + # shrink op need atomic_add, so output is initinized by 0
144 + ref_out_tensor = torch.zeros((total_tokens, max_rank),
145 + dtype=dtype,
146 + device=inputs_tensor.device)
147 + # NOTE shrink kernel using torch.float32 as output type
148 + our_out_tensor = torch.zeros((total_tokens, max_rank),
149 + dtype=torch.float32).to(device)
150 + else:
151 + inputs_tensor = torch.rand(
152 + (total_tokens, max_rank),
153 + dtype=dtype,
154 + ).to(device)
155 + lora_weights = torch.rand(
156 + (lora_nums, hidden_size, max_rank), # col-major
157 + dtype=dtype,
158 + ).to(device)
159 + # expand op needs to complete y+=a@lora_b, so output is
160 + # initinized randomly
161 + ref_out_tensor = torch.rand(
162 + (total_tokens, hidden_size),
163 + dtype=dtype,
164 + ).to(device)
165 + # Ensure the same input.
166 + our_out_tensor = ref_out_tensor.clone()
167 + lora_indices_tensor = torch.randint(0,
168 + lora_nums - 1 if lora_nums > 1 else 1,
169 + (batches, )).to(device)
170 + indices = torch.zeros((total_tokens), dtype=torch.long).to(device)
171 + current_offset = 0
172 + for b_id in range(batches):
173 + lora_index = lora_indices_tensor[b_id]
174 + indices[current_offset:current_offset +
175 + seq_len_tensor[b_id]].copy_(lora_index)
176 + current_offset += seq_len_tensor[b_id].item()
177 + return (
178 + inputs_tensor,
179 + lora_weights,
180 + our_out_tensor,
181 + ref_out_tensor,
182 + b_seq_start_loc,
183 + lora_indices_tensor,
184 + seq_len_tensor,
185 + indices,
186 + )
187 +
188 +
189 +def generate_data_for_expand_nslices(batches, hidden_size, lora_nums, max_rank,
190 + seq_length, dtype, nslices, device):
191 + seq_len_tensor = torch.randint(seq_length, seq_length + 1,
192 + (batches, )).to(device)
193 + b_seq_start_loc = torch.cumsum(
194 + torch.tensor([0] + seq_len_tensor[:-1].tolist(), dtype=torch.long),
195 + dim=0,
196 + ).to(device)
197 + total_tokens = seq_len_tensor.sum()
198 + inputs_tensor = torch.rand(
199 + (total_tokens, max_rank),
200 + dtype=dtype,
201 + ).to(device)
202 + lora_weights_lst = []
203 + for _ in range(nslices):
204 + lora_weights_lst.append(
205 + torch.rand(
206 + (lora_nums, hidden_size, max_rank), # col-major
207 + dtype=dtype,
208 + ).to(device))
209 + # expand op needs to complete y+=a@lora_b, so output is
210 + # initinized randomly
211 + ref_out_tensor = torch.rand((total_tokens, hidden_size * nslices),
212 + dtype=dtype).to(device)
213 + # Ensure the same input.
214 + our_out_tensor = ref_out_tensor.clone()
215 + lora_indices_tensor = torch.randint(0,
216 + lora_nums - 1 if lora_nums > 1 else 1,
217 + (batches, ))
218 + indices = torch.zeros((total_tokens), dtype=torch.long).to(device)
219 + current_offset = 0
220 + for b_id in range(batches):
221 + lora_index = lora_indices_tensor[b_id]
222 + indices[current_offset:current_offset +
223 + seq_len_tensor[b_id]] = lora_index.item()
224 + current_offset += seq_len_tensor[b_id].item()
225 +
226 + lora_indices_tensor = lora_indices_tensor.to(device)
227 + return (
228 + inputs_tensor,
229 + lora_weights_lst,
230 + our_out_tensor,
231 + ref_out_tensor,
232 + b_seq_start_loc,
233 + lora_indices_tensor,
234 + seq_len_tensor,
235 + indices,
236 + )
vllm/_custom_ops.py
+1 -41
@@ -13,12 +13,9 @@ try:
13 except ImportError as e:
14 logger.warning("Failed to import from vllm._C with %r", e)
15
16 -with contextlib.suppress(ImportError):
17 - import vllm._moe_C
18 -
16 with contextlib.suppress(ImportError):
17 # ruff: noqa: F401
21 - import vllm._punica_C
18 + import vllm._moe_C
19
20
21 def is_custom_op_supported(op_name: str) -> bool:
@@ -519,43 +516,6 @@ def register_graph_buffers(fa: int, handles: List[str],
516 torch.ops._C_custom_ar.register_graph_buffers(fa, handles, offsets)
517
518
522 -# punica
523 -def dispatch_bgmv(
524 - y: torch.Tensor,
525 - x: torch.Tensor,
526 - w_t_all: torch.Tensor,
527 - indicies: torch.Tensor,
528 - layer_idx: int,
529 - scale: float,
530 -) -> None:
531 - torch.ops._punica_C.dispatch_bgmv(y, x, w_t_all, indicies, layer_idx,
532 - scale)
533 -
534 -
535 -def dispatch_bgmv_low_level(
536 - y: torch.Tensor,
537 - x: torch.Tensor,
538 - w_t_all: torch.Tensor,
539 - indicies: torch.Tensor,
540 - layer_idx: int,
541 - scale: float,
542 - h_in: int,
543 - h_out: int,
544 - y_offset: int,
545 -) -> None:
546 - torch.ops._punica_C.dispatch_bgmv_low_level(
547 - y,
548 - x,
549 - w_t_all,
550 - indicies,
551 - layer_idx,
552 - scale,
553 - h_in,
554 - h_out,
555 - y_offset,
556 - )
557 -
558 -
519 # temporary fix for https://github.com/vllm-project/vllm/issues/5456
520 # TODO: remove this in v0.6.0
521 names_and_values = globals()
vllm/envs.py
-5
@@ -45,7 +45,6 @@ if TYPE_CHECKING:
45 MAX_JOBS: Optional[str] = None
46 NVCC_THREADS: Optional[str] = None
47 VLLM_USE_PRECOMPILED: bool = False
48 - VLLM_INSTALL_PUNICA_KERNELS: bool = False
48 VLLM_NO_DEPRECATION_WARNING: bool = False
49 CMAKE_BUILD_TYPE: Optional[str] = None
50 VERBOSE: bool = False
@@ -94,10 +93,6 @@ environment_variables: Dict[str, Callable[[], Any]] = {
93 "VLLM_USE_PRECOMPILED":
94 lambda: bool(os.environ.get("VLLM_USE_PRECOMPILED")),
95
97 - # If set, vllm will install Punica kernels
98 - "VLLM_INSTALL_PUNICA_KERNELS":
99 - lambda: bool(int(os.getenv("VLLM_INSTALL_PUNICA_KERNELS", "0"))),
100 -
96 # CMake build type
97 # If not set, defaults to "Debug" or "RelWithDebInfo"
98 # Available options: "Debug", "Release", "RelWithDebInfo"
vllm/lora/fully_sharded_layers.py
+80 -57
@@ -14,7 +14,6 @@ from vllm.lora.layers import (ColumnParallelLinearWithLoRA,
14 MergedQKVParallelLinearWithLora,
15 QKVParallelLinearWithLora,
16 RowParallelLinearWithLoRA)
17 -from vllm.lora.punica import bgmv, dispatch_bgmv_low_level
17
18 if TYPE_CHECKING:
19 pass
@@ -28,7 +27,7 @@ def _fully_sharded_can_replace(can_replace):
27
28 def dec(*args, **kwargs):
29 return (can_replace(*args, **kwargs)
31 - and kwargs['lora_config'].fully_sharded_loras)
30 + and kwargs["lora_config"].fully_sharded_loras)
31
32 return dec
33
@@ -59,25 +58,30 @@ class ColumnParallelLinearWithShardedLoRA(ColumnParallelLinearWithLoRA):
58 x = x.view(-1, x.shape[-1])
59 output, out_orig_shape = output.view(-1,
60 output.shape[-1]), output.shape
62 - buffer = torch.zeros((x.shape[0], self.lora_a_stacked.shape[2]),
63 - dtype=torch.float32,
64 - device=x.device)
65 -
66 - bgmv(buffer, x, self.lora_a_stacked,
67 - self.indices[:self.indices_len[0]], 0, 1.0)
61 + buffer = torch.zeros(
62 + (x.shape[0], self.lora_a_stacked.shape[2]),
63 + dtype=torch.float32,
64 + device=x.device,
65 + )
66 + self.punica_wrapper.add_shrink(buffer, x, self.lora_a_stacked, 1.0)
67 buffer = tensor_model_parallel_all_gather(buffer)
69 - bgmv(output, buffer, self.lora_b_stacked,
70 - self.indices[:self.indices_len[0]], 0, 1.0)
68 + self.punica_wrapper.add_expand(output,
69 + buffer,
70 + self.lora_b_stacked,
71 + add_input=True)
72 # now have column partitioned output
72 -
73 output = output.view(*out_orig_shape)
74 return output
75
76 @classmethod
77 @_fully_sharded_can_replace
78 - def can_replace_layer(cls, source_layer: nn.Module,
79 - lora_config: LoRAConfig, packed_modules_list: List,
80 - model_config: Optional[PretrainedConfig]) -> bool:
78 + def can_replace_layer(
79 + cls,
80 + source_layer: nn.Module,
81 + lora_config: LoRAConfig,
82 + packed_modules_list: List,
83 + model_config: Optional[PretrainedConfig],
84 + ) -> bool:
85 # specifying kwargs so they can be easily accessed in decorator
86 return super().can_replace_layer(
87 source_layer=source_layer,
@@ -88,14 +92,14 @@ class ColumnParallelLinearWithShardedLoRA(ColumnParallelLinearWithLoRA):
92 )
93
94
91 -def _mcp_apply(x, bias, layer):
95 +def _mcp_apply(x, bias, layer: QKVParallelLinearWithLora):
96 """
93 - MergedColumnParallelLinearWithShardedLoRA and
94 - MergedQKVParallelLinearWithShardedLora share the same
97 + MergedColumnParallelLinearWithShardedLoRA and
98 + MergedQKVParallelLinearWithShardedLora share the same
99 LoRa weight application method.
100
101 The main difference is the step by shard_size for lora_b which can
98 - vary for MergedQKVParallelLinearWithShardedLora but is constant for
102 + vary for MergedQKVParallelLinearWithShardedLora but is constant for
103 MergedColumnParallelLinearWithShardedLoRA.
104 """
105 # expecting 2 for column parallel and 3 for qkv
@@ -104,21 +108,27 @@ def _mcp_apply(x, bias, layer):
108
109 x = x.view(-1, x.shape[-1])
110 output, out_orig_shape = output.view(-1, output.shape[-1]), output.shape
107 - buffers = torch.zeros((n, x.shape[0], layer.lora_a_stacked[0].shape[2]),
108 - dtype=torch.float32,
109 - device=x.device)
111 + buffers = torch.zeros(
112 + (n, x.shape[0], layer.lora_a_stacked[0].shape[2]),
113 + dtype=torch.float32,
114 + device=x.device,
115 + )
116 for idx in range(n):
111 - bgmv(buffers[idx], x, layer.lora_a_stacked[idx],
112 - layer.indices[:layer.indices_len[0]], 0, 1.0)
117 + layer.punica_wrapper.add_shrink(buffers[idx], x,
118 + layer.lora_a_stacked[idx], 1.0)
119
120 buffers = tensor_model_parallel_all_gather(buffers)
121 left_offset = 0
122 for idx in range(n):
123 shard_size = layer.lora_b_stacked[idx].shape[2]
118 - dispatch_bgmv_low_level(output, buffers[idx],
119 - layer.lora_b_stacked[idx],
120 - layer.indices[:layer.indices_len[0]], 0, 1.0,
121 - left_offset, shard_size)
124 + layer.punica_wrapper.add_expand_slice(
125 + output,
126 + buffers[idx],
127 + layer.lora_b_stacked[idx],
128 + left_offset,
129 + shard_size,
130 + add_input=True,
131 + )
132 left_offset += shard_size
133
134 output = output.view(*out_orig_shape)
@@ -129,7 +139,7 @@ def _mcp_apply(x, bias, layer):
139 class MergedColumnParallelLinearWithShardedLoRA(
140 MergedColumnParallelLinearWithLoRA):
141 """
132 - Differs from MergedColumnParallelLinearWithLoRA by slicing the
142 + Differs from MergedColumnParallelLinearWithLoRA by slicing the
143 LoRA A's also.
144
145 Based on S-LoRA, slicing happens along the rank dim.
@@ -145,7 +155,8 @@ class MergedColumnParallelLinearWithShardedLoRA(
155 lora_a = [
156 lora_a[0][:,
157 output_start_idx:output_start_idx + output_shard_size],
148 - lora_a[1][:, output_start_idx:output_start_idx + output_shard_size]
158 + lora_a[1][:,
159 + output_start_idx:output_start_idx + output_shard_size],
160 ]
161 return lora_a
162
@@ -155,9 +166,13 @@ class MergedColumnParallelLinearWithShardedLoRA(
166
167 @classmethod
168 @_fully_sharded_can_replace
158 - def can_replace_layer(cls, source_layer: nn.Module,
159 - lora_config: LoRAConfig, packed_modules_list: List,
160 - model_config: Optional[PretrainedConfig]) -> bool:
169 + def can_replace_layer(
170 + cls,
171 + source_layer: nn.Module,
172 + lora_config: LoRAConfig,
173 + packed_modules_list: List,
174 + model_config: Optional[PretrainedConfig],
175 + ) -> bool:
176 # specifying kwargs so they can be easily accessed in decorator
177 return super().can_replace_layer(
178 source_layer=source_layer,
@@ -170,7 +185,7 @@ class MergedColumnParallelLinearWithShardedLoRA(
185
186 class QKVParallelLinearWithShardedLora(QKVParallelLinearWithLora):
187 """
173 - Differs from QKVParallelLinearWithLora by slicing the
188 + Differs from QKVParallelLinearWithLora by slicing the
189 LoRA A's also.
190
191 Based on S-LoRA, slicing happens along the rank dim.
@@ -193,14 +208,13 @@ class QKVParallelLinearWithShardedLora(QKVParallelLinearWithLora):
208 buffer = torch.zeros((x.shape[0], self.lora_a_stacked.shape[2]),
209 dtype=torch.float32,
210 device=x.device)
196 -
197 - bgmv(buffer, x, self.lora_a_stacked,
198 - self.indices[:self.indices_len[0]], 0, 1.0)
211 + self.punica_wrapper.add_shrink(buffer, x, self.lora_a_stacked, 1.0)
212 buffer = tensor_model_parallel_all_gather(buffer)
200 - bgmv(output, buffer, self.lora_b_stacked,
201 - self.indices[:self.indices_len[0]], 0, 1.0)
213 + self.punica_wrapper.add_expand(output,
214 + buffer,
215 + self.lora_b_stacked,
216 + add_input=True)
217 # now have column partitioned output
203 -
218 output = output.view(*out_orig_shape)
219 return output
220
@@ -237,7 +251,7 @@ class MergedQKVParallelLinearWithShardedLora(MergedQKVParallelLinearWithLora):
251 lora_a = [
252 lora_a[0][:, start_idx[0]:start_idx[0] + shard_size[0]],
253 lora_a[1][:, start_idx[1]:start_idx[1] + shard_size[1]],
240 - lora_a[2][:, start_idx[2]:start_idx[2] + shard_size[2]]
254 + lora_a[2][:, start_idx[2]:start_idx[2] + shard_size[2]],
255 ]
256 return lora_a
257
@@ -247,9 +261,13 @@ class MergedQKVParallelLinearWithShardedLora(MergedQKVParallelLinearWithLora):
261
262 @classmethod
263 @_fully_sharded_can_replace
250 - def can_replace_layer(cls, source_layer: nn.Module,
251 - lora_config: LoRAConfig, packed_modules_list: List,
252 - model_config: Optional[PretrainedConfig]) -> bool:
264 + def can_replace_layer(
265 + cls,
266 + source_layer: nn.Module,
267 + lora_config: LoRAConfig,
268 + packed_modules_list: List,
269 + model_config: Optional[PretrainedConfig],
270 + ) -> bool:
271 # specifying kwargs so they can be easily accessed in decorator
272 return super().can_replace_layer(
273 source_layer=source_layer,
@@ -262,11 +280,11 @@ class MergedQKVParallelLinearWithShardedLora(MergedQKVParallelLinearWithLora):
280
281 class RowParallelLinearWithShardedLoRA(RowParallelLinearWithLoRA):
282 """
265 - Differs from RowParallelLinearWithLoRA by slicing the
283 + Differs from RowParallelLinearWithLoRA by slicing the
284 LoRA B's also.
285
286 Based on S-LoRA, slicing happens along the output dim.
269 - This yields a combined partial sum from the row parallel base
287 + This yields a combined partial sum from the row parallel base
288 layer and column partitioned output from the LoRA.
289 """
290
@@ -283,11 +301,13 @@ class RowParallelLinearWithShardedLoRA(RowParallelLinearWithLoRA):
301 x = x.view(-1, x.shape[-1])
302 output, out_orig_shape = output.view(-1,
303 output.shape[-1]), output.shape
286 - buffer = torch.zeros((x.shape[0], self.lora_a_stacked.shape[2]),
287 - dtype=torch.float32,
288 - device=x.device)
289 - bgmv(buffer, x, self.lora_a_stacked,
290 - self.indices[:self.indices_len[0]], 0, 1.0)
304 + buffer = torch.zeros(
305 + (x.shape[0], self.lora_a_stacked.shape[2]),
306 + dtype=torch.float32,
307 + device=x.device,
308 + )
309 +
310 + self.punica_wrapper.add_shrink(buffer, x, self.lora_a_stacked, 1.0)
311 buffer = tensor_model_parallel_all_reduce(buffer)
312
313 # following S-LoRA, allows the fusing of all_gather and all_reduce
@@ -298,18 +318,21 @@ class RowParallelLinearWithShardedLoRA(RowParallelLinearWithLoRA):
318 # reduced before being used
319 shard_size = self.lora_b_stacked.shape[2]
320 start_idx = self.tp_rank * shard_size
301 - dispatch_bgmv_low_level(output, buffer, self.lora_b_stacked,
302 - self.indices[:self.indices_len[0]], 0, 1.0,
303 - start_idx, shard_size)
304 -
321 + self.punica_wrapper.add_expand_slice(output, buffer,
322 + self.lora_b_stacked, start_idx,
323 + shard_size)
324 output = output.view(*out_orig_shape)
325 return output
326
327 @classmethod
328 @_fully_sharded_can_replace
310 - def can_replace_layer(cls, source_layer: nn.Module,
311 - lora_config: LoRAConfig, packed_modules_list: List,
312 - model_config: Optional[PretrainedConfig]) -> bool:
329 + def can_replace_layer(
330 + cls,
331 + source_layer: nn.Module,
332 + lora_config: LoRAConfig,
333 + packed_modules_list: List,
334 + model_config: Optional[PretrainedConfig],
335 + ) -> bool:
336 # specifying kwargs so they can be easily accessed in decorator
337 return super().can_replace_layer(
338 source_layer=source_layer,
vllm/lora/layers.py
+151 -286
@@ -17,7 +17,7 @@ from vllm.distributed import (get_tensor_model_parallel_rank,
17 tensor_model_parallel_all_reduce,
18 tensor_model_parallel_gather)
19 from vllm.distributed.utils import divide
20 -from vllm.lora.punica import add_lora, add_lora_slice, bgmv
20 +from vllm.lora.punica import PunicaWrapper
21 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
22 MergedColumnParallelLinear,
23 QKVParallelLinear,
@@ -55,88 +55,17 @@ def _not_fully_sharded_can_replace(can_replace):
55 """
56
57 def dec(*args, **kwargs):
58 - decorate = kwargs.pop('decorate') if 'decorate' in kwargs else True
59 - condition = (not kwargs['lora_config'].fully_sharded_loras
58 + decorate = kwargs.pop("decorate") if "decorate" in kwargs else True
59 + condition = (not kwargs["lora_config"].fully_sharded_loras
60 if decorate else True)
61 return can_replace(*args, **kwargs) and condition
62
63 return dec
64
65
66 -def _apply_lora(
67 - x: torch.Tensor,
68 - lora_a_stacked: torch.Tensor,
69 - lora_b_stacked: torch.Tensor,
70 - indices: torch.Tensor,
71 - output: torch.Tensor,
72 -):
73 - """Applies lora to each input.
74 -
75 - This method applies all loras to each input. It uses the
76 - indices vector to determine which lora yields the
77 - correct output. An index of -1 means no lora should be
78 - applied. This method adds the final lora results to the
79 - output.
80 -
81 - Input shapes:
82 - x: (batch_size, hidden_dim)
83 - lora_a_stacked: (num_loras, lora_rank, hidden_dim)
84 - lora_b_stacked: (num_loras, output_dim, lora_rank)
85 - indices: (batch_size)
86 - output: (batch_size, output_dim)
87 - """
88 - org_output = output
89 - x = x.view(-1, x.shape[-1])
90 - output = output.view(-1, output.shape[-1])
91 - indices = indices.view(-1)
92 - add_lora(output, x, lora_a_stacked, lora_b_stacked, indices, 0, 1.0)
93 - return output.view_as(org_output)
94 -
95 -
96 -def _apply_lora_packed_nslice(
97 - x: torch.Tensor,
98 - lora_a_stacked: Tuple[torch.Tensor, torch.Tensor, torch.Tensor],
99 - lora_b_stacked: Tuple[torch.Tensor, torch.Tensor, torch.Tensor],
100 - indices: torch.Tensor,
101 - output: torch.Tensor,
102 - output_slices: Tuple[int, ...],
103 -):
104 - """Applies lora to each input.
105 -
106 - This method applies all loras to each input. It uses the
107 - indices vector to determine which lora yields the
108 - correct output. An index of -1 means no lora should be
109 - applied. This method adds the final lora results to the
110 - output.
111 -
112 - This method is used for layers that are composed of multiple sublayers
113 - (slices) packed together.
114 -
115 - Input shapes:
116 - x: (batch_size, hidden_dim)
117 - lora_a_stacked: 3 element tuple of (num_loras, lora_rank, hidden_dim)
118 - lora_b_stacked: 3 element tuple of (num_loras, output_dim, lora_rank)
119 - indices: (batch_size)
120 - output: (batch_size, q_slice_size + 2*kv_slice_size)
121 - output_slices: n-1 element tuple of (slice_size...),
122 - where n is number of slices
123 - """
124 - org_output = output
125 - x = x.view(-1, x.shape[-1])
126 - output = output.view(-1, output.shape[-1])
127 - indices = indices.view(-1)
128 - offset_left = 0
129 - for slice_idx in range(len(output_slices)):
130 - add_lora_slice(output, x, lora_a_stacked[slice_idx],
131 - lora_b_stacked[slice_idx], indices, 0, 1.0, offset_left,
132 - output_slices[slice_idx])
133 - offset_left += output_slices[slice_idx]
134 - return output.view_as(org_output)
135 -
136 -
66 @dataclass
67 class LoRAMapping(AdapterMapping):
139 - pass
68 + is_prefill: bool = False
69
70
71 class BaseLayerWithLoRA(nn.Module):
@@ -154,10 +83,11 @@ class BaseLayerWithLoRA(nn.Module):
83 ...
84
85 def create_lora_weights(
157 - self,
158 - max_loras: int,
159 - lora_config: LoRAConfig,
160 - model_config: Optional[PretrainedConfig] = None) -> None:
86 + self,
87 + max_loras: int,
88 + lora_config: LoRAConfig,
89 + model_config: Optional[PretrainedConfig] = None,
90 + ) -> None:
91 """Initializes lora matrices."""
92 ...
93
@@ -177,20 +107,18 @@ class BaseLayerWithLoRA(nn.Module):
107
108 def set_mapping(
109 self,
180 - base_indices: torch.Tensor,
181 - sampler_indices: torch.Tensor,
182 - sampler_indices_padded: torch.Tensor,
183 - embeddings_indices: torch.Tensor,
184 - long_lora_indices: torch.Tensor,
185 - indices_len: List[int],
110 + punica_wrapper: PunicaWrapper,
111 ):
187 - """Sets the mapping indices."""
188 - ...
112 + self.punica_wrapper: PunicaWrapper = punica_wrapper
113
114 @classmethod
191 - def can_replace_layer(cls, source_layer: nn.Module,
192 - lora_config: LoRAConfig, packed_modules_list: List,
193 - model_config: Optional[PretrainedConfig]) -> bool:
115 + def can_replace_layer(
116 + cls,
117 + source_layer: nn.Module,
118 + lora_config: LoRAConfig,
119 + packed_modules_list: List,
120 + model_config: Optional[PretrainedConfig],
121 + ) -> bool:
122 """Returns True if the layer can be replaced by this LoRA layer."""
123 raise NotImplementedError
124
@@ -259,10 +187,6 @@ class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA):
187 self.lora_a_stacked.shape[0] * self.lora_a_stacked.shape[1],
188 self.lora_a_stacked.shape[2],
189 )
262 - # Lazily initialized.
263 - self.indices: torch.Tensor
264 - self.indices_len: List[int]
265 - self.embeddings_indices: torch.Tensor
190
191 def reset_lora(self, index: int):
192 self.lora_a_stacked[index] = 0
@@ -285,40 +209,27 @@ class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA):
209 if embeddings_tensor is not None:
210 self.embeddings_tensors[
211 index, :embeddings_tensor.shape[0], :embeddings_tensor.
288 - shape[1]].copy_(embeddings_tensor, non_blocking=True)
212 + shape[1], ].copy_(embeddings_tensor, non_blocking=True)
213 if self.embeddings_slice is not None:
214 # TODO(yard1): Optimize this copy, we don't need to copy
215 # everything, just the modified part
216 embeddings = self.embeddings_tensors.view(
217 self.embeddings_tensors.shape[0] *
218 self.embeddings_tensors.shape[1],
295 - self.embeddings_tensors.shape[2]
219 + self.embeddings_tensors.shape[2],
220 )[self.embeddings_slice[0]:self.embeddings_slice[1]]
221 assert self.embeddings_weights is not None
222 self.embeddings_weights[:embeddings.shape[0]].copy_(embeddings)
223
300 - def set_mapping(
301 - self,
302 - base_indices: torch.Tensor,
303 - sampler_indices: torch.Tensor,
304 - sampler_indices_padded: torch.Tensor,
305 - embeddings_indices: torch.Tensor,
306 - long_lora_indices: torch.Tensor,
307 - indices_len: List[int],
308 - ):
309 - self.indices = base_indices
310 - self.embeddings_indices = embeddings_indices
311 - self.indices_len = indices_len
312 -
224 def forward(self, x: torch.Tensor) -> torch.Tensor:
225 added_tokens_mask = x > self.base_layer.org_vocab_size - 1
315 - embedding_len = self.indices_len[3]
316 - indices = self.embeddings_indices[1][:embedding_len].view_as(x)
226 + embeddings_indices = self.punica_wrapper.embeddings_indices
227 + indices = embeddings_indices[1].view_as(x)
228 full_lora_a_embeddings = F.embedding(
229 x + indices,
230 self.lora_a_stacked_2d,
231 )
321 - indices = self.embeddings_indices[0][:embedding_len].view_as(x)
232 + indices = embeddings_indices[0].view_as(x)
233 full_output = self.base_layer.forward(
234 x.add_(indices * added_tokens_mask))
235
@@ -329,22 +240,32 @@ class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA):
240 if full_lora_a_embeddings.ndim == 3:
241 full_lora_a_embeddings = full_lora_a_embeddings.view(
242 full_lora_a_embeddings.shape[0] *
332 - full_lora_a_embeddings.shape[1], -1)
333 - bgmv(full_output, full_lora_a_embeddings, self.lora_b_stacked,
334 - self.indices[:self.indices_len[0]], 0, 1.0)
243 + full_lora_a_embeddings.shape[1],
244 + -1,
245 + )
246 +
247 + # Embedding layer only need expand op
248 + self.punica_wrapper.add_expand(full_output,
249 + full_lora_a_embeddings,
250 + self.lora_b_stacked,
251 + add_input=True)
252 return full_output.view_as(full_output_org)
253
254 @classmethod
338 - def can_replace_layer(cls, source_layer: nn.Module,
339 - lora_config: LoRAConfig, packed_modules_list: List,
340 - model_config: Optional[PretrainedConfig]) -> bool:
255 + def can_replace_layer(
256 + cls,
257 + source_layer: nn.Module,
258 + lora_config: LoRAConfig,
259 + packed_modules_list: List,
260 + model_config: Optional[PretrainedConfig],
261 + ) -> bool:
262 return type(source_layer) is VocabParallelEmbedding
263
264
265 class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
266 """
267 LoRA on top of ColumnParallelLinear layer.
347 -
268 +
269 LoRA B is sliced for tensor parallelism.
270 """
271
@@ -357,10 +278,11 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
278 self.device = _get_lora_device(self.base_layer)
279
280 def create_lora_weights(
360 - self,
361 - max_loras: int,
362 - lora_config: LoRAConfig,
363 - model_config: Optional[PretrainedConfig] = None) -> None:
281 + self,
282 + max_loras: int,
283 + lora_config: LoRAConfig,
284 + model_config: Optional[PretrainedConfig] = None,
285 + ) -> None:
286 self.lora_config = lora_config
287 self.tp_size = get_tensor_model_parallel_world_size()
288 lora_a_output_size_per_partition = (
@@ -384,10 +306,6 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
306 )
307 self.output_dim = self.lora_b_stacked.shape[2]
308
387 - # lazily initialized.
388 - self.indices: torch.Tensor
389 - self.indices_len: List[int]
390 -
309 def reset_lora(self, index: int):
310 self.lora_a_stacked[index] = 0
311 self.lora_b_stacked[index] = 0
@@ -423,28 +341,11 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
341 0, :lora_b.shape[1], :lora_b.shape[0]].copy_(
342 lora_b.T, non_blocking=True)
343
426 - def set_mapping(
427 - self,
428 - base_indices: torch.Tensor,
429 - sampler_indices: torch.Tensor,
430 - sampler_indices_padded: torch.Tensor,
431 - embeddings_indices: torch.Tensor,
432 - long_lora_indices: torch.Tensor,
433 - indices_len: List[int],
434 - ):
435 - self.indices = base_indices
436 - self.indices_len = indices_len
437 -
344 def apply(self, x: torch.Tensor,
345 bias: Optional[torch.Tensor]) -> torch.Tensor:
346 output = self.base_layer.quant_method.apply(self.base_layer, x, bias)
441 - _apply_lora(
442 - x,
443 - self.lora_a_stacked,
444 - self.lora_b_stacked,
445 - self.indices[:self.indices_len[0]],
446 - output,
447 - )
347 + self.punica_wrapper.add_lora(output, x, self.lora_a_stacked,
348 + self.lora_b_stacked, 1.0)
349 return output
350
351 def forward(self, input_):
@@ -473,9 +374,13 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
374
375 @classmethod
376 @_not_fully_sharded_can_replace
476 - def can_replace_layer(cls, source_layer: nn.Module,
477 - lora_config: LoRAConfig, packed_modules_list: List,
478 - model_config: Optional[PretrainedConfig]) -> bool:
377 + def can_replace_layer(
378 + cls,
379 + source_layer: nn.Module,
380 + lora_config: LoRAConfig,
381 + packed_modules_list: List,
382 + model_config: Optional[PretrainedConfig],
383 + ) -> bool:
384 return type(source_layer) is ColumnParallelLinear or (
385 type(source_layer) is MergedColumnParallelLinear
386 and len(packed_modules_list) == 1)
@@ -494,10 +399,11 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
399 super().__init__(base_layer)
400
401 def create_lora_weights(
497 - self,
498 - max_loras: int,
499 - lora_config: LoRAConfig,
500 - model_config: Optional[PretrainedConfig] = None) -> None:
402 + self,
403 + max_loras: int,
404 + lora_config: LoRAConfig,
405 + model_config: Optional[PretrainedConfig] = None,
406 + ) -> None:
407 self.lora_config = lora_config
408 n_slices = 2
409 if not (len(self.base_layer.output_sizes) == n_slices
@@ -533,8 +439,6 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
439 ) for _ in range(n_slices))
440
441 self.output_dim = self.lora_b_stacked[0].shape[2]
536 - # Lazily initialized.
537 - self.indices: torch.Tensor
442
443 def reset_lora(self, index: int):
444 self.lora_a_stacked[0][index] = 0
@@ -556,7 +460,8 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
460 start_idx = self.tp_rank * shard_size
461 end_idx = (self.tp_rank + 1) * shard_size
462 lora_b = [
559 - lora_b[0][:, start_idx:end_idx], lora_b[1][:, start_idx:end_idx]
463 + lora_b[0][:, start_idx:end_idx],
464 + lora_b[1][:, start_idx:end_idx],
465 ]
466 return lora_b
467
@@ -591,34 +496,33 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
496 def apply(self, x: torch.Tensor,
497 bias: Optional[torch.Tensor]) -> torch.Tensor:
498 output = self.base_layer.quant_method.apply(self.base_layer, x, bias)
594 - _apply_lora_packed_nslice(
595 - x,
596 - self.lora_a_stacked,
597 - self.lora_b_stacked,
598 - self.indices[:self.indices_len[0]],
599 - output,
600 - (self.output_dim, self.output_dim),
601 - )
499 + self.punica_wrapper.add_lora_packed_nslice(
500 + output, x, self.lora_a_stacked, self.lora_b_stacked, 1.0,
501 + (self.output_dim, self.output_dim))
502 return output
503
504 @classmethod
505 @_not_fully_sharded_can_replace
606 - def can_replace_layer(cls, source_layer: nn.Module,
607 - lora_config: LoRAConfig, packed_modules_list: List,
608 - model_config: Optional[PretrainedConfig]) -> bool:
609 - return type(source_layer) is MergedColumnParallelLinear and len(
610 - packed_modules_list) == 2
506 + def can_replace_layer(
507 + cls,
508 + source_layer: nn.Module,
509 + lora_config: LoRAConfig,
510 + packed_modules_list: List,
511 + model_config: Optional[PretrainedConfig],
512 + ) -> bool:
513 + return (type(source_layer) is MergedColumnParallelLinear
514 + and len(packed_modules_list) == 2)
515
516
517 class QKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
518 """
615 - ColumnParallelLinear layer that is specifically designed for
616 - qkv_proj. Certain models, such as chtglm3 and baichuan-7b,
617 - only contains a single LoRA within their qkv_proj layer.
519 + ColumnParallelLinear layer that is specifically designed for
520 + qkv_proj. Certain models, such as chtglm3 and baichuan-7b,
521 + only contains a single LoRA within their qkv_proj layer.
522
619 - During inference with Tensor Parallel, the weights of lora_b
523 + During inference with Tensor Parallel, the weights of lora_b
524 must be accurately partitioned according to the respective ranks.
621 -
525 +
526 Q slice may have different shape than K and V slices (which both have
527 the same shape).
528 """
@@ -696,10 +600,11 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
600 super().__init__(base_layer)
601
602 def create_lora_weights(
699 - self,
700 - max_loras: int,
701 - lora_config: LoRAConfig,
702 - model_config: Optional[PretrainedConfig] = None) -> None:
603 + self,
604 + max_loras: int,
605 + lora_config: LoRAConfig,
606 + model_config: Optional[PretrainedConfig] = None,
607 + ) -> None:
608 self.lora_config = lora_config
609 self.tp_size = get_tensor_model_parallel_world_size()
610 self.tp_rank = get_tensor_model_parallel_rank()
@@ -767,11 +672,15 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
672 ),
673 )
674
770 - self.output_slices = (self.q_proj_shard_size, self.kv_proj_shard_size,
771 - self.kv_proj_shard_size)
675 + self.output_slices = (
676 + self.q_proj_shard_size,
677 + self.kv_proj_shard_size,
678 + self.kv_proj_shard_size,
679 + )
680 self.packed_indices: Optional[torch.Tensor] = None
681 self.standard_indices: Optional[torch.Tensor] = None
682 # lazily initialized.
683 + self.indices: torch.Tensor
684 self.indices_len: List[int]
685
686 def reset_lora(self, index: int):
@@ -794,15 +703,15 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
703 if lora_b[0] is not None:
704 lora_b_q = lora_b[0][:, self.q_proj_shard_size *
705 self.q_shard_id:self.q_proj_shard_size *
797 - (self.q_shard_id + 1)]
706 + (self.q_shard_id + 1), ]
707 if lora_b[1] is not None:
708 lora_b_k = lora_b[1][:, self.kv_proj_shard_size *
709 self.kv_shard_id:self.kv_proj_shard_size *
801 - (self.kv_shard_id + 1)]
710 + (self.kv_shard_id + 1), ]
711 if lora_b[2] is not None:
712 lora_b_v = lora_b[2][:, self.kv_proj_shard_size *
713 self.kv_shard_id:self.kv_proj_shard_size *
805 - (self.kv_shard_id + 1)]
714 + (self.kv_shard_id + 1), ]
715 lora_b = [lora_b_q, lora_b_k, lora_b_v]
716 return lora_b
717
@@ -851,23 +760,23 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
760 def apply(self, x: torch.Tensor,
761 bias: Optional[torch.Tensor]) -> torch.Tensor:
762 output = self.base_layer.quant_method.apply(self.base_layer, x, bias)
854 - _apply_lora_packed_nslice(
855 - x,
856 - self.lora_a_stacked,
857 - self.lora_b_stacked,
858 - self.indices[:self.indices_len[0]],
859 - output,
860 - self.output_slices,
861 - )
763 + self.punica_wrapper.add_lora_packed_nslice(output, x,
764 + self.lora_a_stacked,
765 + self.lora_b_stacked, 1.0,
766 + self.output_slices)
767 return output
768
769 @classmethod
770 @_not_fully_sharded_can_replace
866 - def can_replace_layer(cls, source_layer: nn.Module,
867 - lora_config: LoRAConfig, packed_modules_list: List,
868 - model_config: Optional[PretrainedConfig]) -> bool:
869 - return type(source_layer) is QKVParallelLinear and len(
870 - packed_modules_list) == 3
771 + def can_replace_layer(
772 + cls,
773 + source_layer: nn.Module,
774 + lora_config: LoRAConfig,
775 + packed_modules_list: List,
776 + model_config: Optional[PretrainedConfig],
777 + ) -> bool:
778 + return (type(source_layer) is QKVParallelLinear
779 + and len(packed_modules_list) == 3)
780
781
782 class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
@@ -880,10 +789,11 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
789 self.device = _get_lora_device(self.base_layer)
790
791 def create_lora_weights(
883 - self,
884 - max_loras: int,
885 - lora_config: LoRAConfig,
886 - model_config: Optional[PretrainedConfig] = None) -> None:
792 + self,
793 + max_loras: int,
794 + lora_config: LoRAConfig,
795 + model_config: Optional[PretrainedConfig] = None,
796 + ) -> None:
797 self.lora_config = lora_config
798 self.tp_rank = get_tensor_model_parallel_rank()
799 self.lora_a_stacked = torch.zeros(
@@ -911,9 +821,6 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
821 dtype=lora_config.lora_dtype,
822 device=self.device,
823 )
914 - # Lazily initialized
915 - self.indices: torch.Tensor
916 - self.indices_len: List[int]
824
825 def reset_lora(self, index: int):
826 self.lora_a_stacked[index] = 0
@@ -950,27 +857,10 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
857 0, :lora_b.shape[1], :lora_b.shape[0]].copy_(
858 lora_b.T, non_blocking=True)
859
953 - def set_mapping(
954 - self,
955 - base_indices: torch.Tensor,
956 - sampler_indices: torch.Tensor,
957 - sampler_indices_padded: torch.Tensor,
958 - embeddings_indices: torch.Tensor,
959 - long_lora_indices: torch.Tensor,
960 - indices_len: List[int],
961 - ):
962 - self.indices = base_indices
963 - self.indices_len = indices_len
964 -
860 def apply(self, x: torch.Tensor) -> torch.Tensor:
861 output = self.base_layer.quant_method.apply(self.base_layer, x)
967 - _apply_lora(
968 - x,
969 - self.lora_a_stacked,
970 - self.lora_b_stacked,
971 - self.indices[:self.indices_len[0]],
972 - output,
973 - )
862 + self.punica_wrapper.add_lora(output, x, self.lora_a_stacked,
863 + self.lora_b_stacked, 1.0)
864 return output
865
866 def forward(self, input_):
@@ -1013,14 +903,18 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
903
904 @property
905 def weight(self):
1016 - return self.base_layer.weight if hasattr(
1017 - self.base_layer, "weight") else self.base_layer.qweight
906 + return (self.base_layer.weight if hasattr(self.base_layer, "weight")
907 + else self.base_layer.qweight)
908
909 @classmethod
910 @_not_fully_sharded_can_replace
1021 - def can_replace_layer(cls, source_layer: nn.Module,
1022 - lora_config: LoRAConfig, packed_modules_list: List,
1023 - model_config: Optional[PretrainedConfig]) -> bool:
911 + def can_replace_layer(
912 + cls,
913 + source_layer: nn.Module,
914 + lora_config: LoRAConfig,
915 + packed_modules_list: List,
916 + model_config: Optional[PretrainedConfig],
917 + ) -> bool:
918 return type(source_layer) is RowParallelLinear
919
920
@@ -1125,10 +1019,6 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
1019 dtype=torch.long)
1020 else:
1021 self.sharded_to_full_mapping_gpu = None
1128 - # Lazily initialized.
1129 - self.indices: torch.Tensor
1130 - self.indices_len: List[int]
1131 - self.indices_padded: torch.Tensor
1022
1023 def reset_lora(self, index: int):
1024 self.lora_a_stacked[index] = 0
@@ -1154,19 +1044,6 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
1044 index, :embeddings_tensor.shape[0], :embeddings_tensor.
1045 shape[1], ] = embeddings_tensor
1046
1157 - def set_mapping(
1158 - self,
1159 - base_indices: torch.Tensor,
1160 - sampler_indices: torch.Tensor,
1161 - sampler_indices_padded: torch.Tensor,
1162 - embeddings_indices: torch.Tensor,
1163 - long_lora_indices: torch.Tensor,
1164 - indices_len: List[int],
1165 - ):
1166 - self.indices = sampler_indices
1167 - self.indices_padded = sampler_indices_padded
1168 - self.indices_len = indices_len
1169 -
1047 def _get_logits(
1048 self,
1049 hidden_states: torch.Tensor,
@@ -1212,38 +1089,37 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
1089 out=lora_logits[:-1])
1090 lora_logits[-1] = float("-inf")
1091 lora_logits = lora_logits.mT
1092 + indices_padded = self.punica_wrapper.sampler_indices_padded
1093 lora_logits = (lora_logits.reshape(
1094 lora_logits.shape[0] * lora_logits.shape[1],
1095 lora_logits.shape[2],
1218 - ).index_select(0,
1219 - self.indices_padded[:self.indices_len[2]]).nan_to_num_(
1220 - nan=float("-inf"),
1221 - posinf=float("inf"),
1222 - neginf=float("-inf")))
1096 + ).index_select(0, indices_padded).nan_to_num_(nan=float("-inf"),
1097 + posinf=float("inf"),
1098 + neginf=float("-inf")))
1099 logits[:,
1100 self.base_layer.org_vocab_size:self.base_layer.org_vocab_size +
1225 - lora_logits.shape[1]] = lora_logits
1226 -
1227 - _apply_lora(
1228 - hidden_states,
1229 - self.lora_a_stacked,
1230 - self.lora_b_stacked,
1231 - self.indices[:self.indices_len[1]],
1232 - logits,
1233 - )
1101 + lora_logits.shape[1], ] = lora_logits
1102 +
1103 + # LogitsProcessorWithLoRA always using bgmv
1104 + self.punica_wrapper.add_lora_logits(logits, hidden_states,
1105 + self.lora_a_stacked,
1106 + self.lora_b_stacked, 1.0)
1107
1108 # Remove paddings in vocab (if any).
1109 logits = logits[:, :self.base_layer.vocab_size]
1237 -
1110 return logits
1111
1112 def forward(self, *args, **kwargs):
1113 return type(self.base_layer).forward(self, *args, **kwargs)
1114
1115 @classmethod
1244 - def can_replace_layer(cls, source_layer: nn.Module,
1245 - lora_config: LoRAConfig, packed_modules_list: List,
1246 - model_config: Optional[PretrainedConfig]) -> bool:
1116 + def can_replace_layer(
1117 + cls,
1118 + source_layer: nn.Module,
1119 + lora_config: LoRAConfig,
1120 + packed_modules_list: List,
1121 + model_config: Optional[PretrainedConfig],
1122 + ) -> bool:
1123 # Special handling for the LogitsProcessor.
1124 return False
1125
@@ -1259,9 +1135,6 @@ class LinearScalingRotaryEmbeddingWithLora(BaseLayerWithLoRA):
1135 def __init__(self, base_layer: RotaryEmbedding) -> None:
1136 super().__init__()
1137 self.base_layer = base_layer
1262 - # Lazily initialized
1263 - self.long_lora_indices: torch.Tensor
1264 - self.indices_len: List[int]
1138
1139 @property
1140 def scaling_factors(self):
@@ -1277,9 +1150,8 @@ class LinearScalingRotaryEmbeddingWithLora(BaseLayerWithLoRA):
1150 lora_config: LoRAConfig,
1151 model_config: Optional[PretrainedConfig] = None,
1152 ) -> None:
1280 - scaling_factors = list(
1281 - lora_config.long_lora_scaling_factors
1282 - ) if lora_config.long_lora_scaling_factors else []
1153 + scaling_factors = (list(lora_config.long_lora_scaling_factors)
1154 + if lora_config.long_lora_scaling_factors else [])
1155 base_scaling_factor = (self.base_layer.scaling_factor if isinstance(
1156 self.base_layer, LinearScalingRotaryEmbedding) else 1.0)
1157 scaling_factors = sorted(
@@ -1306,18 +1178,6 @@ class LinearScalingRotaryEmbeddingWithLora(BaseLayerWithLoRA):
1178 ):
1179 ...
1180
1309 - def set_mapping(
1310 - self,
1311 - base_indices: torch.Tensor,
1312 - sampler_indices: torch.Tensor,
1313 - sampler_indices_padded: torch.Tensor,
1314 - embeddings_indices: torch.Tensor,
1315 - long_lora_indices: torch.Tensor,
1316 - indices_len: List[int],
1317 - ):
1318 - self.long_lora_indices = long_lora_indices
1319 - self.indices_len = indices_len
1320 -
1181 def forward(
1182 self,
1183 positions: torch.Tensor,
@@ -1328,19 +1188,24 @@ class LinearScalingRotaryEmbeddingWithLora(BaseLayerWithLoRA):
1188 positions,
1189 query,
1190 key,
1331 - offsets=self.long_lora_indices[:self.indices_len[4]])
1191 + offsets=self.punica_wrapper.long_lora_indices,
1192 + )
1193
1194 @property
1195 def scaling_factor_to_offset(self) -> Dict[float, int]:
1196 return self.base_layer.scaling_factor_to_offset
1197
1198 @classmethod
1338 - def can_replace_layer(cls, source_layer: nn.Module,
1339 - lora_config: LoRAConfig, packed_modules_list: List,
1340 - model_config: Optional[PretrainedConfig]) -> bool:
1199 + def can_replace_layer(
1200 + cls,
1201 + source_layer: nn.Module,
1202 + lora_config: LoRAConfig,
1203 + packed_modules_list: List,
1204 + model_config: Optional[PretrainedConfig],
1205 + ) -> bool:
1206 """Returns True if the layer can be replaced by this LoRA layer."""
1342 - return type(source_layer) is LinearScalingRotaryEmbedding or type(
1343 - source_layer) is RotaryEmbedding
1207 + return (type(source_layer) is LinearScalingRotaryEmbedding
1208 + or type(source_layer) is RotaryEmbedding)
1209
1210 def extra_repr(self) -> str:
1211 return self.base_layer.extra_repr()
vllm/lora/models.py
+16 -155
@@ -4,7 +4,7 @@ import math
4 import os
5 import re
6 from dataclasses import dataclass, field
7 -from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union
7 +from typing import Any, Callable, Dict, List, Optional, Type
8
9 import safetensors.torch
10 import torch
@@ -21,6 +21,7 @@ from vllm.lora.layers import (BaseLayerWithLoRA,
21 LinearScalingRotaryEmbeddingWithLora,
22 LoRAMapping)
23 from vllm.lora.lora import LoRALayerWeights, PackedLoRALayerWeights
24 +from vllm.lora.punica import PunicaWrapper
25 from vllm.lora.utils import (from_layer, from_layer_logits_processor,
26 parse_fine_tuned_lora_name, replace_submodule)
27 from vllm.model_executor.models.interfaces import SupportsLoRA
@@ -43,115 +44,6 @@ class LongContextLoRAContext:
44 offsets_by_lora_id: Dict[int, int] = field(default_factory=dict)
45
46
46 -def convert_mapping(
47 - mapping: LoRAMapping,
48 - lora_index_to_id: List[Optional[int]],
49 - max_loras: int,
50 - vocab_size: int,
51 - extra_vocab_size: int,
52 - long_lora_context: Optional[LongContextLoRAContext] = None,
53 -) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor,
54 - Optional[torch.Tensor], List[int]]:
55 - """Converts LoRAMapping to index tensors.
56 -
57 - Args:
58 - mapping: LoRAMapping mapping rows in a batch to LoRA ids.
59 - lora_index_to_id: List mapping LoRA ids to LoRA indices.
60 - max_loras: Maximum number of LoRAs.
61 - vocab_size: Model vocab size.
62 - extra_vocab_size: Extra vocab size each LoRA can have.
63 - long_lora_context: Passed if there are long context lora in a batch.
64 -
65 - Returns:
66 - A tuple of tensors:
67 - base_indices: Tensor of shape [batch_size] mapping batch rows to
68 - LoRA indices.
69 - sampler_indices: Tensor of shape [batch_size] mapping requests to
70 - LoRA indices for sampler. For generation, this will be the
71 - same as base_indicies. For prefill, this will map requests
72 - to LoRA indices.
73 - sampler_indices_padded: Tensor of shape [batch_size] mapping
74 - requests to LoRA indices for sampler with padding.
75 - Same as sampler_indicies, but -1 is replaced with
76 - max_loras.
77 - embeddings_indices: Tensor of shape [2, batch_size] mapping
78 - requests to embedding indices. First row is for embeddings
79 - added by the LoRAs, second row is for the LoRA.lora_a
80 - embeddings.
81 - long_lora_indices: Tensor of shape [batch_size] mapping
82 - requests to RoPE offsets and rot dims for long LoRAs.
83 - None if long context lora doesn't exist.
84 - indices_len: List of lengths of the above tensors.
85 - Used to index into each tensor. It contains length for
86 - (base_indices, sampler_indices, sampler_indices_padded,
87 - embeddings_indices, long_lora_indices). If long_lora doesn't
88 - exist, it only contains first 4 entries.
89 - """
90 - index_mapping_indices: List[int] = list(mapping.index_mapping).copy()
91 - embedding_indices = index_mapping_indices.copy()
92 - lora_indices = index_mapping_indices.copy()
93 - long_lora_offsets: Optional[torch.Tensor] = None
94 - if long_lora_context:
95 - long_lora_offsets = torch.zeros(len(index_mapping_indices),
96 - device="cuda",
97 - dtype=torch.long)
98 - prompt_mapping: List[int] = [
99 - lora_index_to_id.index(x) if x > 0 else -1
100 - for x in mapping.prompt_mapping
101 - ]
102 - lora_idx = None
103 - for i in range(len(index_mapping_indices)):
104 - # TODO index can be slow. optimize
105 - lora_idx = (lora_index_to_id.index(index_mapping_indices[i])
106 - if index_mapping_indices[i] > 0 else -1)
107 - embedding_indices[i] = lora_idx if index_mapping_indices[i] > 0 else 0
108 - lora_indices[i] = lora_idx
109 - if long_lora_context:
110 - assert long_lora_offsets is not None
111 - lora_offset: int = long_lora_context.offsets_by_lora_id.get(
112 - index_mapping_indices[i], 0)
113 - long_lora_offsets[i] = lora_offset
114 -
115 - indices_list: List[Union[List[int], torch.Tensor]] = [
116 - index_mapping_indices, lora_indices, embedding_indices
117 - ]
118 - if long_lora_context:
119 - assert long_lora_offsets is not None
120 - indices_list.append(long_lora_offsets)
121 - indices = torch.tensor(indices_list, dtype=torch.long, device="cuda")
122 - prompt_mapping_tensor = torch.tensor(prompt_mapping,
123 - device="cuda",
124 - dtype=torch.long)
125 - embeddings_indices = torch.stack([
126 - indices[2] * extra_vocab_size,
127 - indices[2] * (vocab_size + extra_vocab_size)
128 - ])
129 - embeddings_indices[embeddings_indices == -1] = max_loras - 1
130 - base_indices = indices[1]
131 - sampler_indices = prompt_mapping_tensor
132 - sampler_indices_padded = sampler_indices.clone()
133 - sampler_indices_padded[sampler_indices_padded == -1] = max_loras - 1
134 - sampler_indices_padded = (
135 - torch.arange(
136 - 0, len(sampler_indices_padded), device="cuda", dtype=torch.long) +
137 - (sampler_indices_padded * len(sampler_indices_padded)))
138 - long_lora_indices = None
139 - long_lora_indices_len: Optional[int] = None
140 - if long_lora_context:
141 - long_lora_indices = indices[3]
142 - long_lora_indices_len = long_lora_indices.shape[-1]
143 - # Contain length of indices tensors. Used to index into each tensor.
144 - indices_len = [
145 - base_indices.shape[-1], sampler_indices.shape[-1],
146 - sampler_indices_padded.shape[-1], embeddings_indices.shape[-1]
147 - ]
148 - if long_lora_indices_len is not None:
149 - indices_len.append(long_lora_indices_len)
150 -
151 - return (base_indices, sampler_indices, sampler_indices_padded,
152 - embeddings_indices, long_lora_indices, indices_len)
153 -
154 -
47 def get_lora_id():
48 global _GLOBAL_LORA_ID
49 _GLOBAL_LORA_ID += 1
@@ -422,29 +314,12 @@ class LoRAModelManager(AdapterModelManager):
314 self.lora_index_to_id: List[Optional[int]] = [None] * self.lora_slots
315 self.vocab_size = vocab_size
316 self.long_lora_context: Optional[LongContextLoRAContext] = None
425 - self.base_indices = torch.empty(self.max_num_batched_tokens,
426 - dtype=torch.long,
427 - device="cuda")
428 - self.sampler_indices = torch.empty(self.max_num_batched_tokens,
429 - dtype=torch.long,
430 - device="cuda")
431 - self.sampler_indices_padded = torch.empty(self.max_num_batched_tokens,
432 - dtype=torch.long,
433 - device="cuda")
434 - self.embeddings_indices = torch.empty(2,
435 - self.max_num_batched_tokens,
436 - dtype=torch.long,
437 - device="cuda")
438 - self.long_lora_indices = torch.empty(self.max_num_batched_tokens,
439 - dtype=torch.long,
440 - device="cuda")
317 + self.punica_wrapper = PunicaWrapper(max_num_batched_tokens,
318 + max_batches=self.max_num_seqs,
319 + device="cuda")
320 # Scaling factor -> offset to the sin_cos_cache to it.
321 # Used for long context lora.
322 self.scaling_factor_to_offset: Dict[float, int] = {}
444 - # 4 is the number of indicies tensors defined above
445 - # base_indices, sampler_indices, sampler_indices_padded,
446 - # embeddings_indices
447 - self.indices_len: List[Optional[int]] = [None] * 4
323 super().__init__(model)
324 if hasattr(self.model, "supported_lora_modules"):
325 self.supported_lora_modules = copy.deepcopy(
@@ -536,28 +411,16 @@ class LoRAModelManager(AdapterModelManager):
411 "Pinning is not supported in LoRAModelManager."
412 "Use LRUCacheLoRAModelManager for pinning") # type: ignore
413
539 - # TODO see if this can be vectorized
414 def _set_adapter_mapping(self, mapping: LoRAMapping) -> None:
541 - (base_indices, sampler_indices, sampler_indices_padded,
542 - embeddings_indices, long_lora_offsets_tensor,
543 - indices_len) = convert_mapping(mapping, self.lora_index_to_id,
544 - self.lora_slots + 1, self.vocab_size,
545 - self.lora_config.lora_extra_vocab_size,
546 - self.long_lora_context)
547 - self.base_indices[:base_indices.shape[0]].copy_(base_indices)
548 - self.sampler_indices[:sampler_indices.shape[0]].copy_(sampler_indices)
549 - self.sampler_indices_padded[:sampler_indices_padded.shape[0]].copy_(
550 - sampler_indices_padded)
551 - self.embeddings_indices[:embeddings_indices.
552 - shape[0], :embeddings_indices.shape[1]].copy_(
553 - embeddings_indices)
554 - if long_lora_offsets_tensor is not None:
555 - self.long_lora_indices[:long_lora_offsets_tensor.shape[0]].copy_(
556 - long_lora_offsets_tensor)
557 - else:
558 - self.long_lora_indices.zero_()
559 - # Maintain the reference
560 - self.indices_len[:] = indices_len
415 + # update lora states
416 + self.punica_wrapper.update_metadata(
417 + mapping,
418 + self.lora_index_to_id,
419 + self.lora_slots + 1,
420 + self.vocab_size,
421 + self.lora_config.lora_extra_vocab_size,
422 + self.long_lora_context,
423 + )
424
425 def remove_all_adapters(self):
426 """Remove all LoRAModels from the manager."""
@@ -595,10 +458,8 @@ class LoRAModelManager(AdapterModelManager):
458 self.model.config))
459 self.register_module(module_name, new_module)
460 self._register_packed_modules(module_name)
598 - new_module.set_mapping(self.base_indices, self.sampler_indices,
599 - self.sampler_indices_padded,
600 - self.embeddings_indices,
601 - self.long_lora_indices, self.indices_len)
461 + # All lora layers share the same punica_wrapper based on reference.
462 + new_module.set_mapping(self.punica_wrapper)
463
464 def register_module(self, module_name: str, module: "BaseLayerWithLoRA"):
465 assert isinstance(module, BaseLayerWithLoRA)
vllm/lora/ops/__init__.py
vllm/lora/ops/bgmv_expand.py new
+169
@@ -0,0 +1,169 @@
1 +"""
2 +Based on:
3 +Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023).
4 +Punica: Multi-Tenant LoRA Serving.
5 +https://arxiv.org/abs/2310.18547
6 +"""
7 +
8 +from typing import Dict, Optional
9 +
10 +import torch
11 +import triton
12 +import triton.language as tl
13 +
14 +from .utils import get_lora_op_configs
15 +
16 +
17 +@triton.jit
18 +def _bgmv_expand_kernel(
19 + input_ptr,
20 + lora_ptr,
21 + out_ptr,
22 + N,
23 + K,
24 + lora_indices,
25 + xm_stride,
26 + xk_stride,
27 + l0_stride,
28 + lora_k_stride,
29 + lora_n_stride,
30 + cm_stride,
31 + cn_stride,
32 + BLOCK_N: tl.constexpr,
33 + BLOCK_K: tl.constexpr,
34 + SPLIT_N: tl.constexpr,
35 + EVEN_K: tl.constexpr,
36 + ADD_INPUTS: tl.constexpr,
37 + CAST_TYPE: tl.constexpr,
38 +):
39 + """
40 + GroupGEMV, additionally, introducing SPLIT_N can improve large hidden_size's
41 + performance
42 + """
43 + pid_sn = tl.program_id(axis=0)
44 + cur_batch = tl.program_id(axis=1)
45 + lora_index = tl.load(lora_indices + cur_batch)
46 + if lora_index == -1:
47 + return
48 + offset_k = tl.arange(0, BLOCK_K)
49 + offset_n = tl.arange(0, BLOCK_N)
50 + if EVEN_K:
51 + tiled_a = tl.load(input_ptr + cur_batch * xm_stride +
52 + offset_k * xk_stride, ) # [BLOCK_K]
53 + else:
54 + tiled_a = tl.load(
55 + input_ptr + cur_batch * xm_stride + offset_k * xk_stride,
56 + mask=offset_k < K,
57 + other=0,
58 + ) # [BLOCK_K]
59 + # N must be divisible by SPLIT_N
60 + split_n_length = tl.cdiv(N, SPLIT_N)
61 + if CAST_TYPE:
62 + tiled_a = tiled_a.to(lora_ptr.dtype.element_ty)
63 + # sliding to next row-block
64 + b_ptr = (lora_ptr + l0_stride * lora_index +
65 + pid_sn * split_n_length * lora_k_stride)
66 + c_ptr = out_ptr + cur_batch * cm_stride + pid_sn * split_n_length
67 + for n in range(0, split_n_length, BLOCK_N):
68 + current_n = n + offset_n
69 + current_n_c = tl.max_contiguous(current_n, BLOCK_N)
70 + b_ptr_mask = (current_n[:, None] < split_n_length) & (offset_k[None, :]
71 + < K)
72 + c_mask = current_n < split_n_length
73 + tiled_b = tl.load(
74 + b_ptr + current_n_c[:, None] * lora_k_stride +
75 + offset_k[None, :] * lora_n_stride,
76 + mask=b_ptr_mask,
77 + other=0.0,
78 + ) # [BLOCK_N,BLOCK_K]
79 + if ADD_INPUTS:
80 + tiled_out = tl.load(c_ptr + current_n * cn_stride, mask=c_mask)
81 + accumulator = tl.sum(tiled_a * tiled_b, 1) + tiled_out
82 + else:
83 + accumulator = tl.sum(tiled_a * tiled_b, 1)
84 +
85 + tl.store(c_ptr + current_n * cn_stride, accumulator, mask=c_mask)
86 +
87 +
88 +@torch.inference_mode()
89 +def bgmv_expand(
90 + inputs: torch.Tensor,
91 + lora_b_weights: torch.Tensor,
92 + output_tensor: torch.Tensor,
93 + lora_indices_tensor: torch.Tensor,
94 + add_inputs: bool = True,
95 + override_config: Optional[Dict[str, int]] = None,
96 +):
97 + """
98 + Args:
99 + inputs (torch.Tensor): input tensor
100 + lora_b_weights (torch.Tensor): lora'a weight
101 + output_tensor (torch.Tensor): output tensor
102 + lora_indices_tensor (torch.Tensor): (batch_size,). The LoRA index
103 + corresponding to each batch, An index of -1 means no lora should be
104 + applied.
105 + batches (int): batch size
106 + add_inputs (bool, optional): Defaults to False. adds the final lora
107 + results to the output.
108 + override_config (Optional[Dict[str, int]], optional): Defaults to None.
109 + Triton grid config
110 + """
111 +
112 + assert inputs.dtype in [torch.float16, torch.bfloat16, torch.float32]
113 + assert lora_b_weights.dtype in [
114 + torch.float16,
115 + torch.bfloat16,
116 + ]
117 + assert inputs.size(1) == lora_b_weights.size(-1)
118 +
119 + assert inputs.is_contiguous()
120 + assert output_tensor.is_contiguous()
121 +
122 + if lora_b_weights.ndim == 4: # shape:(lora_num,1,size,rank)
123 + assert lora_b_weights.size(1) == 1
124 + lora_b_weights = lora_b_weights.squeeze(dim=1)
125 + else:
126 + assert lora_b_weights.ndim == 3 # shape:(lora_num,size,rank)
127 + assert lora_b_weights.is_contiguous()
128 +
129 + # TODO tuning this config
130 + N, K = lora_b_weights.shape[-2:] # K= rank,N=hidden_size
131 + BLOCK_K = triton.next_power_of_2(K)
132 + EVEN_K = K % BLOCK_K == 0
133 + ADD_INPUTS = add_inputs
134 + CAST_TYPE = False
135 + if inputs.dtype == torch.float32 and lora_b_weights.dtype in [
136 + torch.float16,
137 + torch.bfloat16,
138 + ]:
139 + CAST_TYPE = True
140 + batches = lora_indices_tensor.size(0)
141 + if override_config:
142 + config = override_config
143 + else:
144 + config = get_lora_op_configs("expand", batches, N)
145 + grid = lambda META: (
146 + META["SPLIT_N"],
147 + batches,
148 + )
149 + _bgmv_expand_kernel[grid](
150 + inputs,
151 + lora_b_weights,
152 + output_tensor,
153 + N,
154 + K,
155 + lora_indices_tensor,
156 + inputs.stride(0),
157 + inputs.stride(1),
158 + lora_b_weights.stride(0),
159 + lora_b_weights.stride(1),
160 + lora_b_weights.stride(2),
161 + output_tensor.stride(0),
162 + output_tensor.stride(1),
163 + BLOCK_K=BLOCK_K,
164 + EVEN_K=EVEN_K,
165 + ADD_INPUTS=ADD_INPUTS,
166 + CAST_TYPE=CAST_TYPE,
167 + **config,
168 + )
169 + return
vllm/lora/ops/bgmv_expand_slice.py new
+182
@@ -0,0 +1,182 @@
1 +"""
2 +Based on:
3 +Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023).
4 +Punica: Multi-Tenant LoRA Serving.
5 +https://arxiv.org/abs/2310.18547
6 +"""
7 +
8 +from typing import Dict, Optional
9 +
10 +import torch
11 +import triton
12 +import triton.language as tl
13 +
14 +from .utils import get_lora_op_configs
15 +
16 +
17 +@triton.jit
18 +def _bgmv_expand_slice_kernel(
19 + input_ptr,
20 + lora_ptr,
21 + out_ptr,
22 + N,
23 + K,
24 + lora_indices,
25 + xm_stride,
26 + xk_stride,
27 + l0_stride,
28 + lora_k_stride,
29 + lora_n_stride,
30 + cm_stride,
31 + cn_stride,
32 + slice_offset,
33 + BLOCK_N: tl.constexpr,
34 + BLOCK_K: tl.constexpr,
35 + SPLIT_N: tl.constexpr,
36 + EVEN_K: tl.constexpr,
37 + ADD_INPUTS: tl.constexpr,
38 + CAST_TYPE: tl.constexpr,
39 +):
40 + """
41 + GroupGEMV, additionally, introducing SPLIT_N can improve large hidden_size's
42 + performance
43 + """
44 + pid_sn = tl.program_id(axis=0)
45 + cur_batch = tl.program_id(axis=1)
46 + lora_index = tl.load(lora_indices + cur_batch)
47 + if lora_index == -1:
48 + return
49 + offset_k = tl.arange(0, BLOCK_K)
50 + offset_n = tl.arange(0, BLOCK_N)
51 + if EVEN_K:
52 + tiled_a = tl.load(input_ptr + cur_batch * xm_stride +
53 + offset_k * xk_stride, ) # [BLOCK_K]
54 + else:
55 + tiled_a = tl.load(
56 + input_ptr + cur_batch * xm_stride + offset_k * xk_stride,
57 + mask=offset_k < K,
58 + other=0,
59 + ) # [BLOCK_K]
60 + # N must be divisible by SPLIT_N
61 + split_n_length = tl.cdiv(N, SPLIT_N)
62 + if CAST_TYPE:
63 + tiled_a = tiled_a.to(lora_ptr.dtype.element_ty)
64 + # sliding to next row-block
65 + b_ptr = (lora_ptr + l0_stride * lora_index +
66 + pid_sn * split_n_length * lora_k_stride)
67 + c_ptr = (out_ptr + cur_batch * cm_stride + pid_sn * split_n_length +
68 + slice_offset * cn_stride)
69 +
70 + for n in range(0, split_n_length, BLOCK_N):
71 + current_n = n + offset_n
72 + b_ptr_mask = (current_n[:, None] < split_n_length) & (offset_k[None, :]
73 + < K)
74 + c_mask = current_n < split_n_length
75 + tiled_b = tl.load(
76 + b_ptr + current_n[:, None] * lora_k_stride +
77 + offset_k[None, :] * lora_n_stride,
78 + mask=b_ptr_mask,
79 + other=0.0,
80 + ) # [BLOCK_N,BLOCK_K]
81 +
82 + if ADD_INPUTS:
83 + tiled_out = tl.load(c_ptr + current_n * cn_stride, mask=c_mask)
84 + accumulator = tl.sum(tiled_a * tiled_b, 1) + tiled_out
85 + else:
86 + accumulator = tl.sum(tiled_a * tiled_b, 1)
87 +
88 + tl.store(c_ptr + current_n * cn_stride, accumulator, mask=c_mask)
89 +
90 +
91 +@torch.inference_mode()
92 +def bgmv_expand_slice(
93 + inputs: torch.Tensor,
94 + lora_b_weights: torch.Tensor,
95 + output_tensor: torch.Tensor,
96 + lora_indices_tensor: torch.Tensor,
97 + slice_offset: int,
98 + slice_size: int,
99 + add_inputs: bool = True,
100 + override_config: Optional[Dict[str, int]] = None,
101 +):
102 + """
103 + Args:
104 + inputs (torch.Tensor): input tensor
105 + lora_b_weights (torch.Tensor): lora'b weight
106 + output_tensor (torch.Tensor): output tensor
107 + lora_indices_tensor (torch.Tensor): (batch_size,). The LoRA index
108 + corresponding to each batch, An index of -1 means no lora should be
109 + applied.
110 + slice_offst (int): output_tensor's offst
111 + slice_size (int): current output_tensor's size
112 + batches (int): batch size
113 + add_inputs (bool, optional): Defaults to False.
114 + override_config (Optional[Dict[str, int]], optional): Defaults to None.
115 + Triton grid config
116 + """
117 +
118 + assert inputs.dtype in [torch.float16, torch.bfloat16, torch.float32]
119 + assert lora_b_weights.dtype in [
120 + torch.float16,
121 + torch.bfloat16,
122 + ]
123 + assert inputs.size(1) == lora_b_weights.size(-1)
124 +
125 + assert slice_size == lora_b_weights.size(-2)
126 + assert inputs.is_contiguous()
127 + assert output_tensor.is_contiguous()
128 +
129 + if lora_b_weights.ndim == 4: # shape:(lora_num,1,size,rank)
130 + assert lora_b_weights.size(1) == 1
131 + lora_b_weights = lora_b_weights.squeeze(dim=1)
132 + else:
133 + assert lora_b_weights.ndim == 3 # shape:(lora_num,size,rank)
134 +
135 + assert lora_b_weights.is_contiguous()
136 +
137 + # TODO tuning this config
138 +
139 + N, K = lora_b_weights.shape[-2:] # K= rank,N=hidden_size
140 + BLOCK_K = triton.next_power_of_2(K)
141 + EVEN_K = K % BLOCK_K == 0
142 + ADD_INPUTS = add_inputs
143 + CAST_TYPE = False
144 + if inputs.dtype == torch.float32 and lora_b_weights.dtype in [
145 + torch.float16,
146 + torch.bfloat16,
147 + ]:
148 + CAST_TYPE = True
149 +
150 + batches = lora_indices_tensor.size(0)
151 +
152 + if override_config:
153 + config = override_config
154 + else:
155 + config = get_lora_op_configs("expand", batches, N)
156 +
157 + grid = lambda META: (
158 + META["SPLIT_N"],
159 + batches,
160 + )
161 + _bgmv_expand_slice_kernel[grid](
162 + inputs,
163 + lora_b_weights,
164 + output_tensor,
165 + N,
166 + K,
167 + lora_indices_tensor,
168 + inputs.stride(0),
169 + inputs.stride(1),
170 + lora_b_weights.stride(0),
171 + lora_b_weights.stride(1),
172 + lora_b_weights.stride(2),
173 + output_tensor.stride(0),
174 + output_tensor.stride(1),
175 + slice_offset,
176 + BLOCK_K=BLOCK_K,
177 + EVEN_K=EVEN_K,
178 + ADD_INPUTS=ADD_INPUTS,
179 + CAST_TYPE=CAST_TYPE,
180 + **config,
181 + )
182 + return
vllm/lora/ops/bgmv_shrink.py new
+150
@@ -0,0 +1,150 @@
1 +"""
2 +Based on:
3 +Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023).
4 +Punica: Multi-Tenant LoRA Serving.
5 +https://arxiv.org/abs/2310.18547
6 +"""
7 +
8 +from typing import Dict, Optional
9 +
10 +import torch
11 +import triton
12 +import triton.language as tl
13 +
14 +from .utils import get_lora_op_configs
15 +
16 +
17 +@triton.jit
18 +def _bgmv_shrink_kernel(
19 + input_ptr,
20 + lora_ptr,
21 + out_ptr,
22 + N,
23 + K,
24 + lora_indices,
25 + scaling,
26 + xm_stride,
27 + xk_stride,
28 + l0_stride,
29 + lora_k_stride,
30 + lora_n_stride,
31 + cm_stride,
32 + cn_stride,
33 + BLOCK_N: tl.constexpr,
34 + BLOCK_K: tl.constexpr,
35 + SPLIT_K: tl.constexpr,
36 +):
37 + """
38 + GroupGEMV, additionally, introducing SPLIT-K can improve large hidden_size's
39 + performance
40 + """
41 + pid_sk = tl.program_id(axis=0)
42 + cur_batch = tl.program_id(axis=1)
43 + lora_index = tl.load(lora_indices + cur_batch)
44 + if lora_index == -1:
45 + return
46 +
47 + offset_n = tl.arange(0, BLOCK_N)
48 + offset_k = tl.arange(0, BLOCK_K) + pid_sk * BLOCK_K
49 + a_ptr = input_ptr + cur_batch * xm_stride
50 + b_ptr = lora_ptr + l0_stride * lora_index
51 + accumulator = tl.zeros((BLOCK_N, ), dtype=tl.float32)
52 + for k in range(0, K, BLOCK_K * SPLIT_K):
53 + current_k = k + offset_k
54 + current_k_c = tl.max_contiguous(current_k, BLOCK_K)
55 + tiled_a = tl.load(
56 + a_ptr + current_k_c,
57 + mask=current_k < K,
58 + other=0.0,
59 + ) # [BLOCK_K]
60 + b_ptr_mask = (offset_n[:, None] < N) & (current_k[None, :] < K)
61 +
62 + tiled_b = tl.load(
63 + b_ptr + offset_n[:, None] * lora_k_stride +
64 + current_k[None, :] * lora_n_stride,
65 + mask=b_ptr_mask,
66 + other=0.0,
67 + ) # [BLOCK_N,BLOCK_K]
68 +
69 + accumulator += tl.sum(tiled_a * tiled_b, 1)
70 + accumulator *= scaling
71 + offset_cn = tl.arange(0, BLOCK_N)
72 + c_ptr = out_ptr + cur_batch * cm_stride + offset_cn * cn_stride
73 + c_mask = offset_cn < N
74 + if SPLIT_K == 1:
75 + tl.store(c_ptr, accumulator, mask=c_mask)
76 + else:
77 + tl.atomic_add(c_ptr, accumulator, mask=c_mask)
78 +
79 +
80 +@torch.inference_mode()
81 +def bgmv_shrink(
82 + inputs: torch.Tensor,
83 + lora_a_weights: torch.Tensor,
84 + output_tensor: torch.Tensor,
85 + lora_indices_tensor: torch.Tensor,
86 + scaling: float = 1.0,
87 + override_config: Optional[Dict[str, int]] = None,
88 +):
89 + """
90 + Args:
91 + inputs (torch.Tensor): input tensor
92 + lora_a_weights (torch.Tensor): lora'a weight
93 + output_tensor (torch.Tensor): output tensor
94 + lora_indices_tensor (torch.Tensor): (batch_size,). The LoRA index
95 + corresponding to each batch. An index of -1 means no lora should be
96 + applied.
97 + batches (int): batch size
98 + scaling (float): Scaling factor.
99 + override_config (Optional[Dict[str, int]], optional): Defaults to None.
100 + Triton grid config
101 + """
102 + assert inputs.dtype == lora_a_weights.dtype
103 + assert inputs.dtype in [torch.float16, torch.bfloat16]
104 + assert lora_a_weights.dtype in [
105 + torch.float16,
106 + torch.bfloat16,
107 + ]
108 + assert inputs.size(1) == lora_a_weights.size(-1)
109 + assert inputs.is_contiguous()
110 +
111 + if lora_a_weights.ndim == 4: # shape:(lora_num,1,rank, size)
112 + assert lora_a_weights.size(1) == 1
113 + lora_a_weights = lora_a_weights.squeeze(dim=1)
114 + else:
115 + assert lora_a_weights.ndim == 3 # shape:(lora_num,rank, size)
116 + assert lora_a_weights.is_contiguous()
117 + assert output_tensor.is_contiguous()
118 + # TODO tuning this config
119 + batches = lora_indices_tensor.size(0)
120 + N, K = lora_a_weights.shape[-2:] # K=hidden_size,N=rank
121 + BLOCK_N = triton.next_power_of_2(N)
122 + if override_config:
123 + config = override_config
124 + else:
125 + # First try to load optimal config from the file
126 + config = get_lora_op_configs("bgmv_shrink", batches, K)
127 +
128 + grid = lambda META: (
129 + META["SPLIT_K"],
130 + batches,
131 + )
132 + _bgmv_shrink_kernel[grid](
133 + inputs,
134 + lora_a_weights,
135 + output_tensor,
136 + N,
137 + K,
138 + lora_indices_tensor,
139 + scaling,
140 + inputs.stride(0),
141 + inputs.stride(1),
142 + lora_a_weights.stride(0),
143 + lora_a_weights.stride(1),
144 + lora_a_weights.stride(2),
145 + output_tensor.stride(0),
146 + output_tensor.stride(1),
147 + BLOCK_N=BLOCK_N,
148 + **config,
149 + )
150 + return
vllm/lora/ops/sgmv_expand.py new
+192
@@ -0,0 +1,192 @@
1 +"""
2 +Based on:
3 +Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023).
4 +Punica: Multi-Tenant LoRA Serving.
5 +https://arxiv.org/abs/2310.18547
6 +"""
7 +
8 +import torch
9 +import triton
10 +import triton.language as tl
11 +
12 +from vllm.triton_utils import libentry
13 +
14 +
15 +@libentry()
16 +@triton.jit
17 +def _sgmv_expand_kernel(
18 + input_ptr,
19 + lora_ptr,
20 + out_ptr,
21 + N,
22 + K,
23 + b_seq_start_loc,
24 + seq_lens,
25 + lora_indices,
26 + xm_stride,
27 + xk_stride, # 1
28 + l0_stride, # hidden_size*max_rank
29 + lora_k_stride,
30 + lora_n_stride,
31 + cm_stride,
32 + cn_stride,
33 + BLOCK_M: tl.constexpr,
34 + BLOCK_N: tl.constexpr,
35 + BLOCK_K: tl.constexpr,
36 + EVEN_K: tl.constexpr,
37 + ADD_INPUTS: tl.constexpr,
38 + CAST_TYPE: tl.constexpr,
39 +):
40 + """
41 + The sgmv's expand triton kernel is based on GroupGEMM.
42 + """
43 + pid = tl.program_id(axis=0)
44 + cur_batch = tl.program_id(axis=1)
45 + cta_n_num = tl.cdiv(N, BLOCK_N)
46 + pid_m = pid // cta_n_num
47 + pid_n = pid % cta_n_num
48 + M = tl.load(seq_lens + cur_batch)
49 + if pid_m * BLOCK_M > M:
50 + return
51 + lora_index = tl.load(lora_indices + cur_batch)
52 + if lora_index == -1:
53 + return
54 + cur_seq_start = tl.load(b_seq_start_loc + cur_batch)
55 + offset_m = tl.arange(0, BLOCK_M) + pid_m * BLOCK_M
56 + offset_n = tl.arange(0, BLOCK_N) + pid_n * BLOCK_N
57 + offset_k = tl.arange(0, BLOCK_K)
58 + ram = tl.max_contiguous(tl.multiple_of(offset_m % M, BLOCK_M), BLOCK_M)
59 + rbn = tl.max_contiguous(tl.multiple_of(offset_n % N, BLOCK_N), BLOCK_N)
60 +
61 + a_ptr = (input_ptr + cur_seq_start * xm_stride + ram[:, None] * xm_stride +
62 + offset_k[None, :] * xk_stride, )
63 + b_ptr = (lora_ptr + l0_stride * lora_index +
64 + offset_k[:, None] * lora_n_stride + rbn[None, :] * lora_k_stride)
65 + accumulator = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
66 + for k in range(tl.cdiv(K, BLOCK_K)):
67 + if EVEN_K:
68 + tiled_a = tl.load(a_ptr)
69 + tiled_b = tl.load(b_ptr)
70 + else:
71 + tiled_a = tl.load(a_ptr,
72 + mask=offset_k[None, :] < K - k * BLOCK_K,
73 + other=0)
74 + tiled_b = tl.load(b_ptr,
75 + mask=offset_k[:, None] < K - k * BLOCK_K,
76 + other=0)
77 + if CAST_TYPE:
78 + tiled_a = tiled_a.to(lora_ptr.dtype.element_ty)
79 + accumulator += tl.dot(
80 + tiled_a,
81 + tiled_b,
82 + )
83 + a_ptr += BLOCK_K * xk_stride
84 + b_ptr += BLOCK_K * lora_n_stride
85 + tiled_c = accumulator.to(lora_ptr.dtype.element_ty)
86 + offset_cm = cur_seq_start + tl.arange(0, BLOCK_M) + pid_m * BLOCK_M
87 + offset_cn = tl.arange(0, BLOCK_N) + pid_n * BLOCK_N
88 + c_ptr = (out_ptr + offset_cm[:, None] * cm_stride +
89 + offset_cn[None, :] * cn_stride)
90 + M = tl.load(seq_lens + cur_batch)
91 + c_mask = (offset_cm[:, None] <
92 + (cur_seq_start + M)) & (offset_cn[None, :] < N)
93 + if ADD_INPUTS:
94 + tiled_out = tl.load(c_ptr, mask=c_mask)
95 + tiled_c += tiled_out
96 + tl.store(c_ptr, tiled_c, mask=c_mask)
97 +
98 +
99 +@torch.inference_mode()
100 +def sgmv_expand(
101 + inputs: torch.Tensor,
102 + lora_b_weights: torch.Tensor,
103 + output_tensor: torch.Tensor,
104 + b_seq_start_loc: torch.Tensor,
105 + seq_len_tensor: torch.Tensor,
106 + lora_indices_tensor: torch.Tensor,
107 + batches: int,
108 + max_seq_length: int,
109 + add_inputs: bool = False,
110 +):
111 + """
112 + Args:
113 + inputs (torch.Tensor): input tensor
114 + lora_b_weights (torch.Tensor): lora'a weight
115 + output_tensor (torch.Tensor): output tensor
116 + b_seq_start_loc (torch.Tensor): (batch_size,). The cumulative
117 + sequence lengths of the sequences in the batch, used to index
118 + into sequence. E.g.,if the sequence length is [4, 6], it is
119 + [0, 4, 10].
120 + seq_len_tensor (torch.Tensor): (batch_size,). record the sequence
121 + length of the sequences in the batch
122 + lora_indices_tensor (torch.Tensor): (batch_size,). The LoRA index
123 + corresponding to each batch. An index of -1 means no lora should be
124 + applied.
125 + batches (int): batch size
126 + max_seq_length (int): The max sequence lengths of the sequences
127 + in the batch
128 + add_inputs (bool, optional): Defaults to False. adds the final lora
129 + results to the output.
130 + """
131 +
132 + assert inputs.dtype in [torch.float16, torch.bfloat16, torch.float32]
133 + assert lora_b_weights.dtype in [
134 + torch.float16,
135 + torch.bfloat16,
136 + ]
137 + assert inputs.size(1) == lora_b_weights.size(-1)
138 + assert b_seq_start_loc.size(0) == batches
139 + assert lora_indices_tensor.size(0) == batches
140 + assert inputs.is_contiguous()
141 + assert output_tensor.is_contiguous()
142 +
143 + if lora_b_weights.ndim == 4: # shape:(lora_num,1,size,rank)
144 + assert lora_b_weights.size(1) == 1
145 + lora_b_weights = lora_b_weights.squeeze(dim=1)
146 + else:
147 + assert lora_b_weights.ndim == 3 # shape:(lora_num,size,rank)
148 +
149 + assert lora_b_weights.is_contiguous()
150 +
151 + # TODO tuning this config
152 +
153 + N, K = lora_b_weights.shape[-2:] # K= rank,N=hidden_size
154 + BLOCK_M = 32
155 + BLOCK_N = 32
156 + BLOCK_K = 16
157 + EVEN_K = K % BLOCK_K == 0
158 + ADD_INPUTS = add_inputs
159 + CAST_TYPE = False
160 + if inputs.dtype == torch.float32 and lora_b_weights.dtype in [
161 + torch.float16,
162 + torch.bfloat16,
163 + ]:
164 + CAST_TYPE = True
165 + grid = (
166 + triton.cdiv(max_seq_length, BLOCK_M) * triton.cdiv(N, BLOCK_N),
167 + batches,
168 + )
169 + _sgmv_expand_kernel[grid](
170 + inputs,
171 + lora_b_weights,
172 + output_tensor,
173 + N,
174 + K,
175 + b_seq_start_loc,
176 + seq_len_tensor,
177 + lora_indices_tensor,
178 + inputs.stride(0),
179 + inputs.stride(1),
180 + lora_b_weights.stride(0),
181 + lora_b_weights.stride(1),
182 + lora_b_weights.stride(2),
183 + output_tensor.stride(0),
184 + output_tensor.stride(1),
185 + BLOCK_M,
186 + BLOCK_N,
187 + BLOCK_K,
188 + EVEN_K,
189 + ADD_INPUTS,
190 + CAST_TYPE,
191 + )
192 + return
vllm/lora/ops/sgmv_expand_slice.py new
+205
@@ -0,0 +1,205 @@
1 +"""
2 +Based on:
3 +Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023).
4 +Punica: Multi-Tenant LoRA Serving.
5 +https://arxiv.org/abs/2310.18547
6 +"""
7 +
8 +import torch
9 +import triton
10 +import triton.language as tl
11 +
12 +from vllm.triton_utils import libentry
13 +
14 +
15 +@libentry()
16 +@triton.jit
17 +def _sgmv_expand_slice_kernel(
18 + input_ptr,
19 + lora_ptr,
20 + out_ptr,
21 + N,
22 + K,
23 + b_seq_start_loc,
24 + seq_lens,
25 + lora_indices,
26 + xm_stride,
27 + xk_stride, # 1
28 + l0_stride, # hidden_size*max_rank
29 + lora_k_stride,
30 + lora_n_stride,
31 + cm_stride,
32 + cn_stride,
33 + slice_offset,
34 + BLOCK_M: tl.constexpr,
35 + BLOCK_N: tl.constexpr,
36 + BLOCK_K: tl.constexpr,
37 + EVEN_K: tl.constexpr,
38 + ADD_INPUTS: tl.constexpr,
39 + CAST_TYPE: tl.constexpr,
40 +):
41 + """
42 +
43 + Similar to the 'sgmv_expand' operator, but with an added parameter
44 + 'slice_offset'. The reason for not reusing the 'sgmv_expand' operator
45 + might be that in the future, we could implement a fusion operator to
46 + achieve the current functionality instead of having to call it multiple
47 + times.
48 + """
49 + pid = tl.program_id(axis=0)
50 + cur_batch = tl.program_id(axis=1)
51 + cta_n_num = tl.cdiv(N, BLOCK_N)
52 + pid_m = pid // cta_n_num
53 + pid_n = pid % cta_n_num
54 + M = tl.load(seq_lens + cur_batch)
55 + if pid_m * BLOCK_M > M:
56 + return
57 + lora_index = tl.load(lora_indices + cur_batch)
58 + if lora_index == -1:
59 + return
60 + cur_seq_start = tl.load(b_seq_start_loc + cur_batch)
61 + offset_m = tl.arange(0, BLOCK_M) + pid_m * BLOCK_M
62 + offset_n = tl.arange(0, BLOCK_N) + pid_n * BLOCK_N
63 + offset_k = tl.arange(0, BLOCK_K)
64 + ram = tl.max_contiguous(tl.multiple_of(offset_m % M, BLOCK_M), BLOCK_M)
65 + rbn = tl.max_contiguous(tl.multiple_of(offset_n % N, BLOCK_N), BLOCK_N)
66 +
67 + a_ptr = (input_ptr + cur_seq_start * xm_stride + ram[:, None] * xm_stride +
68 + offset_k[None, :] * xk_stride, )
69 + b_ptr = (lora_ptr + l0_stride * lora_index +
70 + offset_k[:, None] * lora_n_stride + rbn[None, :] * lora_k_stride)
71 + accumulator = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
72 + for k in range(tl.cdiv(K, BLOCK_K)):
73 + if EVEN_K:
74 + tiled_a = tl.load(a_ptr)
75 + tiled_b = tl.load(b_ptr)
76 + else:
77 + tiled_a = tl.load(a_ptr,
78 + mask=offset_k[None, :] < K - k * BLOCK_K,
79 + other=0)
80 + tiled_b = tl.load(b_ptr,
81 + mask=offset_k[:, None] < K - k * BLOCK_K,
82 + other=0)
83 + if CAST_TYPE:
84 + tiled_a = tiled_a.to(lora_ptr.dtype.element_ty)
85 + accumulator += tl.dot(
86 + tiled_a,
87 + tiled_b,
88 + )
89 + a_ptr += BLOCK_K * xk_stride
90 + b_ptr += BLOCK_K * lora_n_stride
91 + tiled_c = accumulator.to(lora_ptr.dtype.element_ty)
92 + offset_cm = cur_seq_start + tl.arange(0, BLOCK_M) + pid_m * BLOCK_M
93 + offset_cn = tl.arange(0, BLOCK_N) + pid_n * BLOCK_N + slice_offset
94 + c_ptr = (out_ptr + offset_cm[:, None] * cm_stride +
95 + offset_cn[None, :] * cn_stride)
96 + M = tl.load(seq_lens + cur_batch)
97 + c_mask = (offset_cm[:, None] < (cur_seq_start + M)) & (offset_cn[None, :] <
98 + (slice_offset + N))
99 + if ADD_INPUTS:
100 + tiled_out = tl.load(c_ptr, mask=c_mask)
101 + tiled_c += tiled_out
102 + tl.store(c_ptr, tiled_c, mask=c_mask)
103 +
104 +
105 +@torch.inference_mode()
106 +def sgmv_expand_slice(
107 + inputs: torch.Tensor,
108 + lora_b_weights: torch.Tensor,
109 + output_tensor: torch.Tensor,
110 + b_seq_start_loc: torch.Tensor,
111 + seq_len_tensor: torch.Tensor,
112 + lora_indices_tensor: torch.Tensor,
113 + batches: int,
114 + max_seq_length: int,
115 + slice_offset: int,
116 + slice_size: int,
117 + add_inputs: bool = False,
118 +):
119 + """_summary_
120 +
121 + Args:
122 + inputs (torch.Tensor): input tensor
123 + lora_b_weights (torch.Tensor): lora'a weight
124 + output_tensor (torch.Tensor): output tensor
125 + b_seq_start_loc (torch.Tensor): (batch_size,). The cumulative
126 + sequence lengths of the sequences in the batch, used to index
127 + into sequence. E.g.,if the sequence length is [4, 6], it is
128 + [0, 4, 10].
129 + seq_len_tensor (torch.Tensor): (batch_size,). record the sequence
130 + length of the sequences in the batch
131 + lora_indices_tensor (torch.Tensor): (batch_size,). The LoRA index
132 + corresponding to each batch. An index of -1 means no lora should be
133 + applied.
134 + batches (int): batch size
135 + max_seq_length (int): The max sequence lengths of the sequences
136 + in the batch
137 + slice_offst (int): output_tensor's offst
138 + slice_size (int): current output_tensor's size
139 + add_inputs (bool, optional): Defaults to False. adds the final lora
140 + results to the output..
141 + """
142 +
143 + assert inputs.dtype in [torch.float16, torch.bfloat16, torch.float32]
144 + assert lora_b_weights.dtype in [
145 + torch.float16,
146 + torch.bfloat16,
147 + ]
148 + assert inputs.size(1) == lora_b_weights.size(-1)
149 + assert b_seq_start_loc.size(0) == batches
150 + assert lora_indices_tensor.size(0) == batches
151 + assert slice_size == lora_b_weights.size(-2)
152 + assert inputs.is_contiguous()
153 + assert output_tensor.is_contiguous()
154 +
155 + if lora_b_weights.ndim == 4: # shape:(lora_num,1,size,rank)
156 + assert lora_b_weights.size(1) == 1
157 + lora_b_weights = lora_b_weights.squeeze(dim=1)
158 + else:
159 + assert lora_b_weights.ndim == 3 # shape:(lora_num,size,rank)
160 +
161 + assert lora_b_weights.is_contiguous()
162 +
163 + # TODO tuning this config
164 + N, K = lora_b_weights.shape[-2:] # K= rank,N=hidden_size
165 +
166 + BLOCK_M = 32
167 + BLOCK_N = 32
168 + BLOCK_K = 16
169 + EVEN_K = K % BLOCK_K == 0
170 + ADD_INPUTS = add_inputs
171 + CAST_TYPE = False
172 + if inputs.dtype == torch.float32 and lora_b_weights.dtype in [
173 + torch.float16,
174 + torch.bfloat16,
175 + ]:
176 + CAST_TYPE = True
177 + grid = (
178 + triton.cdiv(max_seq_length, BLOCK_M) * triton.cdiv(N, BLOCK_N),
179 + batches,
180 + )
181 + _sgmv_expand_slice_kernel[grid](
182 + inputs,
183 + lora_b_weights,
184 + output_tensor,
185 + N,
186 + K,
187 + b_seq_start_loc,
188 + seq_len_tensor,
189 + lora_indices_tensor,
190 + inputs.stride(0),
191 + inputs.stride(1),
192 + lora_b_weights.stride(0),
193 + lora_b_weights.stride(1),
194 + lora_b_weights.stride(2),
195 + output_tensor.stride(0),
196 + output_tensor.stride(1),
197 + slice_offset,
198 + BLOCK_M,
199 + BLOCK_N,
200 + BLOCK_K,
201 + EVEN_K,
202 + ADD_INPUTS,
203 + CAST_TYPE,
204 + )
205 + return
vllm/lora/ops/sgmv_shrink.py new
+189
@@ -0,0 +1,189 @@
1 +"""
2 +Based on:
3 +Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023).
4 +Punica: Multi-Tenant LoRA Serving.
5 +https://arxiv.org/abs/2310.18547
6 +"""
7 +
8 +import torch
9 +import triton
10 +import triton.language as tl
11 +
12 +from vllm.triton_utils import libentry
13 +
14 +
15 +@libentry()
16 +@triton.jit
17 +def _sgmv_shrink_kernel(
18 + input_ptr,
19 + lora_ptr,
20 + out_ptr,
21 + N,
22 + K,
23 + b_seq_start_loc,
24 + seq_lens,
25 + lora_indices,
26 + scaling,
27 + xm_stride, # hidden_size
28 + xk_stride, # 1
29 + l0_stride, # hidden_size*max_rank
30 + lora_k_stride,
31 + lora_n_stride,
32 + cm_stride,
33 + cn_stride,
34 + BLOCK_M: tl.constexpr,
35 + BLOCK_N: tl.constexpr,
36 + BLOCK_K: tl.constexpr,
37 + EVEN_K: tl.constexpr,
38 + SPLIT_K: tl.constexpr,
39 +):
40 + """
41 + The sgmv's shrink triton kernel is based on GroupGEMM+SPLIT-K.
42 + The GEMM of Multi-LoRA can be considered as GroupGEMM. Additionally,
43 + introducing SPLIT-K can improve performance
44 + """
45 + pid = tl.program_id(axis=0)
46 + pid_sk = tl.program_id(axis=1)
47 + cur_batch = tl.program_id(axis=2)
48 + cta_n_num = tl.cdiv(N, BLOCK_N)
49 + pid_m = pid // cta_n_num
50 + pid_n = pid % cta_n_num
51 +
52 + M = tl.load(seq_lens + cur_batch)
53 + if pid_m * BLOCK_M > M:
54 + return
55 + lora_index = tl.load(lora_indices + cur_batch)
56 + if lora_index == -1:
57 + return
58 + cur_seq_start = tl.load(b_seq_start_loc + cur_batch)
59 + offset_m = tl.arange(0, BLOCK_M) + pid_m * BLOCK_M
60 + offset_n = tl.arange(0, BLOCK_N) + pid_n * BLOCK_N
61 + offset_k = pid_sk * BLOCK_K + tl.arange(0, BLOCK_K)
62 +
63 + ram = tl.max_contiguous(tl.multiple_of(offset_m % M, BLOCK_M), BLOCK_M)
64 + rbn = tl.max_contiguous(tl.multiple_of(offset_n % N, BLOCK_N), BLOCK_N)
65 +
66 + a_ptr = (input_ptr + cur_seq_start * xm_stride + ram[:, None] * xm_stride +
67 + offset_k[None, :] * xk_stride)
68 + b_ptr = (lora_ptr + l0_stride * lora_index + rbn[None, :] * lora_k_stride +
69 + offset_k[:, None] * lora_n_stride)
70 +
71 + accumulator = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
72 + for k in range(0, tl.cdiv(K, BLOCK_K * SPLIT_K)):
73 + if EVEN_K:
74 + tiled_a = tl.load(a_ptr)
75 + tiled_b = tl.load(b_ptr)
76 + else:
77 + k_remaining = K - k * (BLOCK_K * SPLIT_K)
78 + tiled_a = tl.load(a_ptr,
79 + mask=offset_k[None, :] < k_remaining,
80 + other=0.0)
81 + tiled_b = tl.load(b_ptr,
82 + mask=offset_k[:, None] < k_remaining,
83 + other=0.0)
84 + accumulator += tl.dot(tiled_a, tiled_b)
85 +
86 + a_ptr += BLOCK_K * SPLIT_K * xk_stride
87 + b_ptr += BLOCK_K * SPLIT_K * lora_n_stride
88 + offset_cm = cur_seq_start + tl.arange(0, BLOCK_M) + pid_m * BLOCK_M
89 +
90 + offset_cn = tl.arange(0, BLOCK_N) + pid_n * BLOCK_N
91 + c_ptr = (out_ptr + offset_cm[:, None] * cm_stride +
92 + offset_cn[None, :] * cn_stride)
93 + c_mask = (offset_cm[:, None] <
94 + (cur_seq_start + M)) & (offset_cn[None, :] < N)
95 + accumulator *= scaling
96 + # handles write-back with reduction-splitting
97 + if SPLIT_K == 1:
98 + tl.store(c_ptr, accumulator, mask=c_mask)
99 + else:
100 + tl.atomic_add(c_ptr, accumulator, mask=c_mask)
101 +
102 +
103 +@torch.inference_mode()
104 +def sgmv_shrink(
105 + inputs: torch.Tensor,
106 + lora_a_weights: torch.Tensor,
107 + output_tensor: torch.Tensor,
108 + b_seq_start_loc: torch.Tensor,
109 + seq_len_tensor: torch.Tensor,
110 + lora_indices_tensor: torch.Tensor,
111 + batches: int,
112 + max_seq_length: int,
113 + scaling: float,
114 +):
115 + """
116 +
117 + Args:
118 + inputs (torch.Tensor): input tensor
119 + lora_a_weights (torch.Tensor): lora'a weight
120 + output_tensor (torch.Tensor): output tensor
121 + b_seq_start_loc (torch.Tensor): (batch_size,). The cumulative
122 + sequence lengths of the sequences in the batch, used to index
123 + into sequence. E.g.,if the sequence length is [4, 6], it is
124 + [0, 4].
125 + seq_len_tensor (torch.Tensor): (batch_size,). record the sequence
126 + length of the sequences in the batch
127 + lora_indices_tensor (torch.Tensor): (batch_size,). The LoRA index
128 + corresponding to each batch. An index of -1 means no lora should be
129 + applied.
130 + batches (int): batch size
131 + max_seq_length (int): The max sequence lengths of the sequences
132 + in the batch
133 + scaling (float): Scaling factor.
134 + """
135 + assert inputs.dtype == lora_a_weights.dtype
136 + assert inputs.dtype in [torch.float16, torch.bfloat16]
137 + assert lora_a_weights.dtype in [
138 + torch.float16,
139 + torch.bfloat16,
140 + ]
141 + assert inputs.size(1) == lora_a_weights.size(-1)
142 + assert b_seq_start_loc.size(0) == batches
143 + assert lora_indices_tensor.size(0) == batches
144 + assert inputs.is_contiguous()
145 +
146 + if lora_a_weights.ndim == 4: # shape:(lora_num,1,rank, size)
147 + assert lora_a_weights.size(1) == 1
148 + lora_a_weights = lora_a_weights.squeeze(dim=1)
149 + else:
150 + assert lora_a_weights.ndim == 3 # shape:(lora_num,rank, size)
151 + assert lora_a_weights.is_contiguous()
152 + assert output_tensor.is_contiguous()
153 + # TODO tuning this config
154 + N, K = lora_a_weights.shape[-2:] # K=hidden_size,N=rank
155 + BLOCK_M = 32
156 + BLOCK_N = 16
157 + BLOCK_K = 32
158 + SPLIT_K = 8
159 + EVEN_K = K % (BLOCK_K * SPLIT_K) == 0
160 + grid = (
161 + triton.cdiv(max_seq_length, BLOCK_M) * triton.cdiv(N, BLOCK_N),
162 + SPLIT_K,
163 + batches,
164 + )
165 +
166 + _sgmv_shrink_kernel[grid](
167 + inputs,
168 + lora_a_weights,
169 + output_tensor,
170 + N,
171 + K,
172 + b_seq_start_loc,
173 + seq_len_tensor,
174 + lora_indices_tensor,
175 + scaling,
176 + inputs.stride(0),
177 + inputs.stride(1),
178 + lora_a_weights.stride(0),
179 + lora_a_weights.stride(1),
180 + lora_a_weights.stride(2),
181 + output_tensor.stride(0),
182 + output_tensor.stride(1),
183 + BLOCK_M,
184 + BLOCK_N,
185 + BLOCK_K,
186 + EVEN_K,
187 + SPLIT_K,
188 + )
189 + return
vllm/lora/ops/utils.py new
+46
@@ -0,0 +1,46 @@
1 +import functools
2 +from typing import Dict
3 +
4 +
5 +@functools.lru_cache
6 +def _get_op_configs(op_type: str, batch: int, hidden_size: int):
7 + # TODO: add optimal configurations
8 + return None
9 +
10 +
11 +def _check_divisibility(hidden_size: int):
12 + # The bgmv_expand kernel requires that the hidden_size be divisible by
13 + # the number below.
14 + divisibility = [2, 4, 8, 16, 32, 64]
15 + divisibility.sort(reverse=True)
16 + for div in divisibility:
17 + if hidden_size % div == 0:
18 + return div
19 + # hidden_size is an odd number
20 + return 1
21 +
22 +
23 +def _get_default_config(op_type: str, batch: int, hidden_size: int):
24 + if op_type == "expand":
25 + return {
26 + "BLOCK_N": 256,
27 + "SPLIT_N": _check_divisibility(hidden_size),
28 + "num_warps": 8
29 + }
30 + else:
31 + return {"BLOCK_K": 256, "SPLIT_K": 64, "num_warps": 8}
32 +
33 +
34 +def get_lora_op_configs(op_type: str, batch: int,
35 + hidden_size: int) -> Dict[str, int]:
36 + """Inspired by `fused_moe_kernel`
37 + The return value will be a dictionary mapping an irregular grid of batch
38 + sizes and hidden_size to configurations of the bgmv-related kernel.
39 + NOTE: It currently only supports the default configuration. We plan to
40 + generate optimal configurations for different hardware in the future using
41 + scripts similar to `benchmark_moe.py`.
42 + """
43 + config = _get_op_configs(op_type, batch, hidden_size)
44 + if not config:
45 + config = _get_default_config(op_type, batch, hidden_size)
46 + return config
vllm/lora/punica.py
+581 -184
@@ -1,207 +1,604 @@
1 -# Based on code from https://github.com/punica-ai/punica
1 +"""
2 +Based on:
3 +Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023).
4 +Punica: Multi-Tenant LoRA Serving.
5 +https://arxiv.org/abs/2310.18547
6 +"""
7
3 -from typing import Optional
8 +from typing import TYPE_CHECKING, Callable, List, Optional, Tuple, Union
9
10 import torch
11
7 -from vllm import _custom_ops as ops
8 -from vllm.platforms import current_platform
12 +from vllm.triton_utils import HAS_TRITON
13
14 +if HAS_TRITON:
15 + from vllm.lora.ops.bgmv_expand import bgmv_expand
16 + from vllm.lora.ops.bgmv_expand_slice import bgmv_expand_slice
17 + from vllm.lora.ops.bgmv_shrink import bgmv_shrink
18 + from vllm.lora.ops.sgmv_expand import sgmv_expand
19 + from vllm.lora.ops.sgmv_expand_slice import sgmv_expand_slice
20 + from vllm.lora.ops.sgmv_shrink import sgmv_shrink
21
11 -def _check_punica_support():
12 - if ops.is_custom_op_supported("_punica_C::dispatch_bgmv"):
13 - return
22 +if TYPE_CHECKING:
23 + # avoid circuit import
24 + from vllm.lora.layers import LoRAMapping
25 + from vllm.lora.models import LongContextLoRAContext
26
15 - if current_platform.get_device_capability() < (8, 0):
16 - raise ImportError(
17 - "punica LoRA kernels require compute capability >= 8.0")
18 - else:
19 - raise ImportError(
20 - "punica LoRA kernels could not be imported. If you built vLLM "
21 - "from source, make sure VLLM_INSTALL_PUNICA_KERNELS=1 env var "
22 - "was set.")
23 -
24 -
25 -def bgmv(
26 - y: torch.Tensor,
27 - x: torch.Tensor,
28 - w_t_all: torch.Tensor,
29 - indicies: torch.LongTensor,
30 - layer_idx: int,
31 - scale: float,
32 -):
33 - """
34 - Semantics:
35 - y[i] += (
36 - x[i].unsqueeze(0)
37 - @ w_t_all[indices[i], layer_idx, :, :].transpose(-1, -2)
38 - * scale
39 - ).squeeze(0)
27
41 - Args:
42 - y: Shape: `[B, H2]`. Output vectors. Will be changed in-place.
43 - x: Shape: `[B, H1]`. Input vectors.
44 - w_t_all: Shape: `[None, L, H2, H1]`. All of the transposed weight
45 - matrices.
46 - indicies: Shape: `[B]`. Indices of the weight matrices.
47 - layer_idx: Layer index of the weight matrices.
48 - scale: Scaling factor.
28 +def compute_meta(
29 + token_lora_tensor: torch.Tensor
30 +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, int, int, bool]:
31 + """
32 + Get the information required for the sgmv kernel. With the features:
33 + 1. If consecutive requests in the batch use the same LoRA, this function
34 + will combine them into a single request, improving sgmv kernel inference
35 + performance.
36 + 2. At the beginning of each prefill stage inference, recalculations are
37 + needed based on the input, but only once.
38 """
50 - _check_punica_support()
39
52 - ops.dispatch_bgmv(y, x, w_t_all, indicies, layer_idx, scale)
40 + lora_indices_tensor, seq_length_tensor = torch.unique_consecutive(
41 + token_lora_tensor, return_counts=True)
42 + cum_result = torch.cumsum(seq_length_tensor, dim=0)
43 + b_seq_start_tensor = torch.zeros_like(seq_length_tensor)
44 + b_seq_start_tensor[1:].copy_(cum_result[:-1])
45 + max_length = seq_length_tensor.max().item()
46
47 + batch_size = lora_indices_tensor.size(0)
48 + no_lora = False
49 + # -1 means no lora should be applied. Use `no_lora` to determine whether
50 + # the current step requires LoRA. If LoRA is not needed, the prefill stage
51 + # does not need to launch the triton kernel, which can improve performance
52 + if batch_size == 1 and lora_indices_tensor == -1:
53 + no_lora = True
54 + return (b_seq_start_tensor, seq_length_tensor, lora_indices_tensor,
55 + batch_size, max_length, no_lora)
56
55 -def dispatch_bgmv_low_level(y: torch.Tensor, x: torch.Tensor,
56 - w_t_all: torch.Tensor, indicies: torch.LongTensor,
57 - layer_idx: int, scale: float, y_offset: int,
58 - y_slice_size: int):
59 - """
60 - Same as `bgmv` but you can operate on slices of y.
61 - Pass whole y, define y_offset and y_slice_size.
57
63 - Semantics:
64 - y[i] += (
65 - x[i].unsqueeze(0)
66 - @ w_t_all[indices[i], layer_idx, :, :].transpose(-1, -2)
67 - * scale
68 - ).squeeze(0)
58 +# TODO see if this can be vectorized
59 +def convert_mapping(
60 + mapping: "LoRAMapping",
61 + lora_index_to_id: List[Optional[int]],
62 + max_loras: int,
63 + vocab_size: int,
64 + extra_vocab_size: int,
65 + long_lora_context: Optional["LongContextLoRAContext"] = None,
66 +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor,
67 + Optional[torch.Tensor], List[int]]:
68 + """Converts LoRAMapping to index tensors.
69
70 Args:
71 - y: Shape: `[B, H2]`. Output vectors. Will be changed in-place.
72 - x: Shape: `[B, H1]`. Input vectors.
73 - w_t_all: Shape: `[None, L, y_slice_size, H1]`. Column partition of
74 - all of the transposed LoRA matrices.
75 - indicies: Shape: `[B]`. Indices of the LoRA weights.
76 - layer_idx: Layer index of LoRA weights.
77 - scale: Scaling factor.
78 - y_offset: Offset to apply to the starting column of y.
79 - y_slice_size: Size of the y column slice.
71 + mapping: LoRAMapping mapping rows in a batch to LoRA ids.
72 + lora_index_to_id: List mapping LoRA ids to LoRA indices.
73 + max_loras: Maximum number of LoRAs.
74 + vocab_size: Model vocab size.
75 + extra_vocab_size: Extra vocab size each LoRA can have.
76 + long_lora_context: Passed if there are long context lora in a batch.
77 +
78 + Returns:
79 + A tuple of tensors:
80 + base_indices: Tensor of shape [batch_size] mapping batch rows to
81 + LoRA indices.
82 + sampler_indices: Tensor of shape [batch_size] mapping requests to
83 + LoRA indices for sampler. For generation, this will be the
84 + same as base_indicies. For prefill, this will map requests
85 + to LoRA indices.
86 + sampler_indices_padded: Tensor of shape [batch_size] mapping
87 + requests to LoRA indices for sampler with padding.
88 + Same as sampler_indicies, but -1 is replaced with
89 + max_loras.
90 + embeddings_indices: Tensor of shape [2, batch_size] mapping
91 + requests to embedding indices. First row is for embeddings
92 + added by the LoRAs, second row is for the LoRA.lora_a
93 + embeddings.
94 + long_lora_indices: Tensor of shape [batch_size] mapping
95 + requests to RoPE offsets and rot dims for long LoRAs.
96 + None if long context lora doesn't exist.
97 + indices_len: List of lengths of the above tensors. It contains
98 + (base_indices, sampler_indices, sampler_indices_padded,
99 + embeddings_indices, long_lora_indices).
100 """
81 - _check_punica_support()
82 -
83 - ops.dispatch_bgmv_low_level(
84 - y,
85 - x,
86 - w_t_all,
87 - indicies,
88 - layer_idx,
89 - scale,
90 - x.size(1),
91 - y_slice_size,
92 - y_offset,
93 - )
101 + index_mapping_indices: List[int] = list(mapping.index_mapping).copy()
102 + embedding_indices = index_mapping_indices.copy()
103 + lora_indices = index_mapping_indices.copy()
104 + long_lora_offsets: Optional[torch.Tensor] = None
105 + if long_lora_context:
106 + long_lora_offsets = torch.zeros(len(index_mapping_indices),
107 + device="cuda",
108 + dtype=torch.long)
109 + prompt_mapping: List[int] = [
110 + lora_index_to_id.index(x) if x > 0 else -1
111 + for x in mapping.prompt_mapping
112 + ]
113 + lora_idx = None
114 + for i in range(len(index_mapping_indices)):
115 + # TODO index can be slow. optimize
116 + lora_idx = (lora_index_to_id.index(index_mapping_indices[i])
117 + if index_mapping_indices[i] > 0 else -1)
118 + embedding_indices[i] = lora_idx if index_mapping_indices[i] > 0 else 0
119 + lora_indices[i] = lora_idx
120 + if long_lora_context:
121 + assert long_lora_offsets is not None
122 + lora_offset: int = long_lora_context.offsets_by_lora_id.get(
123 + index_mapping_indices[i], 0)
124 + long_lora_offsets[i] = lora_offset
125
126 + indices_list: List[Union[List[int], torch.Tensor]] = [
127 + index_mapping_indices,
128 + lora_indices,
129 + embedding_indices,
130 + ]
131 + if long_lora_context:
132 + assert long_lora_offsets is not None
133 + indices_list.append(long_lora_offsets)
134 + indices = torch.tensor(indices_list, dtype=torch.long, device="cuda")
135 + prompt_mapping_tensor = torch.tensor(prompt_mapping,
136 + device="cuda",
137 + dtype=torch.long)
138 + embeddings_indices = torch.stack([
139 + indices[2] * extra_vocab_size,
140 + indices[2] * (vocab_size + extra_vocab_size),
141 + ])
142 + embeddings_indices[embeddings_indices == -1] = max_loras - 1
143 + base_indices = indices[1]
144 + sampler_indices = prompt_mapping_tensor
145 + sampler_indices_padded = sampler_indices.clone()
146 + sampler_indices_padded[sampler_indices_padded == -1] = max_loras - 1
147 + sampler_indices_padded = torch.arange(
148 + 0, len(sampler_indices_padded), device="cuda", dtype=torch.long) + (
149 + sampler_indices_padded * len(sampler_indices_padded))
150 + long_lora_indices = None
151 + long_lora_indices_len: Optional[int] = None
152 + if long_lora_context:
153 + long_lora_indices = indices[3]
154 + long_lora_indices_len = long_lora_indices.shape[-1]
155 + # Contain length of indices tensors. Used to index into each tensor.
156 + indices_len = [
157 + base_indices.shape[-1],
158 + sampler_indices.shape[-1],
159 + sampler_indices_padded.shape[-1],
160 + embeddings_indices.shape[-1],
161 + ]
162 + if long_lora_indices_len is not None:
163 + indices_len.append(long_lora_indices_len)
164 + else:
165 + # If long_lora doesn't exist,append None
166 + indices_len.append(None)
167
96 -def add_lora(y: torch.Tensor,
97 - x: torch.Tensor,
98 - wa_t_all: torch.Tensor,
99 - wb_t_all: torch.Tensor,
100 - indicies: torch.LongTensor,
101 - layer_idx: int,
102 - scale: float,
103 - *,
104 - buffer: Optional[torch.Tensor] = None):
105 - """
106 - Semantics:
107 - y[i] += (
108 - x[i].unsqueeze(0)
109 - @ wa_t_all[indices[i], layer_idx, :, :].transpose(-1, -2)
110 - @ wb_t_all[indices[i], layer_idx, :, :].transpose(-1, -2)
111 - * scale
112 - ).squeeze(0)
168 + return (
169 + base_indices,
170 + sampler_indices,
171 + sampler_indices_padded,
172 + embeddings_indices,
173 + long_lora_indices,
174 + indices_len,
175 + )
176
114 - Args:
115 - y: Shape: `[B, H2]`. Output vectors. Will be changed in-place.
116 - x: Shape: `[B, H1]`. Input vectors.
117 - wa_t_all: Shape: `[None, L, R, H1]`. All of the transposed
118 - LoRA A matrices.
119 - wb_t_all: Shape: `[None, L, H2, R]`. All of the transposed
120 - LoRA B matrices.
121 - indicies: Shape: `[B]`. Indices of the LoRA weights.
122 - layer_idx: Layer index of LoRA weights.
123 - scale: Scaling factor.
124 - buffer: Optional. Shape: `[B, R]`. Temporary buffer.
177 +
178 +class PunicaWrapper:
179 """
126 - _check_punica_support()
127 -
128 - r = wb_t_all.size(-1)
129 - if buffer is None:
130 - # We set the buffer to be float32 by default to avoid
131 - # numerical inaccuracies that would otherwise happen
132 - # due to downcasting.
133 - buffer = torch.zeros((x.size(0), r),
134 - dtype=torch.float32,
135 - device=x.device)
136 - ops.dispatch_bgmv(buffer, x, wa_t_all, indicies, layer_idx, 1.0)
137 - ops.dispatch_bgmv(y, buffer, wb_t_all, indicies, layer_idx, scale)
138 -
139 -
140 -def add_lora_slice(y: torch.Tensor,
141 - x: torch.Tensor,
142 - wa_t_all: torch.Tensor,
143 - wb_t_all: torch.Tensor,
144 - indicies: torch.LongTensor,
145 - layer_idx: int,
146 - scale: float,
147 - y_offset: int,
148 - y_slice_size: int,
149 - *,
150 - buffer: Optional[torch.Tensor] = None):
180 + PunicaWrapper is designed to manage and provide metadata for the punica
181 + kernel. The main function is to maintain the state information for
182 + Multi-LoRA, and to provide the interface for the punica kernel.
183 """
152 - Same as `add_lora` but you can operate on slices of y.
153 - Pass whole y, define y_offset and y_slice_size.
184
155 - Semantics:
156 - y[i] += (
157 - x[i].unsqueeze(0)
158 - @ wa_t_all[indices[i], layer_idx, :, :].transpose(-1, -2)
159 - @ wb_t_all[indices[i], layer_idx, :, :].transpose(-1, -2)
160 - * scale
161 - ).squeeze(0)
185 + def __init__(self, max_num_batched_tokens: int, max_batches: int,
186 + device: str):
187 + self._token_lora_indices = torch.empty(max_num_batched_tokens,
188 + dtype=torch.long,
189 + device=device)
190 + self._sampler_indices = torch.empty(max_num_batched_tokens,
191 + dtype=torch.long,
192 + device=device)
193 + self._sampler_indices_padded = torch.empty(max_num_batched_tokens,
194 + dtype=torch.long,
195 + device=device)
196 + self._embeddings_indices = torch.empty(2,
197 + max_num_batched_tokens,
198 + dtype=torch.long,
199 + device=device)
200 + self._long_lora_indices = torch.empty(max_num_batched_tokens,
201 + dtype=torch.long,
202 + device=device)
203
163 - Args:
164 - y: Shape: `[B, H2]`. Output vectors. Will be changed in-place.
165 - x: Shape: `[B, H1]`. Input vectors.
166 - wa_t_all: Shape: `[None, L, R, H1]`. All of the transposed
167 - LoRA A matrices.
168 - wb_t_all: Shape: `[None, L, H2, R]`. All of the transposed
169 - LoRA B matrices.
170 - indicies: Shape: `[B]`. Indices of the LoRA weights.
171 - layer_idx: Layer index of LoRA weights.
172 - scale: Scaling factor.
173 - y_offset: Offset to apply to the starting column of y.
174 - y_slice_size: Size of the y column slice.
175 - """
176 - _check_punica_support()
177 -
178 - r = wb_t_all.size(-1)
179 - if buffer is None:
180 - # We set the buffer to be float32 by default to avoid
181 - # numerical inaccuracies that would otherwise happen
182 - # due to downcasting.
183 - buffer = torch.zeros((x.size(0), r),
184 - dtype=torch.float32,
185 - device=x.device)
186 - ops.dispatch_bgmv_low_level(
187 - buffer,
188 - x,
189 - wa_t_all,
190 - indicies,
191 - layer_idx,
192 - 1.0,
193 - x.size(1),
194 - buffer.size(1),
195 - 0,
196 - )
197 - ops.dispatch_bgmv_low_level(
198 - y,
199 - buffer,
200 - wb_t_all,
201 - indicies,
202 - layer_idx,
203 - scale,
204 - buffer.size(1),
205 - y_slice_size,
206 - y_offset,
207 - )
204 + # 5 is the number of indicies tensors.
205 + # base_indices, sampler_indices, sampler_indices_padded,
206 + # embeddings_indices,long_lora_indices
207 + self.indices_len: List[Optional[int]] = [None] * 5
208 + # these attributes are the information required for sgmv kernel
209 + self._seq_start_locs = torch.empty(max_batches,
210 + dtype=torch.long,
211 + device=device)
212 + self._seq_lengths = torch.empty(max_batches,
213 + dtype=torch.long,
214 + device=device)
215 + self._lora_indices_per_batch = torch.empty(max_batches,
216 + dtype=torch.long,
217 + device=device)
218 + self.max_length: int = 0
219 + self.batch_size: int = -1
220 + self.is_prefill = False
221 + self.no_lora = False
222 +
223 + def update_metadata(
224 + self,
225 + mapping: "LoRAMapping",
226 + lora_index_to_id: List[Optional[int]],
227 + max_loras: int,
228 + vocab_size: int,
229 + extra_vocab_size: int,
230 + long_lora_context: Optional["LongContextLoRAContext"] = None,
231 + ):
232 +
233 + self._update_base_metadata(mapping, lora_index_to_id, max_loras,
234 + vocab_size, extra_vocab_size,
235 + long_lora_context)
236 + if mapping.is_prefill:
237 + # Update metadata required for prefill-related operators.
238 + self._update_prefill_metada(self.token_lora_indices)
239 + self.is_prefill = True
240 + else:
241 + self.is_prefill = False
242 +
243 + def _update_base_metadata(
244 + self,
245 + mapping: "LoRAMapping",
246 + lora_index_to_id: List[Optional[int]],
247 + max_loras: int,
248 + vocab_size: int,
249 + extra_vocab_size: int,
250 + long_lora_context: Optional["LongContextLoRAContext"] = None,
251 + ):
252 + (
253 + base_indices,
254 + sampler_indices,
255 + sampler_indices_padded,
256 + embeddings_indices,
257 + long_lora_offsets_tensor,
258 + indices_len,
259 + ) = convert_mapping(
260 + mapping,
261 + lora_index_to_id,
262 + max_loras,
263 + vocab_size,
264 + extra_vocab_size,
265 + long_lora_context,
266 + )
267 + self._token_lora_indices[:base_indices.shape[0]].copy_(base_indices)
268 + self._sampler_indices[:sampler_indices.shape[0]].copy_(sampler_indices)
269 + self._sampler_indices_padded[:sampler_indices_padded.shape[0]].copy_(
270 + sampler_indices_padded)
271 + self._embeddings_indices[:embeddings_indices.
272 + shape[0], :embeddings_indices.shape[1]].copy_(
273 + embeddings_indices)
274 + if long_lora_offsets_tensor is not None:
275 + self._long_lora_indices[:long_lora_offsets_tensor.shape[0]].copy_(
276 + long_lora_offsets_tensor)
277 + else:
278 + self._long_lora_indices.zero_()
279 +
280 + self.indices_len[:] = indices_len
281 +
282 + def _update_prefill_metada(self, token_lora_tensor: torch.Tensor) -> None:
283 +
284 + (b_seq_start_tensor, seq_length_tensor, lora_indices_tensor,
285 + batch_size, max_length, no_lora) = compute_meta(token_lora_tensor)
286 +
287 + self._seq_start_locs[:b_seq_start_tensor.shape[0]].copy_(
288 + b_seq_start_tensor)
289 + self._seq_lengths[:seq_length_tensor.shape[0]].copy_(seq_length_tensor)
290 + self._lora_indices_per_batch[:lora_indices_tensor.shape[0]].copy_(
291 + lora_indices_tensor)
292 + self.batch_size = batch_size
293 + self.max_length = max_length
294 + self.no_lora = no_lora
295 +
296 + @property
297 + def prefill_metadata(
298 + self) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, int, int]:
299 + """
300 + This property provides a convenient way to access the necessary
301 + metadata for prefill-related kernel computations.
302 + 1. seq_start_locs: Tensor of sequence start positions
303 + 2. seq_lengths: Tensor of sequence lengths
304 + 3. lora_indices_per_batch: Tensor of lora indices, and an index of
305 + -1 means no lora should be applied.
306 + 4. batch_size: batch size after clustering identical lora indices
307 + 5. max_length: The maximum sequence length in the batch
308 + """
309 + return (self._seq_start_locs[:self.batch_size],
310 + self._seq_lengths[:self.batch_size],
311 + self._lora_indices_per_batch[:self.batch_size],
312 + self.batch_size, self.max_length)
313 +
314 + @property
315 + def token_lora_indices(self) -> torch.Tensor:
316 + """
317 + This property provides the lora indices corresponding to each token
318 + in the batch. An index of -1 means no lora should be applied.
319 + """
320 + token_lora_len = self.indices_len[0]
321 + return self._token_lora_indices[:token_lora_len]
322 +
323 + @property
324 + def sampler_indices(self) -> torch.Tensor:
325 + """
326 + This property is used to access the lora indices specifically for
327 + LogitsProcessorWithLoRA
328 + """
329 + sampler_indices_len = self.indices_len[1]
330 + return self._sampler_indices[:sampler_indices_len]
331 +
332 + @property
333 + def sampler_indices_padded(self) -> torch.Tensor:
334 + """
335 + This property provides access to padded sampler indices
336 + """
337 + indices_padded_len = self.indices_len[2]
338 + return self._sampler_indices_padded[:indices_padded_len]
339 +
340 + @property
341 + def embeddings_indices(self) -> torch.Tensor:
342 + """
343 + This property provides access to the indices used for lora embeddings,
344 + specifically for VocabParallelEmbeddingWithLoRA
345 + """
346 + embeddings_indices_len = self.indices_len[3]
347 + return self._embeddings_indices[:, :embeddings_indices_len]
348 +
349 + @property
350 + def long_lora_indices(self) -> torch.Tensor:
351 + """
352 + This property provides access to the indices used for long context
353 + lora, specifically for LinearScalingRotaryEmbeddingWithLora
354 + """
355 + long_lora_len = self.indices_len[4]
356 + return self._long_lora_indices[:long_lora_len]
357 +
358 + def shrink_prefill(
359 + self,
360 + y: torch.Tensor,
361 + x: torch.Tensor,
362 + w_t_all: torch.Tensor,
363 + scale: float,
364 + ):
365 + #No LoRA request, so return directly
366 + if self.no_lora:
367 + return
368 + sgmv_shrink(
369 + x,
370 + w_t_all,
371 + y,
372 + *self.prefill_metadata,
373 + scale,
374 + )
375 +
376 + def shrink_decode(
377 + self,
378 + y: torch.Tensor,
379 + x: torch.Tensor,
380 + w_t_all: torch.Tensor,
381 + scale: float,
382 + ):
383 + bgmv_shrink(x, w_t_all, y, self.token_lora_indices, scale)
384 +
385 + def expand_prefill(
386 + self,
387 + y: torch.Tensor,
388 + x: torch.Tensor,
389 + w_t_all: torch.Tensor,
390 + add_input: bool,
391 + ):
392 + #No LoRA request, so return directly
393 + if self.no_lora:
394 + return
395 + sgmv_expand(
396 + x,
397 + w_t_all,
398 + y,
399 + *self.prefill_metadata,
400 + add_input,
401 + )
402 +
403 + def expand_decode(
404 + self,
405 + y: torch.Tensor,
406 + x: torch.Tensor,
407 + w_t_all: torch.Tensor,
408 + add_input: bool,
409 + ):
410 + bgmv_expand(x, w_t_all, y, self.token_lora_indices, add_input)
411 +
412 + def expand_slice_prefill(
413 + self,
414 + y: torch.Tensor,
415 + x: torch.Tensor,
416 + w_t_all: torch.Tensor,
417 + y_offset: Optional[int],
418 + y_slice_size: Optional[int],
419 + add_input: bool,
420 + ):
421 + #No LoRA request, so return directly
422 + if self.no_lora:
423 + return
424 + sgmv_expand_slice(
425 + x,
426 + w_t_all,
427 + y,
428 + *self.prefill_metadata,
429 + y_offset,
430 + y_slice_size,
431 + add_input,
432 + )
433 +
434 + def expand_slice_decode(
435 + self,
436 + y: torch.Tensor,
437 + x: torch.Tensor,
438 + w_t_all: torch.Tensor,
439 + y_offset: Optional[int],
440 + y_slice_size: Optional[int],
441 + add_input: bool,
442 + ):
443 + bgmv_expand_slice(x, w_t_all, y, self.token_lora_indices, y_offset,
444 + y_slice_size, add_input)
445 +
446 + def add_shrink(
447 + self,
448 + y: torch.Tensor,
449 + x: torch.Tensor,
450 + w_t_all: torch.Tensor,
451 + scale: float,
452 + ):
453 + """
454 + Perform the ` y+=x@w_t_all` computation, which is suitable for the
455 + GEMM of lora'a.
456 + When `is_prefill is` true, it indicates that it is currently the
457 + prefill stage, and the `shrink_prefill` function should be called.
458 + Otherwise, it is the decode stage, and the shrink_decode function
459 + should be called.
460 + """
461 + shrink_fun: Callable = (self.shrink_prefill
462 + if self.is_prefill else self.shrink_decode)
463 + shrink_fun(y, x, w_t_all, scale)
464 +
465 + def add_expand(
466 + self,
467 + y: torch.Tensor,
468 + x: torch.Tensor,
469 + w_t_all: torch.Tensor,
470 + add_input: bool = True,
471 + ):
472 + """
473 + Perform the ` y+=x@w_t_all` computation, which is suitable for the
474 + GEMM of lora'b.
475 + When `is_prefill` is true, it indicates that it is currently the
476 + prefill stage, and the `expand_prefill` function should be called.
477 + Otherwise, it is the decode stage, and the expand_decode function
478 + should be called.
479 + """
480 +
481 + expand_fun: Callable = (self.expand_prefill
482 + if self.is_prefill else self.expand_decode)
483 + expand_fun(y, x, w_t_all, add_input)
484 +
485 + def add_expand_slice(self,
486 + y: torch.Tensor,
487 + x: torch.Tensor,
488 + w_t_all: torch.Tensor,
489 + y_offset: Optional[int],
490 + y_slice_size: Optional[int],
491 + add_input: bool = True):
492 + """
493 + Similar to `add_expand`
494 + """
495 +
496 + expand_slice_fun: Callable = (self.expand_slice_prefill
497 + if self.is_prefill else
498 + self.expand_slice_decode)
499 + expand_slice_fun(y, x, w_t_all, y_offset, y_slice_size, add_input)
500 +
501 + def add_lora(self,
502 + y: torch.Tensor,
503 + x: torch.Tensor,
504 + wa_t_all: torch.Tensor,
505 + wb_t_all: torch.Tensor,
506 + scale: float,
507 + y_offset: Optional[int] = None,
508 + y_slice_size: Optional[int] = None,
509 + *,
510 + buffer: Optional[torch.Tensor] = None) -> None:
511 + """
512 + Semantics:
513 + y[i] += (
514 + x[i].unsqueeze(0)
515 + @ wa_t_all[indices[i], layer_idx, :, :].transpose(-1, -2)
516 + @ wb_t_all[indices[i], layer_idx, :, :].transpose(-1, -2)
517 + * scale
518 + ).squeeze(0)
519 + Args:
520 + y (torch.Tensor): Output tensor. Will be changed in-place.
521 + x (torch.Tensor): Input tensor
522 + wa_t_all (torch.Tensor): lora_a's weight
523 + wb_t_all (torch.Tensor): lora_b's weight
524 + scale (float): Scaling factor.
525 + y_offset (Optional[int], optional): Offset to apply to the starting
526 + column of y.
527 + y_slice_size (Optional[int], optional): Size of the y column slice..
528 + buffer (Optional[torch.Tensor], optional): Defaults to None.
529 + """
530 + y_org = y
531 + y = y.view(-1, y.shape[-1])
532 + x = x.view(-1, x.shape[-1])
533 + r = wb_t_all.size(-1)
534 + if buffer is None:
535 + # We set the buffer to be float32 by default ,refer to:
536 + # https://github.com/triton-lang/triton/issues/1387
537 + buffer = torch.zeros((x.size(0), r),
538 + dtype=torch.float32,
539 + device=x.device)
540 +
541 + self.add_shrink(buffer, x, wa_t_all, scale)
542 + if y_offset is None and y_slice_size is None:
543 + self.add_expand(y, buffer, wb_t_all, add_input=True)
544 + else:
545 + self.add_expand_slice(y,
546 + buffer,
547 + wb_t_all,
548 + y_offset,
549 + y_slice_size,
550 + add_input=True)
551 + y = y.view_as(y_org)
552 +
553 + def add_lora_packed_nslice(self, y: torch.Tensor, x: torch.Tensor,
554 + lora_a_stacked: Tuple[torch.Tensor,
555 + torch.Tensor,
556 + torch.Tensor],
557 + lora_b_stacked: Tuple[torch.Tensor,
558 + torch.Tensor,
559 + torch.Tensor],
560 + scale: float,
561 + output_slices: Tuple[int, ...]) -> None:
562 + """
563 + Applies lora to each input. Similar to add_lora, This method is
564 + used for layers that are composed of multiple sublayers
565 + (slices) packed together.
566 + """
567 + y_org = y
568 + x = x.view(-1, x.shape[-1])
569 + y = y.view(-1, y.shape[-1])
570 + offset_left = 0
571 + # TODO fuse these kernels
572 + for slice_idx in range(len(output_slices)):
573 + self.add_lora(y, x, lora_a_stacked[slice_idx],
574 + lora_b_stacked[slice_idx], scale, offset_left,
575 + output_slices[slice_idx])
576 + offset_left += output_slices[slice_idx]
577 +
578 + y = y.view_as(y_org)
579 +
580 + def add_lora_logits(self,
581 + y: torch.Tensor,
582 + x: torch.Tensor,
583 + wa_t_all: torch.Tensor,
584 + wb_t_all: torch.Tensor,
585 + scale,
586 + *,
587 + buffer: Optional[torch.Tensor] = None) -> None:
588 + """
589 + LogitsProcessorWithLoRA always using bgmv
590 + """
591 + y_org = y
592 + y = y.view(-1, y.shape[-1])
593 + x = x.view(-1, x.shape[-1])
594 + r = wb_t_all.size(-1)
595 + if buffer is None:
596 + # We set the buffer to be float32 by default ,refer to:
597 + # https://github.com/triton-lang/triton/issues/1387
598 + buffer = torch.zeros((x.size(0), r),
599 + dtype=torch.float32,
600 + device=x.device)
601 +
602 + bgmv_shrink(x, wa_t_all, buffer, self.sampler_indices, scale)
603 + bgmv_expand(buffer, wb_t_all, y, self.sampler_indices, add_inputs=True)
604 + y = y.view_as(y_org)
vllm/triton_utils/__init__.py
+2 -1
@@ -6,5 +6,6 @@ if HAS_TRITON:
6
7 from vllm.triton_utils.custom_cache_manager import (
8 maybe_set_triton_cache_manager)
9 + from vllm.triton_utils.libentry import libentry
10
10 - __all__ += ["maybe_set_triton_cache_manager"]
11 + __all__ += ["maybe_set_triton_cache_manager", "libentry"]
vllm/triton_utils/libentry.py new
+167
@@ -0,0 +1,167 @@
1 +# Copied From https://github.com/FlagOpen/FlagGems
2 +
3 +import inspect
4 +
5 +import triton
6 +
7 +
8 +class LibEntry(triton.KernelInterface):
9 +
10 + def __init__(
11 + self,
12 + fn,
13 + ):
14 + self.fn = fn
15 + self.arg_names = fn.arg_names
16 + self.divisibility = 16
17 + self.kernel_cache = dict()
18 + fn = self.fn
19 + while not isinstance(fn, triton.runtime.JITFunction):
20 + fn = fn.fn
21 + self.jit_function: triton.runtime.JITFunction = fn
22 + self.specialize_indices = [
23 + p.num for p in self.jit_function.params
24 + if not p.is_constexpr and not p.do_not_specialize
25 + ]
26 + self.do_not_specialize_indices = [
27 + p.num for p in self.jit_function.params
28 + if not p.is_constexpr and p.do_not_specialize
29 + ]
30 +
31 + def key(self, spec_args, dns_args, const_args):
32 + spec_key = [(arg.dtype, arg.data_ptr() %
33 + self.divisibility == 0) if hasattr(arg, "data_ptr") else
34 + (type(arg), arg) for arg in spec_args]
35 + dns_key = [
36 + arg.dtype if hasattr(
37 + arg, "data_ptr") else type(arg) if not isinstance(arg, int)
38 + else "i32" if -(2**31) <= arg and arg <= 2**31 -
39 + 1 else "u64" if 2**63 <= arg and arg <= 2**64 - 1 else "i64"
40 + for arg in dns_args
41 + ]
42 + # const args passed by position
43 + return tuple(spec_key + dns_key + const_args)
44 +
45 + def run(self, *args, **kwargs):
46 + grid = kwargs["grid"]
47 + # collect all the arguments
48 + spec_args = [] # specialize arguments
49 + dns_args = [] # do not specialize arguments
50 + const_args = [] # constexpr arguments
51 + k_args = [] # kernel arguments
52 + for i, arg in enumerate(args):
53 + if i in self.specialize_indices:
54 + k_args.append(arg)
55 + spec_args.append(arg)
56 + elif i in self.do_not_specialize_indices:
57 + k_args.append(arg)
58 + dns_args.append(arg)
59 + else:
60 + const_args.append(arg)
61 + for p in self.jit_function.params[len(args):]:
62 + if p.name in kwargs:
63 + val = kwargs[p.name]
64 + elif p.default is inspect._empty:
65 + continue
66 + else:
67 + val = p.default
68 +
69 + if p.is_constexpr:
70 + const_args.append(val)
71 + elif p.do_not_specialize:
72 + dns_args.append(val)
73 + k_args.append(val)
74 + else:
75 + spec_args.append(val)
76 + k_args.append(val)
77 +
78 + entry_key = self.key(spec_args, dns_args, const_args)
79 +
80 + if entry_key not in self.kernel_cache:
81 + # compile the kernel also completes the related computations
82 + kernel = self.fn.run(*args, **kwargs)
83 + fn = self.fn
84 + # collect constexpr arguments for grid computation
85 + constexprs = {}
86 + while not isinstance(fn, triton.runtime.JITFunction):
87 + if isinstance(fn, triton.runtime.Autotuner):
88 + config = fn.best_config
89 + constexprs["num_warps"] = config.num_warps
90 + constexprs["num_stages"] = config.num_stages
91 + constexprs["num_ctas"] = config.num_ctas
92 + constexprs = {**constexprs, **config.kwargs}
93 + elif isinstance(fn, triton.runtime.Heuristics):
94 + for v, heur in fn.values.items():
95 + constexprs[v] = heur({
96 + **dict(zip(fn.arg_names, args)),
97 + **kwargs,
98 + **constexprs,
99 + })
100 + else:
101 + raise RuntimeError("Invalid Runtime Function")
102 + fn = fn.fn
103 + # In vLLM, certain kernels like fused_moe_kernel get the
104 + # best_config(as kwargs) from a configuration json file, rather
105 + # than using Autotuner & Heuristics. Therefore, all their constexprs
106 + # (tl.constexpr) are assigned values through the following loop.
107 + for p in self.jit_function.params:
108 + if p.is_constexpr and p.name not in constexprs:
109 + constexprs[p.name] = p.default #default=inspect._empty
110 + self.kernel_cache[entry_key] = (kernel, constexprs)
111 + else:
112 + # load kernel from cache directly
113 + kernel, constexprs = self.kernel_cache[entry_key]
114 +
115 + if callable(grid):
116 + # collect all arguments to the grid fn,ie:
117 + # 1. args,
118 + # 2. kwargs,
119 + # 3. all all other captured arguments in CompiledKernel from
120 + # Autotunner & Heuristics when kwargs & captured args conflict,
121 + # captured args have higher priority
122 + # 4. We must filter out captured args with default value firstly
123 + constexprs = {
124 + k: v
125 + for k, v in constexprs.items() if v is not inspect._empty
126 + }
127 + meta = {
128 + **dict(zip(self.arg_names, args)),
129 + **kwargs,
130 + **constexprs,
131 + }
132 + grid = grid(meta)
133 + if isinstance(grid, tuple):
134 + grid = grid + (1, 1)
135 + elif isinstance(grid, list):
136 + grid = grid + [1, 1]
137 + kernel[grid[0:3]](*k_args)
138 + # maintaining the same return type as the JITFunction.run
139 + return kernel
140 +
141 +
142 +def libentry():
143 + """
144 + Decorator for triton library entries.
145 + Motivation:
146 + The runtime overhead of Triton kernels is the reason for the lower
147 + performance of small kernels, particularly evident with smaller models.
148 + Using this decorator can reduce Triton runtime overhead.
149 + How:
150 + The `run` function of JITFunction needs to accomplish:
151 + - Parameter binding using inspect
152 + - KernelArg type wrapping
153 + - Cache key calculation
154 + When dealing with small size, these steps can become bottlenecks in
155 + Triton runtime. Libentry simplifies these steps to reduce runtime
156 + overhead, thereby improving the runtime expenses of small kernels.
157 + NOTE:
158 + When Triton is upgraded to version 3.0.0, libentry can be removed,
159 + see: https://github.com/vllm-project/vllm/pull/5036#issuecomment-2243396245
160 +
161 +
162 + """
163 +
164 + def decorator(fn):
165 + return LibEntry(fn)
166 +
167 + return decorator
vllm/worker/model_runner.py
+6 -6
@@ -578,9 +578,9 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
578 for inter_data in self.inter_data_list
579 ])
580 lora_mapping = LoRAMapping(
581 - lora_index_mapping,
582 - lora_prompt_mapping,
583 - )
581 + **dict(index_mapping=lora_index_mapping,
582 + prompt_mapping=lora_prompt_mapping,
583 + is_prefill=not self.decode_only))
584
585 # Prompt adapter data.
586 prompt_adapter_requests: Set[PromptAdapterRequest] = set()
@@ -1152,9 +1152,9 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1152
1153 if self.lora_config:
1154 lora_mapping = LoRAMapping(
1155 - [0] * batch_size,
1156 - [0] * batch_size,
1157 - )
1155 + **dict(index_mapping=[0] * batch_size,
1156 + prompt_mapping=[0] * batch_size,
1157 + is_prefill=False))
1158 self.set_active_loras(set(), lora_mapping)
1159
1160 if self.prompt_adapter_config: