A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Kernel][RFC] Refactor the punica kernel based on Triton (#5036)
Jee Jee Li committed
Aug 1, 2024 at 08:12 UTC
7ecee3432110bae563c8756a66b54e5f08dc777d
47 files changed
+3175
-4364
.github/workflows/scripts/build.sh
-2
@@ -13,8 +13,6 @@ $python_executable -m pip install -r requirements-cuda.txt
13
14
# Limit the number of parallel jobs to avoid OOM
15
export MAX_JOBS=1
16
-# Make sure punica is built for the release (for LoRA)
17
-export VLLM_INSTALL_PUNICA_KERNELS=1
16
# Make sure release wheels are built for the following architectures
17
export TORCH_CUDA_ARCH_LIST="7.0 7.5 8.0 8.6 8.9 9.0+PTX"
18
# Build
CMakeLists.txt
-62
@@ -223,61 +223,7 @@ define_gpu_extension_target(
223
USE_SABI 3
224
WITH_SOABI)
225
226
-#
227
-# _punica_C extension
228
-#
229
-
230
-set(VLLM_PUNICA_EXT_SRC
231
- "csrc/punica/bgmv/bgmv_bf16_bf16_bf16.cu"
232
- "csrc/punica/bgmv/bgmv_bf16_fp32_bf16.cu"
233
- "csrc/punica/bgmv/bgmv_fp16_fp16_fp16.cu"
234
- "csrc/punica/bgmv/bgmv_fp16_fp32_fp16.cu"
235
- "csrc/punica/bgmv/bgmv_fp32_bf16_bf16.cu"
236
- "csrc/punica/bgmv/bgmv_fp32_fp16_fp16.cu"
237
- "csrc/punica/punica_ops.cu"
238
- "csrc/punica/torch_bindings.cpp")
239
-
240
-#
241
-# Copy GPU compilation flags+update for punica
242
-#
243
-set(VLLM_PUNICA_GPU_FLAGS ${VLLM_GPU_FLAGS})
244
-list(REMOVE_ITEM VLLM_PUNICA_GPU_FLAGS
245
- "-D__CUDA_NO_HALF_OPERATORS__"
246
- "-D__CUDA_NO_HALF_CONVERSIONS__"
247
- "-D__CUDA_NO_BFLOAT16_CONVERSIONS__"
248
- "-D__CUDA_NO_HALF2_OPERATORS__")
249
-
250
-#
251
-# Filter out CUDA architectures < 8.0 for punica.
252
-#
253
-if (${VLLM_GPU_LANG} STREQUAL "CUDA")
254
- set(VLLM_PUNICA_GPU_ARCHES)
255
- foreach(ARCH ${VLLM_GPU_ARCHES})
256
- string_to_ver(CODE_VER ${ARCH})
257
- if (CODE_VER GREATER_EQUAL 8.0)
258
- list(APPEND VLLM_PUNICA_GPU_ARCHES ${ARCH})
259
- endif()
260
- endforeach()
261
- message(STATUS "Punica target arches: ${VLLM_PUNICA_GPU_ARCHES}")
262
-elseif(${VLLM_GPU_LANG} STREQUAL "HIP")
263
- set(VLLM_PUNICA_GPU_ARCHES ${VLLM_GPU_ARCHES})
264
- message(STATUS "Punica target arches: ${VLLM_PUNICA_GPU_ARCHES}")
265
-endif()
226
267
-if (VLLM_PUNICA_GPU_ARCHES)
268
- define_gpu_extension_target(
269
- _punica_C
270
- DESTINATION vllm
271
- LANGUAGE ${VLLM_GPU_LANG}
272
- SOURCES ${VLLM_PUNICA_EXT_SRC}
273
- COMPILE_FLAGS ${VLLM_PUNICA_GPU_FLAGS}
274
- ARCHITECTURES ${VLLM_PUNICA_GPU_ARCHES}
275
- USE_SABI 3
276
- WITH_SOABI)
277
-else()
278
- message(WARNING "Unable to create _punica_C target because none of the "
279
- "requested architectures (${VLLM_GPU_ARCHES}) are supported, i.e. >= 8.0")
280
-endif()
227
228
#
229
# Add the `default` target which detects which extensions should be
@@ -301,12 +247,4 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
247
message(STATUS "Enabling moe extension.")
248
add_dependencies(default _moe_C)
249
304
- # Enable punica if -DVLLM_INSTALL_PUNICA_KERNELS=ON or
305
- # VLLM_INSTALL_PUNICA_KERNELS is set in the environment and
306
- # there are supported target arches.
307
- if (VLLM_PUNICA_GPU_ARCHES AND
308
- (ENV{VLLM_INSTALL_PUNICA_KERNELS} OR VLLM_INSTALL_PUNICA_KERNELS))
309
- message(STATUS "Enabling punica extension.")
310
- add_dependencies(default _punica_C)
311
- endif()
250
endif()
Dockerfile
-2
@@ -88,8 +88,6 @@ ENV MAX_JOBS=${max_jobs}
88
# number of threads used by nvcc
89
ARG nvcc_threads=8
90
ENV NVCC_THREADS=$nvcc_threads
91
-# make sure punica kernels are built (for LoRA)
92
-ENV VLLM_INSTALL_PUNICA_KERNELS=1
91
92
ARG buildkite_commit
93
ENV BUILDKITE_COMMIT=${buildkite_commit}
Dockerfile.rocm
+1
-2
@@ -131,8 +131,7 @@ COPY . .
131
RUN --mount=type=cache,target=/root/.cache/pip \
132
python3 -m pip install --upgrade numba scipy huggingface-hub[cli]
133
134
-# Make sure punica kernels are built (for LoRA)
135
-ENV VLLM_INSTALL_PUNICA_KERNELS=1
134
+
135
# Workaround for ray >= 2.10.0
136
ENV RAY_EXPERIMENTAL_NOSET_ROCR_VISIBLE_DEVICES=1
137
# Silences the HF Tokenizers warning
csrc/punica/LICENSE
deleted
-217
@@ -1,217 +0,0 @@
1
-Contains code from https://github.com/punica-ai/punica
2
-
3
- Apache License
4
- Version 2.0, January 2004
5
- http://www.apache.org/licenses/
6
-
7
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
8
-
9
- 1. Definitions.
10
-
11
- "License" shall mean the terms and conditions for use, reproduction,
12
- and distribution as defined by Sections 1 through 9 of this document.
13
-
14
- "Licensor" shall mean the copyright owner or entity authorized by
15
- the copyright owner that is granting the License.
16
-
17
- "Legal Entity" shall mean the union of the acting entity and all
18
- other entities that control, are controlled by, or are under common
19
- control with that entity. For the purposes of this definition,
20
- "control" means (i) the power, direct or indirect, to cause the
21
- direction or management of such entity, whether by contract or
22
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
23
- outstanding shares, or (iii) beneficial ownership of such entity.
24
-
25
- "You" (or "Your") shall mean an individual or Legal Entity
26
- exercising permissions granted by this License.
27
-
28
- "Source" form shall mean the preferred form for making modifications,
29
- including but not limited to software source code, documentation
30
- source, and configuration files.
31
-
32
- "Object" form shall mean any form resulting from mechanical
33
- transformation or translation of a Source form, including but
34
- not limited to compiled object code, generated documentation,
35
- and conversions to other media types.
36
-
37
- "Work" shall mean the work of authorship, whether in Source or
38
- Object form, made available under the License, as indicated by a
39
- copyright notice that is included in or attached to the work
40
- (an example is provided in the Appendix below).
41
-
42
- "Derivative Works" shall mean any work, whether in Source or Object
43
- form, that is based on (or derived from) the Work and for which the
44
- editorial revisions, annotations, elaborations, or other modifications
45
- represent, as a whole, an original work of authorship. For the purposes
46
- of this License, Derivative Works shall not include works that remain
47
- separable from, or merely link (or bind by name) to the interfaces of,
48
- the Work and Derivative Works thereof.
49
-
50
- "Contribution" shall mean any work of authorship, including
51
- the original version of the Work and any modifications or additions
52
- to that Work or Derivative Works thereof, that is intentionally
53
- submitted to Licensor for inclusion in the Work by the copyright owner
54
- or by an individual or Legal Entity authorized to submit on behalf of
55
- the copyright owner. For the purposes of this definition, "submitted"
56
- means any form of electronic, verbal, or written communication sent
57
- to the Licensor or its representatives, including but not limited to
58
- communication on electronic mailing lists, source code control systems,
59
- and issue tracking systems that are managed by, or on behalf of, the
60
- Licensor for the purpose of discussing and improving the Work, but
61
- excluding communication that is conspicuously marked or otherwise
62
- designated in writing by the copyright owner as "Not a Contribution."
63
-
64
- "Contributor" shall mean Licensor and any individual or Legal Entity
65
- on behalf of whom a Contribution has been received by Licensor and
66
- subsequently incorporated within the Work.
67
-
68
- 2. Grant of Copyright License. Subject to the terms and conditions of
69
- this License, each Contributor hereby grants to You a perpetual,
70
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
71
- copyright license to reproduce, prepare Derivative Works of,
72
- publicly display, publicly perform, sublicense, and distribute the
73
- Work and such Derivative Works in Source or Object form.
74
-
75
- 3. Grant of Patent License. Subject to the terms and conditions of
76
- this License, each Contributor hereby grants to You a perpetual,
77
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
78
- (except as stated in this section) patent license to make, have made,
79
- use, offer to sell, sell, import, and otherwise transfer the Work,
80
- where such license applies only to those patent claims licensable
81
- by such Contributor that are necessarily infringed by their
82
- Contribution(s) alone or by combination of their Contribution(s)
83
- with the Work to which such Contribution(s) was submitted. If You
84
- institute patent litigation against any entity (including a
85
- cross-claim or counterclaim in a lawsuit) alleging that the Work
86
- or a Contribution incorporated within the Work constitutes direct
87
- or contributory patent infringement, then any patent licenses
88
- granted to You under this License for that Work shall terminate
89
- as of the date such litigation is filed.
90
-
91
- 4. Redistribution. You may reproduce and distribute copies of the
92
- Work or Derivative Works thereof in any medium, with or without
93
- modifications, and in Source or Object form, provided that You
94
- meet the following conditions:
95
-
96
- (a) You must give any other recipients of the Work or
97
- Derivative Works a copy of this License; and
98
-
99
- (b) You must cause any modified files to carry prominent notices
100
- stating that You changed the files; and
101
-
102
- (c) You must retain, in the Source form of any Derivative Works
103
- that You distribute, all copyright, patent, trademark, and
104
- attribution notices from the Source form of the Work,
105
- excluding those notices that do not pertain to any part of
106
- the Derivative Works; and
107
-
108
- (d) If the Work includes a "NOTICE" text file as part of its
109
- distribution, then any Derivative Works that You distribute must
110
- include a readable copy of the attribution notices contained
111
- within such NOTICE file, excluding those notices that do not
112
- pertain to any part of the Derivative Works, in at least one
113
- of the following places: within a NOTICE text file distributed
114
- as part of the Derivative Works; within the Source form or
115
- documentation, if provided along with the Derivative Works; or,
116
- within a display generated by the Derivative Works, if and
117
- wherever such third-party notices normally appear. The contents
118
- of the NOTICE file are for informational purposes only and
119
- do not modify the License. You may add Your own attribution
120
- notices within Derivative Works that You distribute, alongside
121
- or as an addendum to the NOTICE text from the Work, provided
122
- that such additional attribution notices cannot be construed
123
- as modifying the License.
124
-
125
- You may add Your own copyright statement to Your modifications and
126
- may provide additional or different license terms and conditions
127
- for use, reproduction, or distribution of Your modifications, or
128
- for any such Derivative Works as a whole, provided Your use,
129
- reproduction, and distribution of the Work otherwise complies with
130
- the conditions stated in this License.
131
-
132
- 5. Submission of Contributions. Unless You explicitly state otherwise,
133
- any Contribution intentionally submitted for inclusion in the Work
134
- by You to the Licensor shall be under the terms and conditions of
135
- this License, without any additional terms or conditions.
136
- Notwithstanding the above, nothing herein shall supersede or modify
137
- the terms of any separate license agreement you may have executed
138
- with Licensor regarding such Contributions.
139
-
140
- 6. Trademarks. This License does not grant permission to use the trade
141
- names, trademarks, service marks, or product names of the Licensor,
142
- except as required for reasonable and customary use in describing the
143
- origin of the Work and reproducing the content of the NOTICE file.
144
-
145
- 7. Disclaimer of Warranty. Unless required by applicable law or
146
- agreed to in writing, Licensor provides the Work (and each
147
- Contributor provides its Contributions) on an "AS IS" BASIS,
148
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
149
- implied, including, without limitation, any warranties or conditions
150
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
151
- PARTICULAR PURPOSE. You are solely responsible for determining the
152
- appropriateness of using or redistributing the Work and assume any
153
- risks associated with Your exercise of permissions under this License.
154
-
155
- 8. Limitation of Liability. In no event and under no legal theory,
156
- whether in tort (including negligence), contract, or otherwise,
157
- unless required by applicable law (such as deliberate and grossly
158
- negligent acts) or agreed to in writing, shall any Contributor be
159
- liable to You for damages, including any direct, indirect, special,
160
- incidental, or consequential damages of any character arising as a
161
- result of this License or out of the use or inability to use the
162
- Work (including but not limited to damages for loss of goodwill,
163
- work stoppage, computer failure or malfunction, or any and all
164
- other commercial damages or losses), even if such Contributor
165
- has been advised of the possibility of such damages.
166
-
167
- 9. Accepting Warranty or Additional Liability. While redistributing
168
- the Work or Derivative Works thereof, You may choose to offer,
169
- and charge a fee for, acceptance of support, warranty, indemnity,
170
- or other liability obligations and/or rights consistent with this
171
- License. However, in accepting such obligations, You may act only
172
- on Your own behalf and on Your sole responsibility, not on behalf
173
- of any other Contributor, and only if You agree to indemnify,
174
- defend, and hold each Contributor harmless for any liability
175
- incurred by, or claims asserted against, such Contributor by reason
176
- of your accepting any such warranty or additional liability.
177
-
178
- END OF TERMS AND CONDITIONS
179
-
180
- APPENDIX: How to apply the Apache License to your work.
181
-
182
- To apply the Apache License to your work, attach the following
183
- boilerplate notice, with the fields enclosed by brackets "{}"
184
- replaced with your own identifying information. (Don't include
185
- the brackets!) The text should be enclosed in the appropriate
186
- comment syntax for the file format. We also recommend that a
187
- file or class name and description of purpose be included on the
188
- same "printed page" as the copyright notice for easier
189
- identification within third-party archives.
190
-
191
- Copyright {yyyy} {name of copyright owner}
192
-
193
- Licensed under the Apache License, Version 2.0 (the "License");
194
- you may not use this file except in compliance with the License.
195
- You may obtain a copy of the License at
196
-
197
- http://www.apache.org/licenses/LICENSE-2.0
198
-
199
- Unless required by applicable law or agreed to in writing, software
200
- distributed under the License is distributed on an "AS IS" BASIS,
201
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
202
- See the License for the specific language governing permissions and
203
- limitations under the License.
204
-
205
-------------------------------------------------------------------------------------
206
-
207
-This product bundles various third-party components under other open source licenses.
208
-This section summarizes those components and their licenses. See licenses/
209
-for text of these licenses.
210
-
211
-
212
-Apache-2.0
213
-* third_party/nvbench (with LLVM exception)
214
-* third_party/flashinfer
215
-
216
-BSD-3-Clause:
217
-* third_party/cutlass
\ No newline at end of file
csrc/punica/bgmv/bgmv_bf16_bf16_bf16.cu
deleted
-5
@@ -1,5 +0,0 @@
1
-#include "bgmv_config.h"
2
-#include "bgmv_impl.cuh"
3
-
4
-FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_bfloat16, nv_bfloat16, nv_bfloat16)
5
-FOR_INST_BGMV_WIDE_NARROW(INST_BGMV_ONESIDE, nv_bfloat16, nv_bfloat16, nv_bfloat16)
csrc/punica/bgmv/bgmv_bf16_fp32_bf16.cu
deleted
-5
@@ -1,5 +0,0 @@
1
-#include "bgmv_config.h"
2
-#include "bgmv_impl.cuh"
3
-
4
-FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_bfloat16, float, nv_bfloat16)
5
-FOR_INST_BGMV_WIDE_NARROW(INST_BGMV_ONESIDE, nv_bfloat16, float, nv_bfloat16)
csrc/punica/bgmv/bgmv_config.h
deleted
-218
@@ -1,218 +0,0 @@
1
-#pragma once
2
-
3
-template <int feat_in, int feat_out, typename in_T, typename out_T,
4
- typename W_T>
5
-void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
6
- const W_T *__restrict__ W,
7
- const int64_t *__restrict__ indicies, int64_t y_offset,
8
- int64_t full_y_size, int64_t batch_size, int64_t num_layers,
9
- int64_t layer_idx, float scale);
10
-
11
-// clang-format off
12
-
13
-#define FOR_BGMV_WIDE(f, in_T, out_T, W_T, narrow) \
14
- f(in_T, out_T, W_T, narrow, 128) \
15
- f(in_T, out_T, W_T, narrow, 256) \
16
- f(in_T, out_T, W_T, narrow, 512) \
17
- f(in_T, out_T, W_T, narrow, 640) \
18
- f(in_T, out_T, W_T, narrow, 768) \
19
- f(in_T, out_T, W_T, narrow, 896) \
20
- f(in_T, out_T, W_T, narrow, 1024) \
21
- f(in_T, out_T, W_T, narrow, 1152) \
22
- f(in_T, out_T, W_T, narrow, 1216) \
23
- f(in_T, out_T, W_T, narrow, 1280) \
24
- f(in_T, out_T, W_T, narrow, 1536) \
25
- f(in_T, out_T, W_T, narrow, 1664) \
26
- f(in_T, out_T, W_T, narrow, 1728) \
27
- f(in_T, out_T, W_T, narrow, 1792) \
28
- f(in_T, out_T, W_T, narrow, 2048) \
29
- f(in_T, out_T, W_T, narrow, 2240) \
30
- f(in_T, out_T, W_T, narrow, 2304) \
31
- f(in_T, out_T, W_T, narrow, 2368) \
32
- f(in_T, out_T, W_T, narrow, 2432) \
33
- f(in_T, out_T, W_T, narrow, 2560) \
34
- f(in_T, out_T, W_T, narrow, 2752) \
35
- f(in_T, out_T, W_T, narrow, 2816) \
36
- f(in_T, out_T, W_T, narrow, 3072) \
37
- f(in_T, out_T, W_T, narrow, 3328) \
38
- f(in_T, out_T, W_T, narrow, 3456) \
39
- f(in_T, out_T, W_T, narrow, 3584) \
40
- f(in_T, out_T, W_T, narrow, 3712) \
41
- f(in_T, out_T, W_T, narrow, 4096) \
42
- f(in_T, out_T, W_T, narrow, 4480) \
43
- f(in_T, out_T, W_T, narrow, 4608) \
44
- f(in_T, out_T, W_T, narrow, 4736) \
45
- f(in_T, out_T, W_T, narrow, 4864) \
46
- f(in_T, out_T, W_T, narrow, 5120) \
47
- f(in_T, out_T, W_T, narrow, 5504) \
48
- f(in_T, out_T, W_T, narrow, 5632) \
49
- f(in_T, out_T, W_T, narrow, 5888) \
50
- f(in_T, out_T, W_T, narrow, 6144) \
51
- f(in_T, out_T, W_T, narrow, 6400) \
52
- f(in_T, out_T, W_T, narrow, 6848) \
53
- f(in_T, out_T, W_T, narrow, 6912) \
54
- f(in_T, out_T, W_T, narrow, 7168) \
55
- f(in_T, out_T, W_T, narrow, 7424) \
56
- f(in_T, out_T, W_T, narrow, 8192) \
57
- f(in_T, out_T, W_T, narrow, 8960) \
58
- f(in_T, out_T, W_T, narrow, 9216) \
59
- f(in_T, out_T, W_T, narrow, 9472) \
60
- f(in_T, out_T, W_T, narrow, 10240) \
61
- f(in_T, out_T, W_T, narrow, 11008) \
62
- f(in_T, out_T, W_T, narrow, 11264) \
63
- f(in_T, out_T, W_T, narrow, 12288) \
64
- f(in_T, out_T, W_T, narrow, 13696) \
65
- f(in_T, out_T, W_T, narrow, 13824) \
66
- f(in_T, out_T, W_T, narrow, 14336) \
67
- f(in_T, out_T, W_T, narrow, 14784) \
68
- f(in_T, out_T, W_T, narrow, 14848) \
69
- f(in_T, out_T, W_T, narrow, 15360) \
70
- f(in_T, out_T, W_T, narrow, 16384) \
71
- f(in_T, out_T, W_T, narrow, 18944) \
72
- f(in_T, out_T, W_T, narrow, 20480) \
73
- f(in_T, out_T, W_T, narrow, 22016) \
74
- f(in_T, out_T, W_T, narrow, 22528) \
75
- f(in_T, out_T, W_T, narrow, 24576) \
76
- f(in_T, out_T, W_T, narrow, 27392) \
77
- f(in_T, out_T, W_T, narrow, 27648) \
78
- f(in_T, out_T, W_T, narrow, 28672) \
79
- f(in_T, out_T, W_T, narrow, 29568) \
80
- f(in_T, out_T, W_T, narrow, 29696) \
81
- f(in_T, out_T, W_T, narrow, 32000) \
82
- f(in_T, out_T, W_T, narrow, 32256) \
83
- f(in_T, out_T, W_T, narrow, 32512) \
84
- f(in_T, out_T, W_T, narrow, 32768) \
85
- f(in_T, out_T, W_T, narrow, 33024) \
86
- f(in_T, out_T, W_T, narrow, 36864) \
87
- f(in_T, out_T, W_T, narrow, 43264) \
88
- f(in_T, out_T, W_T, narrow, 49152) \
89
- f(in_T, out_T, W_T, narrow, 49408) \
90
- f(in_T, out_T, W_T, narrow, 60544) \
91
- f(in_T, out_T, W_T, narrow, 60672) \
92
- f(in_T, out_T, W_T, narrow, 64000) \
93
- f(in_T, out_T, W_T, narrow, 64256) \
94
- f(in_T, out_T, W_T, narrow, 64512) \
95
- f(in_T, out_T, W_T, narrow, 102400) \
96
- f(in_T, out_T, W_T, narrow, 102656) \
97
- f(in_T, out_T, W_T, narrow, 102912) \
98
- f(in_T, out_T, W_T, narrow, 128000) \
99
- f(in_T, out_T, W_T, narrow, 128256) \
100
- f(in_T, out_T, W_T, narrow, 128512) \
101
-
102
-
103
-// Keep above in sync with vllm/lora/layers::LogitsProcessorWithLoRA
104
-// and vllm/tests/lora/test_punica.py
105
-
106
-// Used for defining kernels going from the variety of
107
-// dim in to the narrow dim out
108
- // Using it for the fully sharded column
109
- // parallel LoRA A which splits the rank dim
110
-#define FOR_INST_BGMV_NARROW(f, in_T, out_T, W_T, narrow) \
111
- f(in_T, out_T, W_T, 128, narrow) \
112
- f(in_T, out_T, W_T, 256, narrow) \
113
- f(in_T, out_T, W_T, 512, narrow) \
114
- f(in_T, out_T, W_T, 640, narrow) \
115
- f(in_T, out_T, W_T, 768, narrow) \
116
- f(in_T, out_T, W_T, 896, narrow) \
117
- f(in_T, out_T, W_T, 1024, narrow) \
118
- f(in_T, out_T, W_T, 1152, narrow) \
119
- f(in_T, out_T, W_T, 1216, narrow) \
120
- f(in_T, out_T, W_T, 1280, narrow) \
121
- f(in_T, out_T, W_T, 1536, narrow) \
122
- f(in_T, out_T, W_T, 1664, narrow) \
123
- f(in_T, out_T, W_T, 1728, narrow) \
124
- f(in_T, out_T, W_T, 1792, narrow) \
125
- f(in_T, out_T, W_T, 2048, narrow) \
126
- f(in_T, out_T, W_T, 2240, narrow) \
127
- f(in_T, out_T, W_T, 2304, narrow) \
128
- f(in_T, out_T, W_T, 2368, narrow) \
129
- f(in_T, out_T, W_T, 2432, narrow) \
130
- f(in_T, out_T, W_T, 2560, narrow) \
131
- f(in_T, out_T, W_T, 2752, narrow) \
132
- f(in_T, out_T, W_T, 2816, narrow) \
133
- f(in_T, out_T, W_T, 3072, narrow) \
134
- f(in_T, out_T, W_T, 3328, narrow) \
135
- f(in_T, out_T, W_T, 3456, narrow) \
136
- f(in_T, out_T, W_T, 3584, narrow) \
137
- f(in_T, out_T, W_T, 3712, narrow) \
138
- f(in_T, out_T, W_T, 4096, narrow) \
139
- f(in_T, out_T, W_T, 4480, narrow) \
140
- f(in_T, out_T, W_T, 4608, narrow) \
141
- f(in_T, out_T, W_T, 4736, narrow) \
142
- f(in_T, out_T, W_T, 4864, narrow) \
143
- f(in_T, out_T, W_T, 5120, narrow) \
144
- f(in_T, out_T, W_T, 5504, narrow) \
145
- f(in_T, out_T, W_T, 5632, narrow) \
146
- f(in_T, out_T, W_T, 5888, narrow) \
147
- f(in_T, out_T, W_T, 6144, narrow) \
148
- f(in_T, out_T, W_T, 6400, narrow) \
149
- f(in_T, out_T, W_T, 6848, narrow) \
150
- f(in_T, out_T, W_T, 6912, narrow) \
151
- f(in_T, out_T, W_T, 7168, narrow) \
152
- f(in_T, out_T, W_T, 7424, narrow) \
153
- f(in_T, out_T, W_T, 8192, narrow) \
154
- f(in_T, out_T, W_T, 8960, narrow) \
155
- f(in_T, out_T, W_T, 9216, narrow) \
156
- f(in_T, out_T, W_T, 9472, narrow) \
157
- f(in_T, out_T, W_T, 10240, narrow) \
158
- f(in_T, out_T, W_T, 11008, narrow) \
159
- f(in_T, out_T, W_T, 11264, narrow) \
160
- f(in_T, out_T, W_T, 12288, narrow) \
161
- f(in_T, out_T, W_T, 13696, narrow) \
162
- f(in_T, out_T, W_T, 13824, narrow) \
163
- f(in_T, out_T, W_T, 14336, narrow) \
164
- f(in_T, out_T, W_T, 14784, narrow) \
165
- f(in_T, out_T, W_T, 14848, narrow) \
166
- f(in_T, out_T, W_T, 15360, narrow) \
167
- f(in_T, out_T, W_T, 16384, narrow) \
168
- f(in_T, out_T, W_T, 18944, narrow) \
169
- f(in_T, out_T, W_T, 20480, narrow) \
170
- f(in_T, out_T, W_T, 22016, narrow) \
171
- f(in_T, out_T, W_T, 22528, narrow) \
172
- f(in_T, out_T, W_T, 24576, narrow) \
173
- f(in_T, out_T, W_T, 27392, narrow) \
174
- f(in_T, out_T, W_T, 27648, narrow) \
175
- f(in_T, out_T, W_T, 28672, narrow) \
176
- f(in_T, out_T, W_T, 29568, narrow) \
177
- f(in_T, out_T, W_T, 29696, narrow) \
178
- f(in_T, out_T, W_T, 32000, narrow) \
179
- f(in_T, out_T, W_T, 32256, narrow) \
180
- f(in_T, out_T, W_T, 32512, narrow) \
181
- f(in_T, out_T, W_T, 32768, narrow) \
182
- f(in_T, out_T, W_T, 33024, narrow) \
183
- f(in_T, out_T, W_T, 36864, narrow) \
184
- f(in_T, out_T, W_T, 43264, narrow) \
185
- f(in_T, out_T, W_T, 49152, narrow) \
186
- f(in_T, out_T, W_T, 49408, narrow) \
187
- f(in_T, out_T, W_T, 60544, narrow) \
188
- f(in_T, out_T, W_T, 60672, narrow) \
189
- f(in_T, out_T, W_T, 64000, narrow) \
190
- f(in_T, out_T, W_T, 64256, narrow) \
191
- f(in_T, out_T, W_T, 64512, narrow) \
192
- f(in_T, out_T, W_T, 102400, narrow) \
193
- f(in_T, out_T, W_T, 102656, narrow) \
194
- f(in_T, out_T, W_T, 102912, narrow) \
195
- f(in_T, out_T, W_T, 128000, narrow) \
196
- f(in_T, out_T, W_T, 128256, narrow) \
197
- f(in_T, out_T, W_T, 128512, narrow) \
198
-// Keep above in sync with vllm/lora/layers::SamplerWithLoRA
199
-
200
-
201
-// Keep this in sync with vllm/config::LoRAConfig
202
-#define FOR_BGMV_WIDE_NARROW(f, in_T, out_T, W_T) \
203
- FOR_BGMV_WIDE(f, in_T, out_T, W_T, 8) \
204
- FOR_BGMV_WIDE(f, in_T, out_T, W_T, 16) \
205
- FOR_BGMV_WIDE(f, in_T, out_T, W_T, 32) \
206
- FOR_BGMV_WIDE(f, in_T, out_T, W_T, 64)
207
-
208
-
209
-#define FOR_INST_BGMV_WIDE_NARROW(f, in_T, out_T, W_T) \
210
- FOR_INST_BGMV_NARROW(f, in_T, out_T, W_T, 1) \
211
- FOR_INST_BGMV_NARROW(f, in_T, out_T, W_T, 2) \
212
- FOR_INST_BGMV_NARROW(f, in_T, out_T, W_T, 4) \
213
- f(in_T, out_T, W_T, 8, 64) \
214
- f(in_T, out_T, W_T, 16, 64) \
215
- f(in_T, out_T, W_T, 32, 64) \
216
- f(in_T, out_T, W_T, 64, 64)
217
-
218
-// clang-format on
csrc/punica/bgmv/bgmv_fp16_fp16_fp16.cu
deleted
-5
@@ -1,5 +0,0 @@
1
-#include "bgmv_config.h"
2
-#include "bgmv_impl.cuh"
3
-
4
-FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_half, nv_half, nv_half)
5
-FOR_INST_BGMV_WIDE_NARROW(INST_BGMV_ONESIDE, nv_half, nv_half, nv_half)
csrc/punica/bgmv/bgmv_fp16_fp32_fp16.cu
deleted
-5
@@ -1,5 +0,0 @@
1
-#include "bgmv_config.h"
2
-#include "bgmv_impl.cuh"
3
-
4
-FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_half, float, nv_half)
5
-FOR_INST_BGMV_WIDE_NARROW(INST_BGMV_ONESIDE, nv_half, float, nv_half)
csrc/punica/bgmv/bgmv_fp32_bf16_bf16.cu
deleted
-5
@@ -1,5 +0,0 @@
1
-#include "bgmv_config.h"
2
-#include "bgmv_impl.cuh"
3
-
4
-FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, float, nv_bfloat16, nv_bfloat16)
5
-FOR_INST_BGMV_WIDE_NARROW(INST_BGMV_ONESIDE, float, nv_bfloat16, nv_bfloat16)
csrc/punica/bgmv/bgmv_fp32_fp16_fp16.cu
deleted
-5
@@ -1,5 +0,0 @@
1
-#include "bgmv_config.h"
2
-#include "bgmv_impl.cuh"
3
-
4
-FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, float, nv_half, nv_half)
5
-FOR_INST_BGMV_WIDE_NARROW(INST_BGMV_ONESIDE, float, nv_half, nv_half)
csrc/punica/bgmv/bgmv_impl.cuh
deleted
-451
@@ -1,451 +0,0 @@
1
-#pragma once
2
-
3
-#include <ATen/cuda/CUDAContext.h>
4
-#ifndef USE_ROCM
5
-#include <cooperative_groups.h>
6
-#else
7
-#include <hip/hip_cooperative_groups.h>
8
-#endif
9
-#ifndef USE_ROCM
10
-#include <cuda/pipeline>
11
-#endif
12
-#include <cuda_runtime.h>
13
-#include <iostream>
14
-#include <stdio.h>
15
-
16
-#include "vec_dtypes.cuh"
17
-
18
-namespace cg = cooperative_groups;
19
-
20
-#ifdef USE_ROCM
21
-template <size_t len>
22
-__host__ __device__
23
-inline void* memcpy_blocking(void *dst, const void *src) {
24
- // Does not handle the case of long datatypes
25
- char *d = reinterpret_cast<char *>(dst);
26
- const char *s = reinterpret_cast<const char *>(src);
27
- size_t i = 0;
28
-#pragma unroll
29
- for (i = 0; i < len; ++i) {
30
- d[i] = s[i];
31
- }
32
- return dst;
33
-}
34
-#endif
35
-
36
-#ifndef USE_ROCM
37
-
38
-// nthrs = (32, 4)
39
-template <int feat_in, int feat_out, size_t vec_size, size_t X_copy_size,
40
- size_t W_copy_size, int tx, int ty, int tz, typename in_T,
41
- typename out_T, typename W_T>
42
-__global__ void
43
-bgmv_shrink_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
44
- const W_T *__restrict__ W,
45
- const int64_t *__restrict__ indicies, int64_t y_offset,
46
- int64_t full_y_size, int64_t num_layers, int64_t layer_idx,
47
- float scale) {
48
- size_t batch_idx = blockIdx.y;
49
- int64_t idx = indicies[batch_idx] * num_layers + layer_idx;
50
- if (idx < 0) {
51
- return;
52
- }
53
-
54
- auto block = cg::this_thread_block();
55
- size_t j = blockIdx.x;
56
- constexpr size_t num_pipeline_stages = 2;
57
- constexpr size_t tile_size = tx * ty * vec_size;
58
- __shared__ W_T W_shared[num_pipeline_stages * tile_size];
59
- __shared__ in_T X_shared[num_pipeline_stages * tile_size];
60
- __shared__ float y_warpwise[ty];
61
-
62
- size_t W_shared_offset[num_pipeline_stages] = {0U, 1U * tile_size};
63
- size_t X_shared_offset[num_pipeline_stages] = {0U, 1U * tile_size};
64
- auto pipe = cuda::make_pipeline();
65
-
66
- // pipeline load W/X and compute WX;
67
- pipe.producer_acquire();
68
- cuda::memcpy_async(W_shared + (threadIdx.y * tx + threadIdx.x) * vec_size,
69
- W + (idx * feat_out + j) * feat_in +
70
- (threadIdx.y * tx + threadIdx.x) * vec_size,
71
- cuda::aligned_size_t<W_copy_size>(W_copy_size), pipe);
72
- cuda::memcpy_async(X_shared + (threadIdx.y * tx + threadIdx.x) * vec_size,
73
- X + (batch_idx * feat_in) +
74
- (threadIdx.y * tx + threadIdx.x) * vec_size,
75
- cuda::aligned_size_t<X_copy_size>(X_copy_size), pipe);
76
- pipe.producer_commit();
77
- size_t copy_idx, compute_idx;
78
- float y = 0.f;
79
- vec_t<in_T, vec_size> x_vec;
80
- vec_t<W_T, vec_size> w_vec;
81
- size_t tile_idx;
82
-
83
-#pragma unroll
84
- for (tile_idx = 1; tile_idx < (feat_in + tile_size - 1) / tile_size;
85
- ++tile_idx) {
86
- copy_idx = tile_idx % num_pipeline_stages;
87
- // pipeline stage: async copy W fragment
88
- pipe.producer_acquire();
89
- if (tile_idx * tile_size + threadIdx.y * tx * vec_size < feat_in) {
90
- cuda::memcpy_async(W_shared + W_shared_offset[copy_idx] +
91
- (threadIdx.y * tx + threadIdx.x) * vec_size,
92
- W + (idx * feat_out + j) * feat_in +
93
- tile_idx * tile_size +
94
- (threadIdx.y * tx + threadIdx.x) * vec_size,
95
- cuda::aligned_size_t<W_copy_size>(W_copy_size), pipe);
96
- cuda::memcpy_async(X_shared + X_shared_offset[copy_idx] +
97
- (threadIdx.y * tx + threadIdx.x) * vec_size,
98
- X + (batch_idx * feat_in) + tile_idx * tile_size +
99
- (threadIdx.y * tx + threadIdx.x) * vec_size,
100
- cuda::aligned_size_t<X_copy_size>(X_copy_size), pipe);
101
- }
102
- pipe.producer_commit();
103
-
104
- compute_idx = (tile_idx - 1) % num_pipeline_stages;
105
- // pipeline stage: compute WX
106
- pipe.consumer_wait();
107
- block.sync();
108
- x_vec.load(X_shared + X_shared_offset[compute_idx] +
109
- (threadIdx.y * tx + threadIdx.x) * vec_size);
110
- w_vec.load(W_shared + W_shared_offset[compute_idx] +
111
- (threadIdx.y * tx + threadIdx.x) * vec_size);
112
- float sum = 0.f;
113
-#pragma unroll
114
- for (size_t i = 0; i < vec_size; ++i) {
115
- sum += float(w_vec[i]) * float(x_vec[i]) * scale;
116
- }
117
-#pragma unroll
118
- for (size_t offset = tx / 2; offset > 0; offset /= 2) {
119
- sum += __shfl_down_sync(0xffffffff, sum, offset);
120
- }
121
- y_warpwise[threadIdx.y] = sum;
122
- block.sync();
123
-#pragma unroll
124
- for (size_t i = 0; i < ty; ++i) {
125
- y += y_warpwise[i];
126
- }
127
-
128
- block.sync();
129
- pipe.consumer_release();
130
- }
131
-
132
- compute_idx = (tile_idx - 1) % num_pipeline_stages;
133
- // final pipeline stage
134
- pipe.consumer_wait();
135
- block.sync();
136
- x_vec.load(X_shared + X_shared_offset[compute_idx] +
137
- (threadIdx.y * tx + threadIdx.x) * vec_size);
138
- w_vec.load(W_shared + W_shared_offset[compute_idx] +
139
- (threadIdx.y * tx + threadIdx.x) * vec_size);
140
- float sum = 0.f;
141
-#pragma unroll
142
- for (size_t i = 0; i < vec_size; ++i) {
143
- sum += float(w_vec[i]) * float(x_vec[i]) * scale;
144
- }
145
-#pragma unroll
146
- for (size_t offset = tx / 2; offset > 0; offset /= 2) {
147
- sum += __shfl_down_sync(0xffffffff, sum, offset);
148
- }
149
- y_warpwise[threadIdx.y] =
150
- ((tile_idx - 1) * tile_size + threadIdx.y * tx * vec_size < feat_in)
151
- ? sum
152
- : 0.f;
153
- block.sync();
154
-#pragma unroll
155
- for (size_t i = 0; i < ty; ++i) {
156
- y += y_warpwise[i];
157
- }
158
-
159
- block.sync();
160
- pipe.consumer_release();
161
-
162
- // write Y;
163
- if (block.thread_rank() == 0) {
164
- Y[batch_idx * full_y_size + y_offset + j] += static_cast<out_T>(y);
165
- }
166
-}
167
-
168
-#else
169
-
170
-template <int feat_in, int feat_out, size_t vec_size, size_t X_copy_size,
171
- size_t W_copy_size, int tx, int ty, int tz, typename in_T,
172
- typename out_T, typename W_T>
173
-__global__ void
174
-bgmv_shrink_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
175
- const W_T *__restrict__ W,
176
- const int64_t *__restrict__ indicies, int64_t y_offset,
177
- int64_t full_y_size, int64_t num_layers, int64_t layer_idx,
178
- float scale) {
179
- size_t batch_idx = blockIdx.y;
180
- int64_t idx = indicies[batch_idx] * num_layers + layer_idx;
181
- if (idx < 0) {
182
- return;
183
- }
184
-
185
- size_t j = blockIdx.x;
186
- constexpr size_t tile_size = tx * ty * vec_size;
187
- constexpr size_t num_tiles = (feat_in + tile_size - 1) / tile_size;
188
- __shared__ float y_warpwise[ty];
189
-
190
- float y = 0;
191
- vec_t<in_T, vec_size> x_vec;
192
- vec_t<W_T, vec_size> w_vec;
193
- size_t tile_idx;
194
-
195
-#pragma unroll
196
- for (tile_idx = 0; tile_idx < num_tiles; ++tile_idx) {
197
- if (tile_idx * tile_size + (threadIdx.y * tx + threadIdx.x + 1) * vec_size - 1 < feat_in) {
198
- x_vec.load(X + (batch_idx * feat_in) +
199
- tile_idx * tile_size +
200
- (threadIdx.y * tx + threadIdx.x) * vec_size);
201
- w_vec.load(W + (idx * feat_out + j) * feat_in +
202
- tile_idx * tile_size +
203
- (threadIdx.y * tx + threadIdx.x) * vec_size);
204
- }
205
-
206
- float sum = 0.f;
207
-#pragma unroll
208
- for (size_t i = 0; i < vec_size; ++i) {
209
- sum += convert_type<W_T, float>(w_vec[i]) * convert_type<in_T, float>(x_vec[i]) * scale;
210
- }
211
-#pragma unroll
212
- for (size_t offset = tx / 2; offset > 0; offset /= 2) {
213
- sum += VLLM_SHFL_DOWN_SYNC(sum, offset);
214
- }
215
-
216
- __syncthreads();
217
-
218
- if (tile_idx * tile_size + (threadIdx.y * tx + threadIdx.x + 1) * vec_size - 1 < feat_in) {
219
- y += sum;
220
- }
221
- }
222
-
223
- if (threadIdx.x == 0) {
224
- y_warpwise[threadIdx.y] = y;
225
- }
226
- __syncthreads();
227
-
228
- float y_write = 0.f;
229
-#pragma unroll
230
- for (size_t i = 0; i < ty; ++i) {
231
- y_write += y_warpwise[i];
232
- }
233
-
234
- // write Y;
235
- if (threadIdx.x == 0 && threadIdx.y == 0) {
236
- size_t y_idx = batch_idx * full_y_size + y_offset + j;
237
- Y[y_idx] = vllm_add<out_T>(Y[y_idx], convert_type<float, out_T>(y_write));
238
- }
239
-}
240
-
241
-#endif
242
-
243
-// nthrs = (2, 16, 4)
244
-template <int feat_in, int feat_out, size_t vec_size, int tx, int ty, int tz,
245
- typename in_T, typename out_T, typename W_T>
246
-__global__ void
247
-bgmv_expand_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
248
- const W_T *__restrict__ W,
249
- const int64_t *__restrict__ indicies, int64_t y_offset,
250
- int64_t full_y_size, int64_t num_layers, int64_t layer_idx,
251
- float scale) {
252
- size_t batch_idx = blockIdx.y;
253
- int64_t idx = indicies[batch_idx] * num_layers + layer_idx;
254
-
255
- if (idx < 0) {
256
- return;
257
- }
258
-
259
- auto block = cg::this_thread_block();
260
- size_t tile_idx = blockIdx.x;
261
-
262
- // load X;
263
- vec_t<in_T, vec_size> x_vec;
264
- x_vec.load(X + batch_idx * feat_in + threadIdx.x * vec_size);
265
-
266
- // load W;
267
- vec_t<W_T, vec_size> w_vec;
268
- w_vec.load(W + (idx * feat_out + tile_idx * tz * ty) * feat_in +
269
- block.thread_rank() * vec_size);
270
-
271
- float sum = 0.f;
272
-#pragma unroll
273
- for (size_t i = 0; i < vec_size; ++i) {
274
-#ifndef USE_ROCM
275
- sum += float(w_vec[i]) * float(x_vec[i]) * scale;
276
-#else
277
- sum += convert_type<W_T, float>(w_vec[i]) * convert_type<in_T, float>(x_vec[i]) * scale;
278
-#endif
279
- }
280
-
281
- cg::thread_block_tile g = cg::tiled_partition<tx>(block);
282
-#pragma unroll
283
- for (size_t offset = tx / 2; offset > 0; offset /= 2) {
284
- sum += g.shfl_down(sum, offset);
285
- }
286
- sum = g.shfl(sum, 0);
287
-
288
- if (threadIdx.x == 0) {
289
-#ifndef USE_ROCM
290
- Y[batch_idx * full_y_size + y_offset + tile_idx * (tz * ty) +
291
- threadIdx.z * ty + threadIdx.y] += static_cast<out_T>(sum);
292
-#else
293
- size_t y_idx = batch_idx * full_y_size + y_offset + tile_idx * (tz * ty) +
294
- threadIdx.z * ty + threadIdx.y;
295
- Y[y_idx] = vllm_add<out_T>(Y[y_idx], convert_type<float, out_T>(sum));
296
-#endif
297
- }
298
-}
299
-
300
-template <int feat_in, int feat_out, typename in_T, typename out_T,
301
- typename W_T>
302
-void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
303
- const W_T *__restrict__ W,
304
- const int64_t *__restrict__ indicies, int64_t y_offset,
305
- int64_t full_y_size, int64_t batch_size, int64_t num_layers,
306
- int64_t layer_idx, float scale) {
307
- constexpr size_t vec_size = 8;
308
- constexpr int tz = 4;
309
- const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
310
-
311
- if constexpr (feat_in <= feat_out) {
312
- static_assert(feat_in % vec_size == 0);
313
- constexpr int tx = feat_in / vec_size;
314
-
315
- static_assert((32 % tx == 0 && feat_out % (32 / tx * tz) == 0) ||
316
- (16 % tx == 0 && feat_out % (16 / tx * tz) == 0) ||
317
- (8 % tx == 0 && feat_out % (8 / tx * tz) == 0));
318
-
319
- if constexpr (32 % tx == 0 && feat_out % (32 / tx * tz) == 0) {
320
- constexpr int ty = 32 / tx;
321
- dim3 nblks(feat_out / (ty * tz), batch_size);
322
- dim3 nthrs(tx, ty, tz);
323
-
324
- bgmv_expand_kernel<feat_in, feat_out, vec_size, tx, ty, tz>
325
- <<<nblks, nthrs, 0, stream>>>(Y, X, W, indicies, y_offset,
326
- full_y_size, num_layers, layer_idx,
327
- scale);
328
- } else if (16 % tx == 0 && feat_out % (16 / tx * tz) == 0) {
329
- constexpr int ty = 16 / tx;
330
- dim3 nblks(feat_out / (ty * tz), batch_size);
331
- dim3 nthrs(tx, ty, tz);
332
-
333
- bgmv_expand_kernel<feat_in, feat_out, vec_size, tx, ty, tz>
334
- <<<nblks, nthrs, 0, stream>>>(Y, X, W, indicies, y_offset,
335
- full_y_size, num_layers, layer_idx,
336
- scale);
337
- } else {
338
- constexpr int ty = 8 / tx;
339
- dim3 nblks(feat_out / (ty * tz), batch_size);
340
- dim3 nthrs(tx, ty, tz);
341
-
342
- bgmv_expand_kernel<feat_in, feat_out, vec_size, tx, ty, tz>
343
- <<<nblks, nthrs, 0, stream>>>(Y, X, W, indicies, y_offset,
344
- full_y_size, num_layers, layer_idx,
345
- scale);
346
- }
347
- } else {
348
-#ifndef USE_ROCM
349
- static_assert(feat_in % (vec_size * 32) == 0 ||
350
- feat_in % (vec_size * 16) == 0 ||
351
- feat_in % (vec_size * 8) == 0);
352
-
353
- if constexpr (feat_in % (vec_size * 32) == 0) {
354
- constexpr int tx = 32;
355
- constexpr int ty = 4;
356
-
357
- dim3 nblks(feat_out, batch_size);
358
- dim3 nthrs(tx, ty);
359
-
360
- bgmv_shrink_kernel<feat_in, feat_out, vec_size, vec_size * sizeof(in_T),
361
- vec_size * sizeof(W_T), tx, ty, tz>
362
- <<<nblks, nthrs, 0, stream>>>(Y, X, W, indicies, y_offset,
363
- full_y_size, num_layers, layer_idx,
364
- scale);
365
- } else if constexpr (feat_in % (vec_size / 2 * 32) == 0) {
366
- constexpr int tx = 32;
367
- constexpr int ty = 4;
368
-
369
- dim3 nblks(feat_out, batch_size);
370
- dim3 nthrs(tx, ty);
371
-
372
- bgmv_shrink_kernel<feat_in, feat_out, vec_size / 2,
373
- vec_size * sizeof(in_T) / 2,
374
- vec_size * sizeof(W_T) / 2, tx, ty, tz>
375
- <<<nblks, nthrs, 0, stream>>>(Y, X, W, indicies, y_offset,
376
- full_y_size, num_layers, layer_idx,
377
- scale);
378
- } else if constexpr (feat_in % (vec_size / 2 * 16) == 0) {
379
- constexpr int tx = 16;
380
- constexpr int ty = 4;
381
-
382
- dim3 nblks(feat_out, batch_size);
383
- dim3 nthrs(tx, ty);
384
-
385
- bgmv_shrink_kernel<feat_in, feat_out, vec_size / 2,
386
- vec_size * sizeof(in_T) / 2,
387
- vec_size * sizeof(W_T) / 2, tx, ty, tz>
388
- <<<nblks, nthrs, 0, stream>>>(Y, X, W, indicies, y_offset,
389
- full_y_size, num_layers, layer_idx,
390
- scale);
391
- }
392
-#else
393
- constexpr size_t rocm_warp_size = warpSize;
394
-
395
-#define CHECK_INPUT_TILEABLE_BY(vec_size_) \
396
- feat_in % (rocm_warp_size * vec_size_) == 0
397
-
398
-#define LAUNCH_BGMV_SHRINK_KERNELS_ROCM(factor_, vec_size_, tx_, ty_) \
399
- if constexpr (CHECK_INPUT_TILEABLE_BY(factor_)) { \
400
- constexpr size_t vec_size_shrink = vec_size_; \
401
- constexpr int tx = tx_; \
402
- constexpr int ty = ty_; \
403
- dim3 nblks(feat_out, batch_size); \
404
- dim3 nthrs(tx, ty); \
405
- bgmv_shrink_kernel<feat_in, feat_out, vec_size_shrink, \
406
- vec_size_shrink * sizeof(in_T), \
407
- vec_size_shrink * sizeof(W_T), \
408
- tx, ty, tz> \
409
- <<<nblks, nthrs, 0, stream>>>(Y, X, W, indicies, y_offset, \
410
- full_y_size, num_layers, layer_idx, \
411
- scale); \
412
- }
413
-
414
- static_assert(CHECK_INPUT_TILEABLE_BY(32) ||
415
- CHECK_INPUT_TILEABLE_BY(16) ||
416
- CHECK_INPUT_TILEABLE_BY( 8) ||
417
- CHECK_INPUT_TILEABLE_BY( 4) ||
418
- CHECK_INPUT_TILEABLE_BY( 2) ||
419
- CHECK_INPUT_TILEABLE_BY( 1));
420
-
421
- LAUNCH_BGMV_SHRINK_KERNELS_ROCM(32, vec_size, rocm_warp_size, 32/vec_size)
422
- else
423
- LAUNCH_BGMV_SHRINK_KERNELS_ROCM(16, vec_size, rocm_warp_size, 16/vec_size)
424
- else
425
- LAUNCH_BGMV_SHRINK_KERNELS_ROCM( 8, vec_size, rocm_warp_size, 8/vec_size)
426
- else
427
- LAUNCH_BGMV_SHRINK_KERNELS_ROCM( 4, vec_size, rocm_warp_size/(vec_size/4), vec_size/4)
428
- else
429
- LAUNCH_BGMV_SHRINK_KERNELS_ROCM( 2, vec_size, rocm_warp_size/(vec_size/2), vec_size/2)
430
- else
431
- LAUNCH_BGMV_SHRINK_KERNELS_ROCM( 1, vec_size, rocm_warp_size/(vec_size/1), vec_size/1)
432
-
433
-#undef CHECK_INPUT_TILEABLE_BY
434
-#undef LAUNCH_BGMV_SHRINK_KERNELS_ROCM
435
-#endif
436
- }
437
-}
438
-
439
-#define INST_BGMV(feat_in, feat_out, in_T, out_T, W_T) \
440
- template void bgmv_kernel<feat_in, feat_out>( \
441
- out_T * __restrict__ Y, const in_T *__restrict__ X, \
442
- const W_T *__restrict__ W, const int64_t *__restrict__ indicies, \
443
- int64_t y_offset, int64_t full_y_size, int64_t batch_size, \
444
- int64_t num_layers, int64_t layer_idx, float scale);
445
-
446
-#define INST_BGMV_ONESIDE(in_T, out_T, W_T, feat_in, feat_out) \
447
- INST_BGMV(feat_in, feat_out, in_T, out_T, W_T)
448
-
449
-#define INST_BGMV_TWOSIDE(in_T, out_T, W_T, narrow, wide) \
450
- INST_BGMV(narrow, wide, in_T, out_T, W_T) \
451
- INST_BGMV(wide, narrow, in_T, out_T, W_T)
csrc/punica/bgmv/generator.py
deleted
-48
@@ -1,48 +0,0 @@
1
-DTYPES = ["fp16", "bf16", "fp32"]
2
-DTYPE_MAP = {
3
- "fp16": "nv_half",
4
- "bf16": "nv_bfloat16",
5
- "fp32": "float",
6
-}
7
-
8
-TEMPLATE = """
9
-#include "bgmv_config.h"
10
-#include "bgmv_impl.cuh"
11
-
12
-FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, {input_dtype}, {output_dtype}, {weight_dtype})
13
-FOR_INST_BGMV_WIDE_NARROW(INST_BGMV_ONESIDE, {input_dtype}, {output_dtype}, {weight_dtype})
14
-""".lstrip() # noqa: E501
15
-
16
-for input_dtype in DTYPES:
17
- for output_dtype in DTYPES:
18
- for weight_dtype in DTYPES:
19
- if weight_dtype == "fp32":
20
- # FP32 weights are not supported.
21
- continue
22
- if output_dtype == "fp32":
23
- # LoRA A matrix.
24
- if input_dtype != weight_dtype:
25
- # NOTE(woosuk): While Punica supports the case where the
26
- # input and weight dtypes are different, we only generate
27
- # the kernels the same dtypes to reduce the binary size.
28
- continue
29
- elif input_dtype == "fp32":
30
- # LoRA B matrix.
31
- if output_dtype != weight_dtype:
32
- # NOTE(woosuk): While Punica supports the case where the
33
- # output and weight dtypes are different, we only generate
34
- # the kernels the same dtypes to reduce the binary size.
35
- continue
36
- elif not (input_dtype == output_dtype == weight_dtype):
37
- # NOTE(woosuk): While Punica supports mixed data types for
38
- # input, output, and weight, we only generate the kernels with
39
- # the same data types to reduce the binary size.
40
- continue
41
-
42
- kernel_definition = TEMPLATE.format(
43
- input_dtype=DTYPE_MAP[input_dtype],
44
- output_dtype=DTYPE_MAP[output_dtype],
45
- weight_dtype=DTYPE_MAP[weight_dtype])
46
- filename = f"bgmv_{input_dtype}_{output_dtype}_{weight_dtype}.cu"
47
- with open(filename, "w") as f:
48
- f.write(kernel_definition)
csrc/punica/bgmv/vec_dtypes.cuh
deleted
-1325
@@ -1,1325 +0,0 @@
1
-#ifndef VEC_DTYPES_CUH_
2
-#define VEC_DTYPES_CUH_
3
-
4
-#ifdef FLASHINFER_USE_FP8
5
-#include <cuda_fp8.h>
6
-#endif
7
-#include <cuda_runtime.h>
8
-
9
-#include <type_traits>
10
-
11
-#include "../type_convert.h"
12
-#include "../../cuda_compat.h"
13
-
14
-#define FLASHINFER_INLINE \
15
- inline __attribute__((always_inline)) __device__ __host__
16
-
17
-template <typename float_t, size_t vec_size>
18
-struct vec_t {
19
- FLASHINFER_INLINE float_t &operator[](size_t i);
20
- FLASHINFER_INLINE const float_t &operator[](size_t i) const;
21
- FLASHINFER_INLINE void fill(float_t val);
22
- FLASHINFER_INLINE void load(const float_t *ptr);
23
- FLASHINFER_INLINE void store(float_t *ptr) const;
24
- template <typename T>
25
- FLASHINFER_INLINE void cast_from(const vec_t<T, vec_size> &src);
26
- template <typename T>
27
- FLASHINFER_INLINE void cast_load(const T *ptr);
28
- template <typename T>
29
- FLASHINFER_INLINE void cast_store(T *ptr) const;
30
- FLASHINFER_INLINE static void memcpy(float_t *dst, const float_t *src);
31
-};
32
-
33
-template <typename src_float_t, typename tgt_float_t, size_t vec_size>
34
-FLASHINFER_INLINE void cast_from_impl(const vec_t<src_float_t, vec_size> &src,
35
- vec_t<tgt_float_t, vec_size> &dst) {
36
-#pragma unroll
37
- for (size_t i = 0; i < vec_size; ++i) {
38
- dst[i] = tgt_float_t(src[i]);
39
- }
40
-}
41
-
42
-template <typename src_float_t, typename tgt_float_t, size_t vec_size>
43
-FLASHINFER_INLINE void cast_load_impl(const src_float_t *src_ptr,
44
- vec_t<tgt_float_t, vec_size> &dst) {
45
- if constexpr (std::is_same<src_float_t, tgt_float_t>::value) {
46
- dst.load(src_ptr);
47
- } else {
48
- vec_t<src_float_t, vec_size> tmp;
49
- tmp.load(src_ptr);
50
- dst.cast_from(tmp);
51
- }
52
-}
53
-
54
-template <typename src_float_t, typename tgt_float_t, size_t vec_size>
55
-FLASHINFER_INLINE void cast_store_impl(const vec_t<src_float_t, vec_size> &src,
56
- tgt_float_t *dst_ptr) {
57
- if constexpr (std::is_same<src_float_t, tgt_float_t>::value) {
58
- src.store(dst_ptr);
59
- } else {
60
- vec_t<tgt_float_t, vec_size> tmp;
61
- tmp.cast_from(src);
62
- tmp.store(dst_ptr);
63
- }
64
-}
65
-
66
-#ifdef FLASHINFER_USE_FP8
67
-/******************* vec_t<__nv_fp8_e4m3> *******************/
68
-
69
-// __nv_fp8_e4m3 x 1
70
-template <>
71
-struct vec_t<__nv_fp8_e4m3, 1> {
72
- __nv_fp8_e4m3 data;
73
-
74
- FLASHINFER_INLINE __nv_fp8_e4m3 &operator[](size_t i) {
75
- return ((__nv_fp8_e4m3 *)(&data))[i];
76
- }
77
- FLASHINFER_INLINE const __nv_fp8_e4m3 &operator[](size_t i) const {
78
- return ((const __nv_fp8_e4m3 *)(&data))[i];
79
- }
80
- FLASHINFER_INLINE void fill(__nv_fp8_e4m3 val);
81
- FLASHINFER_INLINE void load(const __nv_fp8_e4m3 *ptr);
82
- FLASHINFER_INLINE void store(__nv_fp8_e4m3 *ptr) const;
83
- template <typename T>
84
- FLASHINFER_INLINE void cast_from(const vec_t<T, 1> &src) {
85
- cast_from_impl(src, *this);
86
- }
87
- template <typename T>
88
- FLASHINFER_INLINE void cast_load(const T *ptr) {
89
- cast_load_impl(ptr, *this);
90
- }
91
- template <typename T>
92
- FLASHINFER_INLINE void cast_store(T *ptr) const {
93
- cast_store_impl(*this, ptr);
94
- }
95
-
96
- FLASHINFER_INLINE static void memcpy(__nv_fp8_e4m3 *dst,
97
- const __nv_fp8_e4m3 *src);
98
-};
99
-
100
-FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 1>::fill(__nv_fp8_e4m3 val) {
101
- data = val;
102
-}
103
-
104
-FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 1>::load(const __nv_fp8_e4m3 *ptr) {
105
- data = *ptr;
106
-}
107
-
108
-FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 1>::store(
109
- __nv_fp8_e4m3 *ptr) const {
110
- *ptr = data;
111
-}
112
-
113
-FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 1>::memcpy(
114
- __nv_fp8_e4m3 *dst, const __nv_fp8_e4m3 *src) {
115
- *dst = *src;
116
-}
117
-
118
-// __nv_fp8_e4m3 x 2
119
-template <>
120
-struct vec_t<__nv_fp8_e4m3, 2> {
121
- __nv_fp8x2_e4m3 data;
122
-
123
- FLASHINFER_INLINE __nv_fp8_e4m3 &operator[](size_t i) {
124
- return ((__nv_fp8_e4m3 *)(&data))[i];
125
- }
126
- FLASHINFER_INLINE const __nv_fp8_e4m3 &operator[](size_t i) const {
127
- return ((const __nv_fp8_e4m3 *)(&data))[i];
128
- }
129
- FLASHINFER_INLINE void fill(__nv_fp8_e4m3 val);
130
- FLASHINFER_INLINE void load(const __nv_fp8_e4m3 *ptr);
131
- FLASHINFER_INLINE void store(__nv_fp8_e4m3 *ptr) const;
132
- template <typename T>
133
- FLASHINFER_INLINE void cast_from(const vec_t<T, 2> &src) {
134
- cast_from_impl(src, *this);
135
- }
136
- template <typename T>
137
- FLASHINFER_INLINE void cast_load(const T *ptr) {
138
- cast_load_impl(ptr, *this);
139
- }
140
- template <typename T>
141
- FLASHINFER_INLINE void cast_store(T *ptr) const {
142
- cast_store_impl(*this, ptr);
143
- }
144
-
145
- FLASHINFER_INLINE static void memcpy(__nv_fp8_e4m3 *dst,
146
- const __nv_fp8_e4m3 *src);
147
-};
148
-
149
-FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 2>::fill(__nv_fp8_e4m3 val) {
150
- data.__x =
151
- (__nv_fp8x2_storage_t(val.__x) << 8) | __nv_fp8x2_storage_t(val.__x);
152
-}
153
-
154
-FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 2>::load(const __nv_fp8_e4m3 *ptr) {
155
- data = *((__nv_fp8x2_e4m3 *)ptr);
156
-}
157
-
158
-FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 2>::store(
159
- __nv_fp8_e4m3 *ptr) const {
160
- *((__nv_fp8x2_e4m3 *)ptr) = data;
161
-}
162
-
163
-FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 2>::memcpy(
164
- __nv_fp8_e4m3 *dst, const __nv_fp8_e4m3 *src) {
165
- *((__nv_fp8x2_e4m3 *)dst) = *((__nv_fp8x2_e4m3 *)src);
166
-}
167
-
168
-// __nv_fp8_e4m3 x 4
169
-
170
-template <>
171
-struct vec_t<__nv_fp8_e4m3, 4> {
172
- __nv_fp8x4_e4m3 data;
173
-
174
- FLASHINFER_INLINE __nv_fp8_e4m3 &operator[](size_t i) {
175
- return ((__nv_fp8_e4m3 *)(&data))[i];
176
- }
177
- FLASHINFER_INLINE const __nv_fp8_e4m3 &operator[](size_t i) const {
178
- return ((const __nv_fp8_e4m3 *)(&data))[i];
179
- }
180
- FLASHINFER_INLINE void fill(__nv_fp8_e4m3 val);
181
- FLASHINFER_INLINE void load(const __nv_fp8_e4m3 *ptr);
182
- FLASHINFER_INLINE void store(__nv_fp8_e4m3 *ptr) const;
183
- template <typename T>
184
- FLASHINFER_INLINE void cast_from(const vec_t<T, 4> &src) {
185
- cast_from_impl(src, *this);
186
- }
187
- template <typename T>
188
- FLASHINFER_INLINE void cast_load(const T *ptr) {
189
- cast_load_impl(ptr, *this);
190
- }
191
- template <typename T>
192
- FLASHINFER_INLINE void cast_store(T *ptr) const {
193
- cast_store_impl(*this, ptr);
194
- }
195
-
196
- FLASHINFER_INLINE static void memcpy(__nv_fp8_e4m3 *dst,
197
- const __nv_fp8_e4m3 *src);
198
-};
199
-
200
-FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 4>::fill(__nv_fp8_e4m3 val) {
201
- data.__x = (__nv_fp8x4_storage_t(val.__x) << 24) |
202
- (__nv_fp8x4_storage_t(val.__x) << 16) |
203
- (__nv_fp8x4_storage_t(val.__x) << 8) |
204
- __nv_fp8x4_storage_t(val.__x);
205
-}
206
-
207
-FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 4>::load(const __nv_fp8_e4m3 *ptr) {
208
- data = *((__nv_fp8x4_e4m3 *)ptr);
209
-}
210
-
211
-FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 4>::store(
212
- __nv_fp8_e4m3 *ptr) const {
213
- *((__nv_fp8x4_e4m3 *)ptr) = data;
214
-}
215
-
216
-FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 4>::memcpy(
217
- __nv_fp8_e4m3 *dst, const __nv_fp8_e4m3 *src) {
218
- *((__nv_fp8x4_e4m3 *)dst) = *((__nv_fp8x4_e4m3 *)src);
219
-}
220
-
221
-// __nv_fp8_e4m3 x 8
222
-
223
-template <>
224
-struct vec_t<__nv_fp8_e4m3, 8> {
225
- uint2 data;
226
-
227
- FLASHINFER_INLINE __nv_fp8_e4m3 &operator[](size_t i) {
228
- return ((__nv_fp8_e4m3 *)(&data))[i];
229
- }
230
- FLASHINFER_INLINE const __nv_fp8_e4m3 &operator[](size_t i) const {
231
- return ((const __nv_fp8_e4m3 *)(&data))[i];
232
- }
233
- FLASHINFER_INLINE void fill(__nv_fp8_e4m3 val);
234
- FLASHINFER_INLINE void load(const __nv_fp8_e4m3 *ptr);
235
- FLASHINFER_INLINE void store(__nv_fp8_e4m3 *ptr) const;
236
- template <typename T>
237
- FLASHINFER_INLINE void cast_from(const vec_t<T, 8> &src) {
238
- cast_from_impl(src, *this);
239
- }
240
- template <typename T>
241
- FLASHINFER_INLINE void cast_load(const T *ptr) {
242
- cast_load_impl(ptr, *this);
243
- }
244
- template <typename T>
245
- FLASHINFER_INLINE void cast_store(T *ptr) const {
246
- cast_store_impl(*this, ptr);
247
- }
248
-
249
- FLASHINFER_INLINE static void memcpy(__nv_fp8_e4m3 *dst,
250
- const __nv_fp8_e4m3 *src);
251
-};
252
-
253
-FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 8>::fill(__nv_fp8_e4m3 val) {
254
- ((__nv_fp8x4_e4m3 *)(&data.x))->__x = (__nv_fp8x4_storage_t(val.__x) << 24) |
255
- (__nv_fp8x4_storage_t(val.__x) << 16) |
256
- (__nv_fp8x4_storage_t(val.__x) << 8) |
257
- __nv_fp8x4_storage_t(val.__x);
258
- ((__nv_fp8x4_e4m3 *)(&data.y))->__x = (__nv_fp8x4_storage_t(val.__x) << 24) |
259
- (__nv_fp8x4_storage_t(val.__x) << 16) |
260
- (__nv_fp8x4_storage_t(val.__x) << 8) |
261
- __nv_fp8x4_storage_t(val.__x);
262
-}
263
-
264
-FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 8>::load(const __nv_fp8_e4m3 *ptr) {
265
- data = *((uint2 *)ptr);
266
-}
267
-
268
-FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 8>::store(
269
- __nv_fp8_e4m3 *ptr) const {
270
- *((uint2 *)ptr) = data;
271
-}
272
-
273
-FLASHINFER_INLINE void vec_t<__nv_fp8_e4m3, 8>::memcpy(
274
- __nv_fp8_e4m3 *dst, const __nv_fp8_e4m3 *src) {
275
- *((__nv_fp8_e4m3 *)dst) = *((__nv_fp8_e4m3 *)src);
276
-}
277
-
278
-// __nv_fp8_e4m3 x 16 or more
279
-template <size_t vec_size>
280
-struct vec_t<__nv_fp8_e4m3, vec_size> {
281
- uint4 data[vec_size / 16];
282
-
283
- FLASHINFER_INLINE __nv_fp8_e4m3 &operator[](size_t i) {
284
- return ((__nv_fp8_e4m3 *)data)[i];
285
- }
286
- FLASHINFER_INLINE const __nv_fp8_e4m3 &operator[](size_t i) const {
287
- return ((const __nv_fp8_e4m3 *)data)[i];
288
- }
289
- FLASHINFER_INLINE void fill(__nv_fp8_e4m3 val) {
290
-#pragma unroll
291
- for (size_t i = 0; i < vec_size / 16; ++i) {
292
- ((__nv_fp8x4_e4m3 *)(&(data[i].x)))->__x =
293
- (__nv_fp8x4_storage_t(val.__x) << 24) |
294
- (__nv_fp8x4_storage_t(val.__x) << 16) |
295
- (__nv_fp8x4_storage_t(val.__x) << 8) | __nv_fp8x4_storage_t(val.__x);
296
- ((__nv_fp8x4_e4m3 *)(&(data[i].y)))->__x =
297
- (__nv_fp8x4_storage_t(val.__x) << 24) |
298
- (__nv_fp8x4_storage_t(val.__x) << 16) |
299
- (__nv_fp8x4_storage_t(val.__x) << 8) | __nv_fp8x4_storage_t(val.__x);
300
- ((__nv_fp8x4_e4m3 *)(&(data[i].z)))->__x =
301
- (__nv_fp8x4_storage_t(val.__x) << 24) |
302
- (__nv_fp8x4_storage_t(val.__x) << 16) |
303
- (__nv_fp8x4_storage_t(val.__x) << 8) | __nv_fp8x4_storage_t(val.__x);
304
- ((__nv_fp8x4_e4m3 *)(&(data[i].w)))->__x =
305
- (__nv_fp8x4_storage_t(val.__x) << 24) |
306
- (__nv_fp8x4_storage_t(val.__x) << 16) |
307
- (__nv_fp8x4_storage_t(val.__x) << 8) | __nv_fp8x4_storage_t(val.__x);
308
- }
309
- }
310
- FLASHINFER_INLINE void load(const __nv_fp8_e4m3 *ptr) {
311
-#pragma unroll
312
- for (size_t i = 0; i < vec_size / 16; ++i) {
313
- data[i] = ((uint4 *)ptr)[i];
314
- }
315
- }
316
- FLASHINFER_INLINE void store(__nv_fp8_e4m3 *ptr) const {
317
-#pragma unroll
318
- for (size_t i = 0; i < vec_size / 16; ++i) {
319
- ((uint4 *)ptr)[i] = data[i];
320
- }
321
- }
322
- template <typename T>
323
- FLASHINFER_INLINE void cast_from(const vec_t<T, vec_size> &src) {
324
- cast_from_impl(src, *this);
325
- }
326
- template <typename T>
327
- FLASHINFER_INLINE void cast_load(const T *ptr) {
328
- cast_load_impl(ptr, *this);
329
- }
330
- template <typename T>
331
- FLASHINFER_INLINE void cast_store(T *ptr) const {
332
- cast_store_impl(*this, ptr);
333
- }
334
-
335
- FLASHINFER_INLINE static void memcpy(__nv_fp8_e4m3 *dst,
336
- const __nv_fp8_e4m3 *src) {
337
-#pragma unroll
338
- for (size_t i = 0; i < vec_size / 16; ++i) {
339
- ((uint4 *)dst)[i] = ((uint4 *)src)[i];
340
- }
341
- }
342
-};
343
-
344
-/******************* vec_t<__nv_fp8_e5m2> *******************/
345
-
346
-// __nv_fp8_e5m2 x 1
347
-template <>
348
-struct vec_t<__nv_fp8_e5m2, 1> {
349
- __nv_fp8_e5m2 data;
350
-
351
- FLASHINFER_INLINE __nv_fp8_e5m2 &operator[](size_t i) {
352
- return ((__nv_fp8_e5m2 *)(&data))[i];
353
- }
354
- FLASHINFER_INLINE const __nv_fp8_e5m2 &operator[](size_t i) const {
355
- return ((const __nv_fp8_e5m2 *)(&data))[i];
356
- }
357
- FLASHINFER_INLINE void fill(__nv_fp8_e5m2 val);
358
- FLASHINFER_INLINE void load(const __nv_fp8_e5m2 *ptr);
359
- FLASHINFER_INLINE void store(__nv_fp8_e5m2 *ptr) const;
360
- template <typename T>
361
- FLASHINFER_INLINE void cast_from(const vec_t<T, 1> &src) {
362
- cast_from_impl(src, *this);
363
- }
364
- template <typename T>
365
- FLASHINFER_INLINE void cast_load(const T *ptr) {
366
- cast_load_impl(ptr, *this);
367
- }
368
- template <typename T>
369
- FLASHINFER_INLINE void cast_store(T *ptr) const {
370
- cast_store_impl(*this, ptr);
371
- }
372
-
373
- FLASHINFER_INLINE static void memcpy(__nv_fp8_e5m2 *dst,
374
- const __nv_fp8_e5m2 *src);
375
-};
376
-
377
-FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 1>::fill(__nv_fp8_e5m2 val) {
378
- data = val;
379
-}
380
-
381
-FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 1>::load(const __nv_fp8_e5m2 *ptr) {
382
- data = *ptr;
383
-}
384
-
385
-FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 1>::store(
386
- __nv_fp8_e5m2 *ptr) const {
387
- *ptr = data;
388
-}
389
-
390
-FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 1>::memcpy(
391
- __nv_fp8_e5m2 *dst, const __nv_fp8_e5m2 *src) {
392
- *dst = *src;
393
-}
394
-
395
-// __nv_fp8_e5m2 x 2
396
-template <>
397
-struct vec_t<__nv_fp8_e5m2, 2> {
398
- __nv_fp8x2_e5m2 data;
399
-
400
- FLASHINFER_INLINE __nv_fp8_e5m2 &operator[](size_t i) {
401
- return ((__nv_fp8_e5m2 *)(&data))[i];
402
- }
403
- FLASHINFER_INLINE const __nv_fp8_e5m2 &operator[](size_t i) const {
404
- return ((const __nv_fp8_e5m2 *)(&data))[i];
405
- }
406
- FLASHINFER_INLINE void fill(__nv_fp8_e5m2 val);
407
- FLASHINFER_INLINE void load(const __nv_fp8_e5m2 *ptr);
408
- FLASHINFER_INLINE void store(__nv_fp8_e5m2 *ptr) const;
409
- template <typename T>
410
- FLASHINFER_INLINE void cast_from(const vec_t<T, 2> &src) {
411
- cast_from_impl(src, *this);
412
- }
413
- template <typename T>
414
- FLASHINFER_INLINE void cast_load(const T *ptr) {
415
- cast_load_impl(ptr, *this);
416
- }
417
- template <typename T>
418
- FLASHINFER_INLINE void cast_store(T *ptr) const {
419
- cast_store_impl(*this, ptr);
420
- }
421
-
422
- FLASHINFER_INLINE static void memcpy(__nv_fp8_e5m2 *dst,
423
- const __nv_fp8_e5m2 *src);
424
-};
425
-
426
-FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 2>::fill(__nv_fp8_e5m2 val) {
427
- data.__x =
428
- (__nv_fp8x2_storage_t(val.__x) << 8) | __nv_fp8x2_storage_t(val.__x);
429
-}
430
-
431
-FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 2>::load(const __nv_fp8_e5m2 *ptr) {
432
- data = *((__nv_fp8x2_e5m2 *)ptr);
433
-}
434
-
435
-FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 2>::store(
436
- __nv_fp8_e5m2 *ptr) const {
437
- *((__nv_fp8x2_e5m2 *)ptr) = data;
438
-}
439
-
440
-FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 2>::memcpy(
441
- __nv_fp8_e5m2 *dst, const __nv_fp8_e5m2 *src) {
442
- *((__nv_fp8x2_e5m2 *)dst) = *((__nv_fp8x2_e5m2 *)src);
443
-}
444
-
445
-// __nv_fp8_e5m2 x 4
446
-
447
-template <>
448
-struct vec_t<__nv_fp8_e5m2, 4> {
449
- __nv_fp8x4_e5m2 data;
450
-
451
- FLASHINFER_INLINE __nv_fp8_e5m2 &operator[](size_t i) {
452
- return ((__nv_fp8_e5m2 *)(&data))[i];
453
- }
454
- FLASHINFER_INLINE const __nv_fp8_e5m2 &operator[](size_t i) const {
455
- return ((const __nv_fp8_e5m2 *)(&data))[i];
456
- }
457
- FLASHINFER_INLINE void fill(__nv_fp8_e5m2 val);
458
- FLASHINFER_INLINE void load(const __nv_fp8_e5m2 *ptr);
459
- FLASHINFER_INLINE void store(__nv_fp8_e5m2 *ptr) const;
460
- template <typename T>
461
- FLASHINFER_INLINE void cast_from(const vec_t<T, 4> &src) {
462
- cast_from_impl(src, *this);
463
- }
464
- template <typename T>
465
- FLASHINFER_INLINE void cast_load(const T *ptr) {
466
- cast_load_impl(ptr, *this);
467
- }
468
- template <typename T>
469
- FLASHINFER_INLINE void cast_store(T *ptr) const {
470
- cast_store_impl(*this, ptr);
471
- }
472
-
473
- FLASHINFER_INLINE static void memcpy(__nv_fp8_e5m2 *dst,
474
- const __nv_fp8_e5m2 *src);
475
-};
476
-
477
-FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 4>::fill(__nv_fp8_e5m2 val) {
478
- data.__x = (__nv_fp8x4_storage_t(val.__x) << 24) |
479
- (__nv_fp8x4_storage_t(val.__x) << 16) |
480
- (__nv_fp8x4_storage_t(val.__x) << 8) |
481
- __nv_fp8x4_storage_t(val.__x);
482
-}
483
-
484
-FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 4>::load(const __nv_fp8_e5m2 *ptr) {
485
- data = *((__nv_fp8x4_e5m2 *)ptr);
486
-}
487
-
488
-FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 4>::store(
489
- __nv_fp8_e5m2 *ptr) const {
490
- *((__nv_fp8x4_e5m2 *)ptr) = data;
491
-}
492
-
493
-FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 4>::memcpy(
494
- __nv_fp8_e5m2 *dst, const __nv_fp8_e5m2 *src) {
495
- *((__nv_fp8x4_e5m2 *)dst) = *((__nv_fp8x4_e5m2 *)src);
496
-}
497
-
498
-// __nv_fp8_e5m2 x 8
499
-
500
-template <>
501
-struct vec_t<__nv_fp8_e5m2, 8> {
502
- uint2 data;
503
-
504
- FLASHINFER_INLINE __nv_fp8_e5m2 &operator[](size_t i) {
505
- return ((__nv_fp8_e5m2 *)(&data))[i];
506
- }
507
- FLASHINFER_INLINE const __nv_fp8_e5m2 &operator[](size_t i) const {
508
- return ((const __nv_fp8_e5m2 *)(&data))[i];
509
- }
510
- FLASHINFER_INLINE void fill(__nv_fp8_e5m2 val);
511
- FLASHINFER_INLINE void load(const __nv_fp8_e5m2 *ptr);
512
- FLASHINFER_INLINE void store(__nv_fp8_e5m2 *ptr) const;
513
- template <typename T>
514
- FLASHINFER_INLINE void cast_from(const vec_t<T, 8> &src) {
515
- cast_from_impl(src, *this);
516
- }
517
- template <typename T>
518
- FLASHINFER_INLINE void cast_load(const T *ptr) {
519
- cast_load_impl(ptr, *this);
520
- }
521
- template <typename T>
522
- FLASHINFER_INLINE void cast_store(T *ptr) const {
523
- cast_store_impl(*this, ptr);
524
- }
525
-
526
- FLASHINFER_INLINE static void memcpy(__nv_fp8_e5m2 *dst,
527
- const __nv_fp8_e5m2 *src);
528
-};
529
-
530
-FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 8>::fill(__nv_fp8_e5m2 val) {
531
- ((__nv_fp8x4_e5m2 *)(&data.x))->__x = (__nv_fp8x4_storage_t(val.__x) << 24) |
532
- (__nv_fp8x4_storage_t(val.__x) << 16) |
533
- (__nv_fp8x4_storage_t(val.__x) << 8) |
534
- __nv_fp8x4_storage_t(val.__x);
535
- ((__nv_fp8x4_e5m2 *)(&data.y))->__x = (__nv_fp8x4_storage_t(val.__x) << 24) |
536
- (__nv_fp8x4_storage_t(val.__x) << 16) |
537
- (__nv_fp8x4_storage_t(val.__x) << 8) |
538
- __nv_fp8x4_storage_t(val.__x);
539
-}
540
-
541
-FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 8>::load(const __nv_fp8_e5m2 *ptr) {
542
- data = *((uint2 *)ptr);
543
-}
544
-
545
-FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 8>::store(
546
- __nv_fp8_e5m2 *ptr) const {
547
- *((uint2 *)ptr) = data;
548
-}
549
-
550
-FLASHINFER_INLINE void vec_t<__nv_fp8_e5m2, 8>::memcpy(
551
- __nv_fp8_e5m2 *dst, const __nv_fp8_e5m2 *src) {
552
- *((__nv_fp8_e5m2 *)dst) = *((__nv_fp8_e5m2 *)src);
553
-}
554
-
555
-// __nv_fp8_e5m2 x 16 or more
556
-
557
-template <size_t vec_size>
558
-struct vec_t<__nv_fp8_e5m2, vec_size> {
559
- uint4 data[vec_size / 16];
560
-
561
- FLASHINFER_INLINE __nv_fp8_e5m2 &operator[](size_t i) {
562
- return ((__nv_fp8_e5m2 *)data)[i];
563
- }
564
- FLASHINFER_INLINE const __nv_fp8_e5m2 &operator[](size_t i) const {
565
- return ((const __nv_fp8_e5m2 *)data)[i];
566
- }
567
- FLASHINFER_INLINE void fill(__nv_fp8_e5m2 val) {
568
-#pragma unroll
569
- for (size_t i = 0; i < vec_size / 16; ++i) {
570
- ((__nv_fp8x4_e5m2 *)(&(data[i].x)))->__x =
571
- (__nv_fp8x4_storage_t(val.__x) << 24) |
572
- (__nv_fp8x4_storage_t(val.__x) << 16) |
573
- (__nv_fp8x4_storage_t(val.__x) << 8) | __nv_fp8x4_storage_t(val.__x);
574
- ((__nv_fp8x4_e5m2 *)(&(data[i].y)))->__x =
575
- (__nv_fp8x4_storage_t(val.__x) << 24) |
576
- (__nv_fp8x4_storage_t(val.__x) << 16) |
577
- (__nv_fp8x4_storage_t(val.__x) << 8) | __nv_fp8x4_storage_t(val.__x);
578
- ((__nv_fp8x4_e5m2 *)(&(data[i].z)))->__x =
579
- (__nv_fp8x4_storage_t(val.__x) << 24) |
580
- (__nv_fp8x4_storage_t(val.__x) << 16) |
581
- (__nv_fp8x4_storage_t(val.__x) << 8) | __nv_fp8x4_storage_t(val.__x);
582
- ((__nv_fp8x4_e5m2 *)(&(data[i].w)))->__x =
583
- (__nv_fp8x4_storage_t(val.__x) << 24) |
584
- (__nv_fp8x4_storage_t(val.__x) << 16) |
585
- (__nv_fp8x4_storage_t(val.__x) << 8) | __nv_fp8x4_storage_t(val.__x);
586
- }
587
- }
588
- FLASHINFER_INLINE void load(const __nv_fp8_e5m2 *ptr) {
589
-#pragma unroll
590
- for (size_t i = 0; i < vec_size / 16; ++i) {
591
- data[i] = ((uint4 *)ptr)[i];
592
- }
593
- }
594
- FLASHINFER_INLINE void store(__nv_fp8_e5m2 *ptr) const {
595
-#pragma unroll
596
- for (size_t i = 0; i < vec_size / 16; ++i) {
597
- ((uint4 *)ptr)[i] = data[i];
598
- }
599
- }
600
- template <typename T>
601
- FLASHINFER_INLINE void cast_from(const vec_t<T, vec_size> &src) {
602
- cast_from_impl(src, *this);
603
- }
604
- template <typename T>
605
- FLASHINFER_INLINE void cast_load(const T *ptr) {
606
- cast_load_impl(ptr, *this);
607
- }
608
- template <typename T>
609
- FLASHINFER_INLINE void cast_store(T *ptr) const {
610
- cast_store_impl(*this, ptr);
611
- }
612
-
613
- FLASHINFER_INLINE static void memcpy(__nv_fp8_e5m2 *dst,
614
- const __nv_fp8_e5m2 *src) {
615
-#pragma unroll
616
- for (size_t i = 0; i < vec_size / 16; ++i) {
617
- ((uint4 *)dst)[i] = ((uint4 *)src)[i];
618
- }
619
- }
620
-};
621
-#endif
622
-
623
-/******************* vec_t<half> *******************/
624
-
625
-// half x 1
626
-template <>
627
-struct vec_t<half, 1> {
628
- half data;
629
-
630
- FLASHINFER_INLINE half &operator[](size_t i) { return ((half *)(&data))[i]; }
631
- FLASHINFER_INLINE const half &operator[](size_t i) const {
632
- return ((const half *)(&data))[i];
633
- }
634
- FLASHINFER_INLINE void fill(half val);
635
- FLASHINFER_INLINE void load(const half *ptr);
636
- FLASHINFER_INLINE void store(half *ptr) const;
637
- template <typename T>
638
- FLASHINFER_INLINE void cast_from(const vec_t<T, 1> &src) {
639
- cast_from_impl(src, *this);
640
- }
641
- template <typename T>
642
- FLASHINFER_INLINE void cast_load(const T *ptr) {
643
- cast_load_impl(ptr, *this);
644
- }
645
- template <typename T>
646
- FLASHINFER_INLINE void cast_store(T *ptr) const {
647
- cast_store_impl(*this, ptr);
648
- }
649
-
650
- FLASHINFER_INLINE static void memcpy(half *dst, const half *src);
651
-};
652
-
653
-FLASHINFER_INLINE void vec_t<half, 1>::fill(half val) { data = val; }
654
-
655
-FLASHINFER_INLINE void vec_t<half, 1>::load(const half *ptr) { data = *ptr; }
656
-
657
-FLASHINFER_INLINE void vec_t<half, 1>::store(half *ptr) const { *ptr = data; }
658
-
659
-FLASHINFER_INLINE void vec_t<half, 1>::memcpy(half *dst, const half *src) {
660
- *dst = *src;
661
-}
662
-
663
-// half x 2
664
-template <>
665
-struct vec_t<half, 2> {
666
- half2 data;
667
-
668
- FLASHINFER_INLINE half &operator[](size_t i) { return ((half *)(&data))[i]; }
669
- FLASHINFER_INLINE const half &operator[](size_t i) const {
670
- return ((const half *)(&data))[i];
671
- }
672
- FLASHINFER_INLINE void fill(half val);
673
- FLASHINFER_INLINE void load(const half *ptr);
674
- FLASHINFER_INLINE void store(half *ptr) const;
675
- template <typename T>
676
- FLASHINFER_INLINE void cast_from(const vec_t<T, 2> &src) {
677
- cast_from_impl(src, *this);
678
- }
679
- template <typename T>
680
- FLASHINFER_INLINE void cast_load(const T *ptr) {
681
- cast_load_impl(ptr, *this);
682
- }
683
- template <typename T>
684
- FLASHINFER_INLINE void cast_store(T *ptr) const {
685
- cast_store_impl(*this, ptr);
686
- }
687
-
688
- FLASHINFER_INLINE static void memcpy(half *dst, const half *src);
689
-};
690
-
691
-FLASHINFER_INLINE void vec_t<half, 2>::fill(half val) {
692
- data = make_half2(val, val);
693
-}
694
-
695
-FLASHINFER_INLINE void vec_t<half, 2>::load(const half *ptr) {
696
- data = *((half2 *)ptr);
697
-}
698
-
699
-FLASHINFER_INLINE void vec_t<half, 2>::store(half *ptr) const {
700
- *((half2 *)ptr) = data;
701
-}
702
-
703
-FLASHINFER_INLINE void vec_t<half, 2>::memcpy(half *dst, const half *src) {
704
- *((half2 *)dst) = *((half2 *)src);
705
-}
706
-
707
-// half x 4
708
-
709
-template <>
710
-struct vec_t<half, 4> {
711
- uint2 data;
712
-
713
- FLASHINFER_INLINE half &operator[](size_t i) { return ((half *)(&data))[i]; }
714
- FLASHINFER_INLINE const half &operator[](size_t i) const {
715
- return ((const half *)(&data))[i];
716
- }
717
- FLASHINFER_INLINE void fill(half val);
718
- FLASHINFER_INLINE void load(const half *ptr);
719
- FLASHINFER_INLINE void store(half *ptr) const;
720
- template <typename T>
721
- FLASHINFER_INLINE void cast_from(const vec_t<T, 4> &src) {
722
- cast_from_impl(src, *this);
723
- }
724
- template <typename T>
725
- FLASHINFER_INLINE void cast_load(const T *ptr) {
726
- cast_load_impl(ptr, *this);
727
- }
728
- template <typename T>
729
- FLASHINFER_INLINE void cast_store(T *ptr) const {
730
- cast_store_impl(*this, ptr);
731
- }
732
-
733
- FLASHINFER_INLINE static void memcpy(half *dst, const half *src);
734
-};
735
-
736
-FLASHINFER_INLINE void vec_t<half, 4>::fill(half val) {
737
- *(half2 *)(&data.x) = make_half2(val, val);
738
- *(half2 *)(&data.y) = make_half2(val, val);
739
-}
740
-
741
-FLASHINFER_INLINE void vec_t<half, 4>::load(const half *ptr) {
742
- data = *((uint2 *)ptr);
743
-}
744
-
745
-FLASHINFER_INLINE void vec_t<half, 4>::store(half *ptr) const {
746
- *((uint2 *)ptr) = data;
747
-}
748
-
749
-FLASHINFER_INLINE void vec_t<half, 4>::memcpy(half *dst, const half *src) {
750
- *((uint2 *)dst) = *((uint2 *)src);
751
-}
752
-
753
-// half x 8 or more
754
-
755
-template <size_t vec_size>
756
-struct vec_t<half, vec_size> {
757
- uint4 data[vec_size / 8];
758
- FLASHINFER_INLINE half &operator[](size_t i) { return ((half *)data)[i]; }
759
- FLASHINFER_INLINE const half &operator[](size_t i) const {
760
- return ((const half *)data)[i];
761
- }
762
- FLASHINFER_INLINE void fill(half val) {
763
-#pragma unroll
764
- for (size_t i = 0; i < vec_size; ++i) {
765
- *(half2 *)(&(data[i].x)) = make_half2(val, val);
766
- *(half2 *)(&(data[i].y)) = make_half2(val, val);
767
- *(half2 *)(&(data[i].z)) = make_half2(val, val);
768
- *(half2 *)(&(data[i].w)) = make_half2(val, val);
769
- }
770
- }
771
- FLASHINFER_INLINE void load(const half *ptr) {
772
-#pragma unroll
773
- for (size_t i = 0; i < vec_size / 8; ++i) {
774
- data[i] = ((uint4 *)ptr)[i];
775
- }
776
- }
777
- FLASHINFER_INLINE void store(half *ptr) const {
778
-#pragma unroll
779
- for (size_t i = 0; i < vec_size / 8; ++i) {
780
- ((uint4 *)ptr)[i] = data[i];
781
- }
782
- }
783
- template <typename T>
784
- FLASHINFER_INLINE void cast_from(const vec_t<T, vec_size> &src) {
785
- cast_from_impl(src, *this);
786
- }
787
- template <typename T>
788
- FLASHINFER_INLINE void cast_load(const T *ptr) {
789
- cast_load_impl(ptr, *this);
790
- }
791
- template <typename T>
792
- FLASHINFER_INLINE void cast_store(T *ptr) const {
793
- cast_store_impl(*this, ptr);
794
- }
795
-
796
- FLASHINFER_INLINE static void memcpy(half *dst, const half *src) {
797
-#pragma unroll
798
- for (size_t i = 0; i < vec_size / 8; ++i) {
799
- ((uint4 *)dst)[i] = ((uint4 *)src)[i];
800
- }
801
- }
802
-};
803
-
804
-/******************* vec_t<nv_bfloat16> *******************/
805
-
806
-// nv_bfloat16 x 1
807
-template <>
808
-struct vec_t<nv_bfloat16, 1> {
809
- nv_bfloat16 data;
810
-
811
- FLASHINFER_INLINE nv_bfloat16 &operator[](size_t i) {
812
- return ((nv_bfloat16 *)(&data))[i];
813
- }
814
- FLASHINFER_INLINE const nv_bfloat16 &operator[](size_t i) const {
815
- return ((const nv_bfloat16 *)(&data))[i];
816
- }
817
- FLASHINFER_INLINE void fill(nv_bfloat16 val);
818
- FLASHINFER_INLINE void load(const nv_bfloat16 *ptr);
819
- FLASHINFER_INLINE void store(nv_bfloat16 *ptr) const;
820
- template <typename T>
821
- FLASHINFER_INLINE void cast_from(const vec_t<T, 1> &src) {
822
- cast_from_impl(src, *this);
823
- }
824
- template <typename T>
825
- FLASHINFER_INLINE void cast_load(const T *ptr) {
826
- cast_load_impl(ptr, *this);
827
- }
828
- template <typename T>
829
- FLASHINFER_INLINE void cast_store(T *ptr) const {
830
- cast_store_impl(*this, ptr);
831
- }
832
-
833
- FLASHINFER_INLINE static void memcpy(nv_bfloat16 *dst,
834
- const nv_bfloat16 *src);
835
-};
836
-
837
-FLASHINFER_INLINE void vec_t<nv_bfloat16, 1>::fill(nv_bfloat16 val) {
838
- data = val;
839
-}
840
-
841
-FLASHINFER_INLINE void vec_t<nv_bfloat16, 1>::load(const nv_bfloat16 *ptr) {
842
- data = *ptr;
843
-}
844
-
845
-FLASHINFER_INLINE void vec_t<nv_bfloat16, 1>::store(nv_bfloat16 *ptr) const {
846
- *ptr = data;
847
-}
848
-
849
-FLASHINFER_INLINE void vec_t<nv_bfloat16, 1>::memcpy(nv_bfloat16 *dst,
850
- const nv_bfloat16 *src) {
851
- *dst = *src;
852
-}
853
-
854
-// nv_bfloat16 x 2
855
-template <>
856
-struct vec_t<nv_bfloat16, 2> {
857
- nv_bfloat162 data;
858
-
859
- FLASHINFER_INLINE nv_bfloat16 &operator[](size_t i) {
860
- return ((nv_bfloat16 *)(&data))[i];
861
- }
862
- FLASHINFER_INLINE const nv_bfloat16 &operator[](size_t i) const {
863
- return ((const nv_bfloat16 *)(&data))[i];
864
- }
865
- FLASHINFER_INLINE void fill(nv_bfloat16 val);
866
- FLASHINFER_INLINE void load(const nv_bfloat16 *ptr);
867
- FLASHINFER_INLINE void store(nv_bfloat16 *ptr) const;
868
- template <typename T>
869
- FLASHINFER_INLINE void cast_from(const vec_t<T, 2> &src) {
870
- cast_from_impl(src, *this);
871
- }
872
- template <typename T>
873
- FLASHINFER_INLINE void cast_load(const T *ptr) {
874
- cast_load_impl(ptr, *this);
875
- }
876
- template <typename T>
877
- FLASHINFER_INLINE void cast_store(T *ptr) const {
878
- cast_store_impl(*this, ptr);
879
- }
880
-
881
- FLASHINFER_INLINE static void memcpy(nv_bfloat16 *dst,
882
- const nv_bfloat16 *src);
883
-};
884
-
885
-FLASHINFER_INLINE void vec_t<nv_bfloat16, 2>::fill(nv_bfloat16 val) {
886
- data = make_bfloat162(val, val);
887
-}
888
-
889
-FLASHINFER_INLINE void vec_t<nv_bfloat16, 2>::load(const nv_bfloat16 *ptr) {
890
- data = *((nv_bfloat162 *)ptr);
891
-}
892
-
893
-FLASHINFER_INLINE void vec_t<nv_bfloat16, 2>::store(nv_bfloat16 *ptr) const {
894
- *((nv_bfloat162 *)ptr) = data;
895
-}
896
-
897
-FLASHINFER_INLINE void vec_t<nv_bfloat16, 2>::memcpy(nv_bfloat16 *dst,
898
- const nv_bfloat16 *src) {
899
- *((nv_bfloat162 *)dst) = *((nv_bfloat162 *)src);
900
-}
901
-
902
-// nv_bfloat16 x 4
903
-
904
-template <>
905
-struct vec_t<nv_bfloat16, 4> {
906
- uint2 data;
907
-
908
- FLASHINFER_INLINE nv_bfloat16 &operator[](size_t i) {
909
- return ((nv_bfloat16 *)(&data))[i];
910
- }
911
- FLASHINFER_INLINE const nv_bfloat16 &operator[](size_t i) const {
912
- return ((const nv_bfloat16 *)(&data))[i];
913
- }
914
- FLASHINFER_INLINE void fill(nv_bfloat16 val);
915
- FLASHINFER_INLINE void load(const nv_bfloat16 *ptr);
916
- FLASHINFER_INLINE void store(nv_bfloat16 *ptr) const;
917
- template <typename T>
918
- FLASHINFER_INLINE void cast_from(const vec_t<T, 4> &src) {
919
- cast_from_impl(src, *this);
920
- }
921
- template <typename T>
922
- FLASHINFER_INLINE void cast_load(const T *ptr) {
923
- cast_load_impl(ptr, *this);
924
- }
925
- template <typename T>
926
- FLASHINFER_INLINE void cast_store(T *ptr) const {
927
- cast_store_impl(*this, ptr);
928
- }
929
-
930
- FLASHINFER_INLINE static void memcpy(nv_bfloat16 *dst,
931
- const nv_bfloat16 *src);
932
-};
933
-
934
-FLASHINFER_INLINE void vec_t<nv_bfloat16, 4>::fill(nv_bfloat16 val) {
935
- *(nv_bfloat162 *)(&data.x) = make_bfloat162(val, val);
936
- *(nv_bfloat162 *)(&data.y) = make_bfloat162(val, val);
937
-}
938
-
939
-FLASHINFER_INLINE void vec_t<nv_bfloat16, 4>::load(const nv_bfloat16 *ptr) {
940
- data = *((uint2 *)ptr);
941
-}
942
-
943
-FLASHINFER_INLINE void vec_t<nv_bfloat16, 4>::store(nv_bfloat16 *ptr) const {
944
- *((uint2 *)ptr) = data;
945
-}
946
-
947
-FLASHINFER_INLINE void vec_t<nv_bfloat16, 4>::memcpy(nv_bfloat16 *dst,
948
- const nv_bfloat16 *src) {
949
- *((uint2 *)dst) = *((uint2 *)src);
950
-}
951
-
952
-// nv_bfloat16 x 8 or more
953
-
954
-template <size_t vec_size>
955
-struct vec_t<nv_bfloat16, vec_size> {
956
- uint4 data[vec_size / 8];
957
-
958
- FLASHINFER_INLINE nv_bfloat16 &operator[](size_t i) {
959
- return ((nv_bfloat16 *)data)[i];
960
- }
961
- FLASHINFER_INLINE const nv_bfloat16 &operator[](size_t i) const {
962
- return ((const nv_bfloat16 *)data)[i];
963
- }
964
- FLASHINFER_INLINE void fill(nv_bfloat16 val) {
965
-#pragma unoll
966
- for (size_t i = 0; i < vec_size / 8; ++i) {
967
- *(nv_bfloat162 *)(&(data[i].x)) = make_bfloat162(val, val);
968
- *(nv_bfloat162 *)(&(data[i].y)) = make_bfloat162(val, val);
969
- *(nv_bfloat162 *)(&(data[i].z)) = make_bfloat162(val, val);
970
- *(nv_bfloat162 *)(&(data[i].w)) = make_bfloat162(val, val);
971
- }
972
- }
973
- FLASHINFER_INLINE void load(const nv_bfloat16 *ptr) {
974
-#pragma unoll
975
- for (size_t i = 0; i < vec_size / 8; ++i) {
976
- data[i] = ((uint4 *)ptr)[i];
977
- }
978
- }
979
- FLASHINFER_INLINE void store(nv_bfloat16 *ptr) const {
980
-#pragma unoll
981
- for (size_t i = 0; i < vec_size / 8; ++i) {
982
- ((uint4 *)ptr)[i] = data[i];
983
- }
984
- }
985
- template <typename T>
986
- FLASHINFER_INLINE void cast_from(const vec_t<T, vec_size> &src) {
987
- cast_from_impl(src, *this);
988
- }
989
- template <typename T>
990
- FLASHINFER_INLINE void cast_load(const T *ptr) {
991
- cast_load_impl(ptr, *this);
992
- }
993
- template <typename T>
994
- FLASHINFER_INLINE void cast_store(T *ptr) const {
995
- cast_store_impl(*this, ptr);
996
- }
997
-
998
- FLASHINFER_INLINE static void memcpy(nv_bfloat16 *dst,
999
- const nv_bfloat16 *src) {
1000
-#pragma unoll
1001
- for (size_t i = 0; i < vec_size / 8; ++i) {
1002
- ((uint4 *)dst)[i] = ((uint4 *)src)[i];
1003
- }
1004
- }
1005
-};
1006
-
1007
-/******************* vec_t<float> *******************/
1008
-
1009
-// float x 1
1010
-
1011
-template <>
1012
-struct vec_t<float, 1> {
1013
- float data;
1014
-
1015
- FLASHINFER_INLINE float &operator[](size_t i) {
1016
- return ((float *)(&data))[i];
1017
- }
1018
- FLASHINFER_INLINE const float &operator[](size_t i) const {
1019
- return ((const float *)(&data))[i];
1020
- }
1021
- FLASHINFER_INLINE void fill(float val);
1022
- FLASHINFER_INLINE void load(const float *ptr);
1023
- FLASHINFER_INLINE void store(float *ptr) const;
1024
- template <typename T>
1025
- FLASHINFER_INLINE void cast_from(const vec_t<T, 1> &src) {
1026
- cast_from_impl(src, *this);
1027
- }
1028
- template <typename T>
1029
- FLASHINFER_INLINE void cast_load(const T *ptr) {
1030
- cast_load_impl(ptr, *this);
1031
- }
1032
- template <typename T>
1033
- FLASHINFER_INLINE void cast_store(T *ptr) const {
1034
- cast_store_impl(*this, ptr);
1035
- }
1036
-
1037
- FLASHINFER_INLINE static void memcpy(float *dst, const float *src);
1038
-};
1039
-
1040
-FLASHINFER_INLINE void vec_t<float, 1>::fill(float val) { data = val; }
1041
-
1042
-FLASHINFER_INLINE void vec_t<float, 1>::load(const float *ptr) { data = *ptr; }
1043
-
1044
-FLASHINFER_INLINE void vec_t<float, 1>::store(float *ptr) const { *ptr = data; }
1045
-
1046
-FLASHINFER_INLINE void vec_t<float, 1>::memcpy(float *dst, const float *src) {
1047
- *dst = *src;
1048
-}
1049
-
1050
-// float x 2
1051
-
1052
-template <>
1053
-struct vec_t<float, 2> {
1054
- float2 data;
1055
-
1056
- FLASHINFER_INLINE float &operator[](size_t i) {
1057
- return ((float *)(&data))[i];
1058
- }
1059
- FLASHINFER_INLINE const float &operator[](size_t i) const {
1060
- return ((const float *)(&data))[i];
1061
- }
1062
- FLASHINFER_INLINE void fill(float val);
1063
- FLASHINFER_INLINE void load(const float *ptr);
1064
- FLASHINFER_INLINE void store(float *ptr) const;
1065
- template <typename T>
1066
- FLASHINFER_INLINE void cast_from(const vec_t<T, 2> &src) {
1067
- cast_from_impl(src, *this);
1068
- }
1069
- template <typename T>
1070
- FLASHINFER_INLINE void cast_load(const T *ptr) {
1071
- cast_load_impl(ptr, *this);
1072
- }
1073
- template <typename T>
1074
- FLASHINFER_INLINE void cast_store(T *ptr) const {
1075
- cast_store_impl(*this, ptr);
1076
- }
1077
- FLASHINFER_INLINE static void memcpy(float *dst, const float *src);
1078
-};
1079
-
1080
-FLASHINFER_INLINE void vec_t<float, 2>::fill(float val) {
1081
- data = make_float2(val, val);
1082
-}
1083
-
1084
-FLASHINFER_INLINE void vec_t<float, 2>::load(const float *ptr) {
1085
- data = *((float2 *)ptr);
1086
-}
1087
-
1088
-FLASHINFER_INLINE void vec_t<float, 2>::store(float *ptr) const {
1089
- *((float2 *)ptr) = data;
1090
-}
1091
-
1092
-FLASHINFER_INLINE void vec_t<float, 2>::memcpy(float *dst, const float *src) {
1093
- *((float2 *)dst) = *((float2 *)src);
1094
-}
1095
-
1096
-// float x 4 or more
1097
-template <size_t vec_size>
1098
-struct vec_t<float, vec_size> {
1099
- float4 data[vec_size / 4];
1100
-
1101
- FLASHINFER_INLINE float &operator[](size_t i) { return ((float *)(data))[i]; }
1102
- FLASHINFER_INLINE const float &operator[](size_t i) const {
1103
- return ((const float *)(data))[i];
1104
- }
1105
- FLASHINFER_INLINE void fill(float val) {
1106
-#pragma unroll
1107
- for (size_t i = 0; i < vec_size / 4; ++i) {
1108
- data[i] = make_float4(val, val, val, val);
1109
- }
1110
- }
1111
- FLASHINFER_INLINE void load(const float *ptr) {
1112
-#pragma unroll
1113
- for (size_t i = 0; i < vec_size / 4; ++i) {
1114
- data[i] = ((float4 *)ptr)[i];
1115
- }
1116
- }
1117
- FLASHINFER_INLINE void store(float *ptr) const {
1118
-#pragma unroll
1119
- for (size_t i = 0; i < vec_size / 4; ++i) {
1120
- ((float4 *)ptr)[i] = data[i];
1121
- }
1122
- }
1123
- template <typename T>
1124
- FLASHINFER_INLINE void cast_from(const vec_t<T, vec_size> &src) {
1125
- cast_from_impl(src, *this);
1126
- }
1127
- template <typename T>
1128
- FLASHINFER_INLINE void cast_load(const T *ptr) {
1129
- cast_load_impl(ptr, *this);
1130
- }
1131
- template <typename T>
1132
- FLASHINFER_INLINE void cast_store(T *ptr) const {
1133
- cast_store_impl(*this, ptr);
1134
- }
1135
- FLASHINFER_INLINE static void memcpy(float *dst, const float *src) {
1136
-#pragma unroll
1137
- for (size_t i = 0; i < vec_size / 4; ++i) {
1138
- ((float4 *)dst)[i] = ((float4 *)src)[i];
1139
- }
1140
- }
1141
-};
1142
-
1143
-/******************* vec_t type cast *******************/
1144
-
1145
-template <size_t vec_size>
1146
-FLASHINFER_INLINE void cast_from_impl(const vec_t<half, vec_size> &src,
1147
- vec_t<float, vec_size> &dst) {
1148
- if constexpr (vec_size == 1) {
1149
- dst.data = float(src.data);
1150
- } else {
1151
-#pragma unroll
1152
- for (size_t i = 0; i < vec_size / 2; ++i) {
1153
- ((float2 *)(&dst.data))[i] = __half22float2(((half2 *)(&src.data))[i]);
1154
- }
1155
- }
1156
-}
1157
-
1158
-template <size_t vec_size>
1159
-FLASHINFER_INLINE void cast_from_impl(const vec_t<float, vec_size> &src,
1160
- vec_t<half, vec_size> &dst) {
1161
- if constexpr (vec_size == 1) {
1162
- dst.data = half(src.data);
1163
- } else {
1164
-#pragma unroll
1165
- for (size_t i = 0; i < vec_size / 2; ++i) {
1166
- ((half2 *)(&dst.data))[i] = __float22half2_rn(((float2 *)(&src.data))[i]);
1167
- }
1168
- }
1169
-}
1170
-
1171
-template <size_t vec_size>
1172
-FLASHINFER_INLINE void cast_from_impl(const vec_t<nv_bfloat16, vec_size> &src,
1173
- vec_t<float, vec_size> &dst) {
1174
- if constexpr (vec_size == 1) {
1175
- dst.data = float(src.data);
1176
- } else {
1177
-#pragma unroll
1178
- for (size_t i = 0; i < vec_size / 2; ++i) {
1179
- ((float2 *)(&dst.data))[i] =
1180
- __bfloat1622float2(((nv_bfloat162 *)(&src.data))[i]);
1181
- }
1182
- }
1183
-}
1184
-
1185
-template <size_t vec_size>
1186
-FLASHINFER_INLINE void cast_from_impl(const vec_t<float, vec_size> &src,
1187
- vec_t<nv_bfloat16, vec_size> &dst) {
1188
- if constexpr (vec_size == 1) {
1189
- dst.data = nv_bfloat16(src.data);
1190
- } else {
1191
-#pragma unroll
1192
- for (size_t i = 0; i < vec_size / 2; ++i) {
1193
- ((nv_bfloat162 *)(&dst.data))[i] =
1194
- __float22bfloat162_rn(((float2 *)(&src.data))[i]);
1195
- }
1196
- }
1197
-}
1198
-
1199
-#ifdef FLASHINFER_USE_FP8
1200
-
1201
-template <size_t vec_size>
1202
-FLASHINFER_INLINE void cast_from_impl(const vec_t<__nv_fp8_e4m3, vec_size> &src,
1203
- vec_t<float, vec_size> &dst) {
1204
- if constexpr (vec_size == 1) {
1205
- dst.data = float(src.data);
1206
- } else if constexpr (vec_size == 2) {
1207
- *(float2 *)(&dst.data) = float2(*(__nv_fp8x2_e4m3 *)(&src.data));
1208
- } else {
1209
-#pragma unroll
1210
- for (size_t i = 0; i < vec_size / 4; ++i) {
1211
- ((float4 *)(&dst.data))[i] = float4(((__nv_fp8x4_e4m3 *)(&src.data))[i]);
1212
- }
1213
- }
1214
-}
1215
-
1216
-template <size_t vec_size>
1217
-FLASHINFER_INLINE void cast_from_impl(const vec_t<__nv_fp8_e4m3, vec_size> &src,
1218
- vec_t<half, vec_size> &dst) {
1219
- if constexpr (vec_size == 1) {
1220
- dst.data = float(src.data);
1221
- } else {
1222
-#pragma unroll
1223
- for (size_t i = 0; i < vec_size / 2; ++i) {
1224
- ((half2 *)(&dst.data))[i] = half2(((__nv_fp8x2_e4m3 *)(&src.data))[i]);
1225
- }
1226
- }
1227
-}
1228
-
1229
-template <size_t vec_size>
1230
-FLASHINFER_INLINE void cast_from_impl(const vec_t<float, vec_size> &src,
1231
- vec_t<__nv_fp8_e4m3, vec_size> &dst) {
1232
- if constexpr (vec_size == 1) {
1233
- dst.data = __nv_fp8_e4m3(src.data);
1234
- } else if constexpr (vec_size == 2) {
1235
- *(__nv_fp8x2_e4m3 *)(&dst.data) = __nv_fp8x2_e4m3(*(float2 *)(&src.data));
1236
- } else {
1237
-#pragma unroll
1238
- for (size_t i = 0; i < vec_size / 4; ++i) {
1239
- ((__nv_fp8x4_e4m3 *)(&dst.data))[i] =
1240
- __nv_fp8x4_e4m3(((float4 *)(&src.data))[i]);
1241
- }
1242
- }
1243
-}
1244
-
1245
-template <size_t vec_size>
1246
-FLASHINFER_INLINE void cast_from_impl(const vec_t<half, vec_size> &src,
1247
- vec_t<__nv_fp8_e4m3, vec_size> &dst) {
1248
- if constexpr (vec_size == 1) {
1249
- dst.data = __nv_fp8_e4m3(src.data);
1250
- } else if constexpr (vec_size == 2) {
1251
- *(__nv_fp8x2_e4m3 *)(&dst.data) = __nv_fp8x2_e4m3(*(half2 *)(&src.data));
1252
- } else {
1253
-#pragma unroll
1254
- for (size_t i = 0; i < vec_size / 4; ++i) {
1255
- // NOTE(Zihao): need to double check if we properly handle flo and fhi
1256
- ((__nv_fp8x4_e4m3 *)(&dst.data))[i] = __nv_fp8x4_e4m3(
1257
- ((half2 *)(&src.data))[i * 2], ((half2 *)(&src.data))[i * 2 + 1]);
1258
- }
1259
- }
1260
-}
1261
-
1262
-template <size_t vec_size>
1263
-FLASHINFER_INLINE void cast_from_impl(const vec_t<__nv_fp8_e5m2, vec_size> &src,
1264
- vec_t<float, vec_size> &dst) {
1265
- if constexpr (vec_size == 1) {
1266
- dst.data = float(src.data);
1267
- } else if constexpr (vec_size == 2) {
1268
- *(float2 *)(&dst.data) = float2(*(__nv_fp8x2_e5m2 *)(&src.data));
1269
- } else {
1270
-#pragma unroll
1271
- for (size_t i = 0; i < vec_size / 4; ++i) {
1272
- ((float4 *)(&dst.data))[i] = float4(((__nv_fp8x4_e5m2 *)(&src.data))[i]);
1273
- }
1274
- }
1275
-}
1276
-
1277
-template <size_t vec_size>
1278
-FLASHINFER_INLINE void cast_from_impl(const vec_t<__nv_fp8_e5m2, vec_size> &src,
1279
- vec_t<half, vec_size> &dst) {
1280
- if constexpr (vec_size == 1) {
1281
- dst.data = float(src.data);
1282
- } else {
1283
-#pragma unroll
1284
- for (size_t i = 0; i < vec_size / 2; ++i) {
1285
- ((half2 *)(&dst.data))[i] = half2(((__nv_fp8x2_e5m2 *)(&src.data))[i]);
1286
- }
1287
- }
1288
-}
1289
-
1290
-template <size_t vec_size>
1291
-FLASHINFER_INLINE void cast_from_impl(const vec_t<float, vec_size> &src,
1292
- vec_t<__nv_fp8_e5m2, vec_size> &dst) {
1293
- if constexpr (vec_size == 1) {
1294
- dst.data = __nv_fp8_e5m2(src.data);
1295
- } else if constexpr (vec_size == 2) {
1296
- *(__nv_fp8x2_e5m2 *)(&dst.data) = __nv_fp8x2_e5m2(*(float2 *)(&src.data));
1297
- } else {
1298
-#pragma unroll
1299
- for (size_t i = 0; i < vec_size / 4; ++i) {
1300
- ((__nv_fp8x4_e5m2 *)(&dst.data))[i] =
1301
- __nv_fp8x4_e5m2(((float4 *)(&src.data))[i]);
1302
- }
1303
- }
1304
-}
1305
-
1306
-template <size_t vec_size>
1307
-FLASHINFER_INLINE void cast_from_impl(const vec_t<half, vec_size> &src,
1308
- vec_t<__nv_fp8_e5m2, vec_size> &dst) {
1309
- if constexpr (vec_size == 1) {
1310
- dst.data = __nv_fp8_e4m3(src.data);
1311
- } else if constexpr (vec_size == 2) {
1312
- *(__nv_fp8x2_e5m2 *)(&dst.data) = __nv_fp8x2_e5m2(*(half2 *)(&src.data));
1313
- } else {
1314
-#pragma unroll
1315
- for (size_t i = 0; i < vec_size / 4; ++i) {
1316
- // NOTE(Zihao): need to double check if we properly handle flo and fhi
1317
- ((__nv_fp8x4_e5m2 *)(&dst.data))[i] = __nv_fp8x4_e5m2(
1318
- ((half2 *)(&src.data))[i * 2], ((half2 *)(&src.data))[i * 2 + 1]);
1319
- }
1320
- }
1321
-}
1322
-
1323
-#endif // FLASHINFER_USE_FP8
1324
-
1325
-#endif // VEC_DTYPES_CUH_
csrc/punica/punica_ops.cu
deleted
-569
@@ -1,569 +0,0 @@
1
-#include <torch/all.h>
2
-#include <c10/cuda/CUDAGuard.h>
3
-#include <cstdint>
4
-
5
-#include "type_convert.h"
6
-#include "../cuda_compat.h"
7
-#include "bgmv/bgmv_config.h"
8
-
9
-
10
-//====== utils ======
11
-
12
-inline void check_shape(const torch::Tensor &a, const torch::Tensor &b,
13
- const char *a_name, const char *b_name) {
14
- TORCH_CHECK(a.dim() == b.dim(), a_name, ".dim() != ", b_name, ".dim(). ",
15
- a.dim(), " vs ", b.dim());
16
- for (int i = 0; i < a.dim(); ++i) {
17
- TORCH_CHECK(a.size(i) == b.size(i), a_name, ".size(", i, ") != ", b_name,
18
- ".size(", i, ")");
19
- }
20
-}
21
-
22
-inline constexpr uint64_t pack_u32(uint32_t a, uint32_t b) {
23
- return (uint64_t(a) << 32) | uint64_t(b);
24
-}
25
-
26
-#define CHECK_CUDA(x) TORCH_CHECK(x.is_cuda(), #x " must be a CUDA tensor")
27
-
28
-#define CHECK_CONTIGUOUS(x) \
29
- TORCH_CHECK(x.is_contiguous(), #x " must be contiguous")
30
-
31
-#define CHECK_INPUT(x) \
32
- CHECK_CUDA(x); \
33
- CHECK_CONTIGUOUS(x)
34
-
35
-#define CHECK_DIM(d, x) \
36
- TORCH_CHECK(x.dim() == d, #x " must be a " #d "D tensor")
37
-
38
-#define CHECK_SHAPE(a, b) check_shape(a, b, #a, #b)
39
-
40
-#define CHECK_EQ(a, b) \
41
- TORCH_CHECK(a == b, "CHECK_EQ(" #a ", " #b ") failed. ", a, " vs ", b)
42
-
43
-//====== bgmv ======
44
-
45
-template <typename in_T, typename out_T, typename W_T>
46
-inline bool launch_bgmv_kernel(out_T *Y, const in_T *X, const W_T *W,
47
- const int64_t *lora_indices,
48
- uint32_t in_features, uint32_t out_features,
49
- int64_t y_offset, int64_t full_y_size,
50
- int64_t batch_size, int64_t num_layers,
51
- int64_t layer_idx, float scale) {
52
- // NOTE(woosuk): While Punica supports various combinations of input/output
53
- // data types, we limit the supported data types to reduce the binary size.
54
- constexpr bool is_input_float = std::is_same<in_T, float>::value;
55
- constexpr bool is_output_float = std::is_same<out_T, float>::value;
56
- if (is_input_float) {
57
- if (!std::is_same<out_T, W_T>::value) {
58
- return false;
59
- }
60
- } else if (is_output_float) {
61
- if (!std::is_same<in_T, W_T>::value) {
62
- return false;
63
- }
64
- } else if (!(std::is_same<in_T, W_T>::value &&
65
- std::is_same<out_T, W_T>::value)) {
66
- return false;
67
- }
68
-
69
- switch (pack_u32(in_features, out_features)) {
70
-#define CASE_ONESIDE(_in_T, _out_T, _W_T, feat_in, feat_out) \
71
- case pack_u32(feat_in, feat_out): \
72
- bgmv_kernel<feat_in, feat_out>(Y, X, W, lora_indices, y_offset, \
73
- full_y_size, batch_size, num_layers, \
74
- layer_idx, scale); \
75
- break;
76
-#define CASE(_in_T, _out_T, _W_T, narrow, wide) \
77
- CASE_ONESIDE(in_T, out_T, W_T, narrow, wide) \
78
- CASE_ONESIDE(in_T, out_T, W_T, wide, narrow)
79
-
80
- FOR_BGMV_WIDE_NARROW(CASE, _, _, _)
81
- FOR_INST_BGMV_WIDE_NARROW(CASE_ONESIDE, _, _, _)
82
-#undef CASE
83
-#undef CASE_ONESIDE
84
- default:
85
- return false;
86
- }
87
- return true;
88
-}
89
-
90
-void dispatch_bgmv(torch::Tensor y, torch::Tensor x, torch::Tensor w,
91
- torch::Tensor indicies, int64_t layer_idx, double scale) {
92
- CHECK_INPUT(y);
93
- CHECK_INPUT(x);
94
- CHECK_INPUT(w);
95
- CHECK_INPUT(indicies);
96
-
97
- CHECK_DIM(2, y);
98
- CHECK_DIM(2, x);
99
- CHECK_DIM(4, w);
100
- CHECK_DIM(1, indicies);
101
-
102
- int64_t B = x.size(0);
103
- int64_t h_in = x.size(1);
104
- int64_t h_out = y.size(1);
105
- int64_t num_layers = w.size(1);
106
- CHECK_EQ(w.size(3), h_in);
107
- CHECK_EQ(w.size(2), h_out);
108
- CHECK_EQ(indicies.size(0), x.size(0));
109
- CHECK_EQ(y.size(0), x.size(0));
110
- const at::cuda::OptionalCUDAGuard device_guard(device_of(x));
111
- bool ok = false;
112
- if (h_in <= 128512 && h_out <= 128512) {
113
- // TODO: See if we can get rid of this massive nested switch
114
- switch (x.scalar_type()) {
115
- case at::ScalarType::Half:
116
- switch (y.scalar_type()) {
117
- case at::ScalarType::Half:
118
- switch (w.scalar_type()) {
119
- case at::ScalarType::Half:
120
- ok = launch_bgmv_kernel(static_cast<nv_half *>(y.data_ptr()),
121
- static_cast<nv_half *>(x.data_ptr()),
122
- static_cast<nv_half *>(w.data_ptr()),
123
- indicies.data_ptr<int64_t>(), h_in, h_out, 0,
124
- h_out, B, num_layers, layer_idx, scale);
125
- break;
126
- case at::ScalarType::BFloat16:
127
- ok = launch_bgmv_kernel(static_cast<nv_half *>(y.data_ptr()),
128
- static_cast<nv_half *>(x.data_ptr()),
129
- static_cast<nv_bfloat16 *>(w.data_ptr()),
130
- indicies.data_ptr<int64_t>(), h_in, h_out, 0,
131
- h_out, B, num_layers, layer_idx, scale);
132
- break;
133
- default:
134
- break;
135
- }
136
- break;
137
- case at::ScalarType::BFloat16:
138
- switch (w.scalar_type()) {
139
- case at::ScalarType::Half:
140
- ok = launch_bgmv_kernel(static_cast<nv_bfloat16 *>(y.data_ptr()),
141
- static_cast<nv_half *>(x.data_ptr()),
142
- static_cast<nv_half *>(w.data_ptr()),
143
- indicies.data_ptr<int64_t>(), h_in, h_out, 0,
144
- h_out, B, num_layers, layer_idx, scale);
145
- break;
146
- case at::ScalarType::BFloat16:
147
- ok = launch_bgmv_kernel(static_cast<nv_bfloat16 *>(y.data_ptr()),
148
- static_cast<nv_half *>(x.data_ptr()),
149
- static_cast<nv_bfloat16 *>(w.data_ptr()),
150
- indicies.data_ptr<int64_t>(), h_in, h_out, 0,
151
- h_out, B, num_layers, layer_idx, scale);
152
- break;
153
- default:
154
- break;
155
- }
156
- break;
157
- case at::ScalarType::Float:
158
- switch (w.scalar_type()) {
159
- case at::ScalarType::Half:
160
- ok = launch_bgmv_kernel(static_cast<float *>(y.data_ptr()),
161
- static_cast<nv_half *>(x.data_ptr()),
162
- static_cast<nv_half *>(w.data_ptr()),
163
- indicies.data_ptr<int64_t>(), h_in, h_out, 0,
164
- h_out, B, num_layers, layer_idx, scale);
165
- break;
166
- case at::ScalarType::BFloat16:
167
- ok = launch_bgmv_kernel(static_cast<float *>(y.data_ptr()),
168
- static_cast<nv_half *>(x.data_ptr()),
169
- static_cast<nv_bfloat16 *>(w.data_ptr()),
170
- indicies.data_ptr<int64_t>(), h_in, h_out, 0,
171
- h_out, B, num_layers, layer_idx, scale);
172
- break;
173
- default:
174
- break;
175
- }
176
- break;
177
- default:
178
- break;
179
- }
180
- break;
181
- case at::ScalarType::BFloat16:
182
- switch (y.scalar_type()) {
183
- case at::ScalarType::Half:
184
- switch (w.scalar_type()) {
185
- case at::ScalarType::Half:
186
- ok = launch_bgmv_kernel(static_cast<nv_half *>(y.data_ptr()),
187
- static_cast<nv_bfloat16 *>(x.data_ptr()),
188
- static_cast<nv_half *>(w.data_ptr()),
189
- indicies.data_ptr<int64_t>(), h_in, h_out, 0,
190
- h_out, B, num_layers, layer_idx, scale);
191
- break;
192
- case at::ScalarType::BFloat16:
193
- ok = launch_bgmv_kernel(static_cast<nv_half *>(y.data_ptr()),
194
- static_cast<nv_bfloat16 *>(x.data_ptr()),
195
- static_cast<nv_bfloat16 *>(w.data_ptr()),
196
- indicies.data_ptr<int64_t>(), h_in, h_out, 0,
197
- h_out, B, num_layers, layer_idx, scale);
198
- break;
199
- default:
200
- break;
201
- }
202
- break;
203
- case at::ScalarType::BFloat16:
204
- switch (w.scalar_type()) {
205
- case at::ScalarType::Half:
206
- ok = launch_bgmv_kernel(static_cast<nv_bfloat16 *>(y.data_ptr()),
207
- static_cast<nv_bfloat16 *>(x.data_ptr()),
208
- static_cast<nv_half *>(w.data_ptr()),
209
- indicies.data_ptr<int64_t>(), h_in, h_out, 0,
210
- h_out, B, num_layers, layer_idx, scale);
211
- break;
212
- case at::ScalarType::BFloat16:
213
- ok = launch_bgmv_kernel(static_cast<nv_bfloat16 *>(y.data_ptr()),
214
- static_cast<nv_bfloat16 *>(x.data_ptr()),
215
- static_cast<nv_bfloat16 *>(w.data_ptr()),
216
- indicies.data_ptr<int64_t>(), h_in, h_out, 0,
217
- h_out, B, num_layers, layer_idx, scale);
218
- break;
219
- default:
220
- break;
221
- }
222
- break;
223
- case at::ScalarType::Float:
224
- switch (w.scalar_type()) {
225
- case at::ScalarType::Half:
226
- ok = launch_bgmv_kernel(static_cast<float *>(y.data_ptr()),
227
- static_cast<nv_bfloat16 *>(x.data_ptr()),
228
- static_cast<nv_half *>(w.data_ptr()),
229
- indicies.data_ptr<int64_t>(), h_in, h_out, 0,
230
- h_out, B, num_layers, layer_idx, scale);
231
- break;
232
- case at::ScalarType::BFloat16:
233
- ok = launch_bgmv_kernel(static_cast<float *>(y.data_ptr()),
234
- static_cast<nv_bfloat16 *>(x.data_ptr()),
235
- static_cast<nv_bfloat16 *>(w.data_ptr()),
236
- indicies.data_ptr<int64_t>(), h_in, h_out, 0,
237
- h_out, B, num_layers, layer_idx, scale);
238
- break;
239
- default:
240
- break;
241
- }
242
- break;
243
- default:
244
- break;
245
- }
246
- break;
247
- case at::ScalarType::Float:
248
- switch (y.scalar_type()) {
249
- case at::ScalarType::Half:
250
- switch (w.scalar_type()) {
251
- case at::ScalarType::Half:
252
- ok = launch_bgmv_kernel(static_cast<nv_half *>(y.data_ptr()),
253
- static_cast<float *>(x.data_ptr()),
254
- static_cast<nv_half *>(w.data_ptr()),
255
- indicies.data_ptr<int64_t>(), h_in, h_out, 0,
256
- h_out, B, num_layers, layer_idx, scale);
257
- break;
258
- case at::ScalarType::BFloat16:
259
- ok = launch_bgmv_kernel(static_cast<nv_half *>(y.data_ptr()),
260
- static_cast<float *>(x.data_ptr()),
261
- static_cast<nv_bfloat16 *>(w.data_ptr()),
262
- indicies.data_ptr<int64_t>(), h_in, h_out, 0,
263
- h_out, B, num_layers, layer_idx, scale);
264
- break;
265
- default:
266
- break;
267
- }
268
- break;
269
- case at::ScalarType::BFloat16:
270
- switch (w.scalar_type()) {
271
- case at::ScalarType::Half:
272
- ok = launch_bgmv_kernel(static_cast<nv_bfloat16 *>(y.data_ptr()),
273
- static_cast<float *>(x.data_ptr()),
274
- static_cast<nv_half *>(w.data_ptr()),
275
- indicies.data_ptr<int64_t>(), h_in, h_out, 0,
276
- h_out, B, num_layers, layer_idx, scale);
277
- break;
278
- case at::ScalarType::BFloat16:
279
- ok = launch_bgmv_kernel(static_cast<nv_bfloat16 *>(y.data_ptr()),
280
- static_cast<float *>(x.data_ptr()),
281
- static_cast<nv_bfloat16 *>(w.data_ptr()),
282
- indicies.data_ptr<int64_t>(), h_in, h_out, 0,
283
- h_out, B, num_layers, layer_idx, scale);
284
- break;
285
- default:
286
- break;
287
- }
288
- break;
289
- case at::ScalarType::Float:
290
- switch (w.scalar_type()) {
291
- case at::ScalarType::Half:
292
- ok = launch_bgmv_kernel(static_cast<float *>(y.data_ptr()),
293
- static_cast<float *>(x.data_ptr()),
294
- static_cast<nv_half *>(w.data_ptr()),
295
- indicies.data_ptr<int64_t>(), h_in, h_out, 0,
296
- h_out, B, num_layers, layer_idx, scale);
297
- break;
298
- case at::ScalarType::BFloat16:
299
- ok = launch_bgmv_kernel(static_cast<float *>(y.data_ptr()),
300
- static_cast<float *>(x.data_ptr()),
301
- static_cast<nv_bfloat16 *>(w.data_ptr()),
302
- indicies.data_ptr<int64_t>(), h_in, h_out, 0,
303
- h_out, B, num_layers, layer_idx, scale);
304
- break;
305
- default:
306
- break;
307
- }
308
- break;
309
- default:
310
- break;
311
- }
312
- break;
313
- default:
314
- break;
315
- }
316
- }
317
- TORCH_CHECK(ok, "No suitable kernel.", " h_in=", h_in, " h_out=", h_out,
318
- " dtype=", x.scalar_type(), " out_dtype=", y.scalar_type());
319
-}
320
-
321
-void dispatch_bgmv_low_level(torch::Tensor y, torch::Tensor x, torch::Tensor w,
322
- torch::Tensor indicies, int64_t layer_idx,
323
- double scale, int64_t h_in, int64_t h_out,
324
- int64_t y_offset) {
325
- CHECK_INPUT(y);
326
- CHECK_INPUT(x);
327
- CHECK_INPUT(w);
328
- CHECK_INPUT(indicies);
329
-
330
- CHECK_DIM(2, y);
331
- CHECK_DIM(2, x);
332
- CHECK_DIM(4, w);
333
- CHECK_DIM(1, indicies);
334
-
335
- int64_t B = x.size(0);
336
- int64_t num_layers = w.size(1);
337
- int64_t full_y_size = y.size(1);
338
- CHECK_EQ(w.size(3), h_in);
339
- CHECK_EQ(w.size(2), h_out);
340
- CHECK_EQ(indicies.size(0), x.size(0));
341
- CHECK_EQ(y.size(0), x.size(0));
342
- const at::cuda::OptionalCUDAGuard device_guard(device_of(x));
343
- bool ok = false;
344
- if (h_in <= 128512 && h_out <= 128512) {
345
- // TODO: See if we can get rid of this massive nested switch
346
- switch (x.scalar_type()) {
347
- case at::ScalarType::Half:
348
- switch (y.scalar_type()) {
349
- case at::ScalarType::Half:
350
- switch (w.scalar_type()) {
351
- case at::ScalarType::Half:
352
- ok = launch_bgmv_kernel(static_cast<nv_half *>(y.data_ptr()),
353
- static_cast<nv_half *>(x.data_ptr()),
354
- static_cast<nv_half *>(w.data_ptr()),
355
- indicies.data_ptr<int64_t>(), h_in, h_out,
356
- y_offset, full_y_size, B, num_layers,
357
- layer_idx, scale);
358
- break;
359
- case at::ScalarType::BFloat16:
360
- ok = launch_bgmv_kernel(static_cast<nv_half *>(y.data_ptr()),
361
- static_cast<nv_half *>(x.data_ptr()),
362
- static_cast<nv_bfloat16 *>(w.data_ptr()),
363
- indicies.data_ptr<int64_t>(), h_in, h_out,
364
- y_offset, full_y_size, B, num_layers,
365
- layer_idx, scale);
366
- break;
367
- default:
368
- break;
369
- }
370
- break;
371
- case at::ScalarType::BFloat16:
372
- switch (w.scalar_type()) {
373
- case at::ScalarType::Half:
374
- ok = launch_bgmv_kernel(static_cast<nv_bfloat16 *>(y.data_ptr()),
375
- static_cast<nv_half *>(x.data_ptr()),
376
- static_cast<nv_half *>(w.data_ptr()),
377
- indicies.data_ptr<int64_t>(), h_in, h_out,
378
- y_offset, full_y_size, B, num_layers,
379
- layer_idx, scale);
380
- break;
381
- case at::ScalarType::BFloat16:
382
- ok = launch_bgmv_kernel(static_cast<nv_bfloat16 *>(y.data_ptr()),
383
- static_cast<nv_half *>(x.data_ptr()),
384
- static_cast<nv_bfloat16 *>(w.data_ptr()),
385
- indicies.data_ptr<int64_t>(), h_in, h_out,
386
- y_offset, full_y_size, B, num_layers,
387
- layer_idx, scale);
388
- break;
389
- default:
390
- break;
391
- }
392
- break;
393
- case at::ScalarType::Float:
394
- switch (w.scalar_type()) {
395
- case at::ScalarType::Half:
396
- ok = launch_bgmv_kernel(static_cast<float *>(y.data_ptr()),
397
- static_cast<nv_half *>(x.data_ptr()),
398
- static_cast<nv_half *>(w.data_ptr()),
399
- indicies.data_ptr<int64_t>(), h_in, h_out,
400
- y_offset, full_y_size, B, num_layers,
401
- layer_idx, scale);
402
- break;
403
- case at::ScalarType::BFloat16:
404
- ok = launch_bgmv_kernel(static_cast<float *>(y.data_ptr()),
405
- static_cast<nv_half *>(x.data_ptr()),
406
- static_cast<nv_bfloat16 *>(w.data_ptr()),
407
- indicies.data_ptr<int64_t>(), h_in, h_out,
408
- y_offset, full_y_size, B, num_layers,
409
- layer_idx, scale);
410
- break;
411
- default:
412
- break;
413
- }
414
- break;
415
- default:
416
- break;
417
- }
418
- break;
419
- case at::ScalarType::BFloat16:
420
- switch (y.scalar_type()) {
421
- case at::ScalarType::Half:
422
- switch (w.scalar_type()) {
423
- case at::ScalarType::Half:
424
- ok = launch_bgmv_kernel(static_cast<nv_half *>(y.data_ptr()),
425
- static_cast<nv_bfloat16 *>(x.data_ptr()),
426
- static_cast<nv_half *>(w.data_ptr()),
427
- indicies.data_ptr<int64_t>(), h_in, h_out,
428
- y_offset, full_y_size, B, num_layers,
429
- layer_idx, scale);
430
- break;
431
- case at::ScalarType::BFloat16:
432
- ok = launch_bgmv_kernel(static_cast<nv_half *>(y.data_ptr()),
433
- static_cast<nv_bfloat16 *>(x.data_ptr()),
434
- static_cast<nv_bfloat16 *>(w.data_ptr()),
435
- indicies.data_ptr<int64_t>(), h_in, h_out,
436
- y_offset, full_y_size, B, num_layers,
437
- layer_idx, scale);
438
- break;
439
- default:
440
- break;
441
- }
442
- break;
443
- case at::ScalarType::BFloat16:
444
- switch (w.scalar_type()) {
445
- case at::ScalarType::Half:
446
- ok = launch_bgmv_kernel(static_cast<nv_bfloat16 *>(y.data_ptr()),
447
- static_cast<nv_bfloat16 *>(x.data_ptr()),
448
- static_cast<nv_half *>(w.data_ptr()),
449
- indicies.data_ptr<int64_t>(), h_in, h_out,
450
- y_offset, full_y_size, B, num_layers,
451
- layer_idx, scale);
452
- break;
453
- case at::ScalarType::BFloat16:
454
- ok = launch_bgmv_kernel(static_cast<nv_bfloat16 *>(y.data_ptr()),
455
- static_cast<nv_bfloat16 *>(x.data_ptr()),
456
- static_cast<nv_bfloat16 *>(w.data_ptr()),
457
- indicies.data_ptr<int64_t>(), h_in, h_out,
458
- y_offset, full_y_size, B, num_layers,
459
- layer_idx, scale);
460
- break;
461
- default:
462
- break;
463
- }
464
- break;
465
- case at::ScalarType::Float:
466
- switch (w.scalar_type()) {
467
- case at::ScalarType::Half:
468
- ok = launch_bgmv_kernel(static_cast<float *>(y.data_ptr()),
469
- static_cast<nv_bfloat16 *>(x.data_ptr()),
470
- static_cast<nv_half *>(w.data_ptr()),
471
- indicies.data_ptr<int64_t>(), h_in, h_out,
472
- y_offset, full_y_size, B, num_layers,
473
- layer_idx, scale);
474
- break;
475
- case at::ScalarType::BFloat16:
476
- ok = launch_bgmv_kernel(static_cast<float *>(y.data_ptr()),
477
- static_cast<nv_bfloat16 *>(x.data_ptr()),
478
- static_cast<nv_bfloat16 *>(w.data_ptr()),
479
- indicies.data_ptr<int64_t>(), h_in, h_out,
480
- y_offset, full_y_size, B, num_layers,
481
- layer_idx, scale);
482
- break;
483
- default:
484
- break;
485
- }
486
- break;
487
- default:
488
- break;
489
- }
490
- break;
491
- case at::ScalarType::Float:
492
- switch (y.scalar_type()) {
493
- case at::ScalarType::Half:
494
- switch (w.scalar_type()) {
495
- case at::ScalarType::Half:
496
- ok = launch_bgmv_kernel(static_cast<nv_half *>(y.data_ptr()),
497
- static_cast<float *>(x.data_ptr()),
498
- static_cast<nv_half *>(w.data_ptr()),
499
- indicies.data_ptr<int64_t>(), h_in, h_out,
500
- y_offset, full_y_size, B, num_layers,
501
- layer_idx, scale);
502
- break;
503
- case at::ScalarType::BFloat16:
504
- ok = launch_bgmv_kernel(static_cast<nv_half *>(y.data_ptr()),
505
- static_cast<float *>(x.data_ptr()),
506
- static_cast<nv_bfloat16 *>(w.data_ptr()),
507
- indicies.data_ptr<int64_t>(), h_in, h_out,
508
- y_offset, full_y_size, B, num_layers,
509
- layer_idx, scale);
510
- break;
511
- default:
512
- break;
513
- }
514
- break;
515
- case at::ScalarType::BFloat16:
516
- switch (w.scalar_type()) {
517
- case at::ScalarType::Half:
518
- ok = launch_bgmv_kernel(static_cast<nv_bfloat16 *>(y.data_ptr()),
519
- static_cast<float *>(x.data_ptr()),
520
- static_cast<nv_half *>(w.data_ptr()),
521
- indicies.data_ptr<int64_t>(), h_in, h_out,
522
- y_offset, full_y_size, B, num_layers,
523
- layer_idx, scale);
524
- break;
525
- case at::ScalarType::BFloat16:
526
- ok = launch_bgmv_kernel(static_cast<nv_bfloat16 *>(y.data_ptr()),
527
- static_cast<float *>(x.data_ptr()),
528
- static_cast<nv_bfloat16 *>(w.data_ptr()),
529
- indicies.data_ptr<int64_t>(), h_in, h_out,
530
- y_offset, full_y_size, B, num_layers,
531
- layer_idx, scale);
532
- break;
533
- default:
534
- break;
535
- }
536
- break;
537
- case at::ScalarType::Float:
538
- switch (w.scalar_type()) {
539
- case at::ScalarType::Half:
540
- ok = launch_bgmv_kernel(static_cast<float *>(y.data_ptr()),
541
- static_cast<float *>(x.data_ptr()),
542
- static_cast<nv_half *>(w.data_ptr()),
543
- indicies.data_ptr<int64_t>(), h_in, h_out,
544
- y_offset, full_y_size, B, num_layers,
545
- layer_idx, scale);
546
- break;
547
- case at::ScalarType::BFloat16:
548
- ok = launch_bgmv_kernel(static_cast<float *>(y.data_ptr()),
549
- static_cast<float *>(x.data_ptr()),
550
- static_cast<nv_bfloat16 *>(w.data_ptr()),
551
- indicies.data_ptr<int64_t>(), h_in, h_out,
552
- y_offset, full_y_size, B, num_layers,
553
- layer_idx, scale);
554
- break;
555
- default:
556
- break;
557
- }
558
- break;
559
- default:
560
- break;
561
- }
562
- break;
563
- default:
564
- break;
565
- }
566
- }
567
- TORCH_CHECK(ok, "No suitable kernel.", " h_in=", h_in, " h_out=", h_out,
568
- " dtype=", x.scalar_type(), " out_dtype=", y.scalar_type());
569
-}
csrc/punica/punica_ops.h
deleted
-11
@@ -1,11 +0,0 @@
1
-#pragma once
2
-
3
-#include <torch/all.h>
4
-
5
-void dispatch_bgmv(torch::Tensor y, torch::Tensor x, torch::Tensor w,
6
- torch::Tensor indicies, int64_t layer_idx, double scale);
7
-
8
-void dispatch_bgmv_low_level(torch::Tensor y, torch::Tensor x, torch::Tensor w,
9
- torch::Tensor indicies, int64_t layer_idx,
10
- double scale, int64_t h_in, int64_t h_out,
11
- int64_t y_offset);
csrc/punica/torch_bindings.cpp
deleted
-18
@@ -1,18 +0,0 @@
1
-#include "registration.h"
2
-#include "punica_ops.h"
3
-
4
-TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) {
5
- m.def(
6
- "dispatch_bgmv(Tensor! y, Tensor x, Tensor w, Tensor indicies, int "
7
- "layer_idx, float scale) -> ()");
8
- m.impl("dispatch_bgmv", torch::kCUDA, &dispatch_bgmv);
9
-
10
- m.def(
11
- "dispatch_bgmv_low_level(Tensor! y, Tensor x, Tensor w,"
12
- "Tensor indicies, int layer_idx,"
13
- "float scale, int h_in, int h_out,"
14
- "int y_offset) -> ()");
15
- m.impl("dispatch_bgmv_low_level", torch::kCUDA, &dispatch_bgmv_low_level);
16
-}
17
-
18
-REGISTER_EXTENSION(TORCH_EXTENSION_NAME)
csrc/punica/type_convert.h
deleted
-82
@@ -1,82 +0,0 @@
1
-#ifndef CSRC__PUNICA__TYPE_CONVERT_H__
2
-#define CSRC__PUNICA__TYPE_CONVERT_H__
3
-
4
-#ifndef USE_ROCM
5
-
6
-#include <cuda_bf16.h>
7
-#include <cuda_fp16.h>
8
-
9
-#else
10
-
11
-#include <hip/hip_bf16.h>
12
-#include <hip/hip_fp16.h>
13
-
14
-#define __TYPE_CONVERT__HOST_DEVICE__ __host__ __device__
15
-
16
-typedef __half nv_half;
17
-typedef __hip_bfloat16 nv_bfloat16;
18
-typedef __hip_bfloat162 nv_bfloat162;
19
-
20
-__TYPE_CONVERT__HOST_DEVICE__
21
-inline __hip_bfloat162 make_bfloat162(__hip_bfloat16 val) {
22
- return __hip_bfloat162{val, val};
23
-}
24
-
25
-__TYPE_CONVERT__HOST_DEVICE__
26
-inline __hip_bfloat162 make_bfloat162(__hip_bfloat16 vall, __hip_bfloat16 valr) {
27
- return __hip_bfloat162{vall, valr};
28
-}
29
-
30
-template <typename T_src, typename T_dst>
31
-__TYPE_CONVERT__HOST_DEVICE__
32
-inline T_dst convert_type(T_src val) {
33
- return static_cast<T_dst>(val);
34
-}
35
-
36
-template <>
37
-__TYPE_CONVERT__HOST_DEVICE__
38
-inline float convert_type<__half, float>(__half val) {
39
- return __half2float(val);
40
-}
41
-
42
-template <>
43
-__TYPE_CONVERT__HOST_DEVICE__
44
-inline __half convert_type<float, __half>(float val) {
45
- return __float2half(val);
46
-}
47
-
48
-template <>
49
-__TYPE_CONVERT__HOST_DEVICE__
50
-inline float convert_type<__hip_bfloat16, float>(__hip_bfloat16 val) {
51
- return __bfloat162float(val);
52
-}
53
-
54
-template <>
55
-__TYPE_CONVERT__HOST_DEVICE__
56
-inline __hip_bfloat16 convert_type<float, __hip_bfloat16>(float val) {
57
- return __float2bfloat16(val);
58
-}
59
-
60
-template <typename T>
61
-__TYPE_CONVERT__HOST_DEVICE__
62
-inline T vllm_add(T a, T b) {
63
- return a + b;
64
-}
65
-
66
-template <>
67
-__TYPE_CONVERT__HOST_DEVICE__
68
-inline __half vllm_add<__half>(__half a, __half b) {
69
- return __hadd(a, b);
70
-}
71
-
72
-template <>
73
-__TYPE_CONVERT__HOST_DEVICE__
74
-inline __hip_bfloat16 vllm_add<__hip_bfloat16>(__hip_bfloat16 a, __hip_bfloat16 b) {
75
- return __hadd(a, b);
76
-}
77
-
78
-#undef __TYPE_CONVERT__HOST_DEVICE__
79
-
80
-#endif // USE_ROCM
81
-
82
-#endif // CSRC__PUNICA__TYPE_CONVERT_H__
docs/source/getting_started/installation.rst
-1
@@ -66,7 +66,6 @@ You can also build and install vLLM from source:
66
67
$ git clone https://github.com/vllm-project/vllm.git
68
$ cd vllm
69
- $ # export VLLM_INSTALL_PUNICA_KERNELS=1 # optionally build for multi-LoRA capability
69
$ pip install -e . # This may take 5-10 minutes.
70
71
.. tip::
setup.py
-10
@@ -181,9 +181,6 @@ class cmake_build_ext(build_ext):
181
# match.
182
cmake_args += ['-DVLLM_PYTHON_EXECUTABLE={}'.format(sys.executable)]
183
184
- if _install_punica():
185
- cmake_args += ['-DVLLM_INSTALL_PUNICA_KERNELS=ON']
186
-
184
#
185
# Setup parallelism and build tool
186
#
@@ -274,10 +271,6 @@ def _build_custom_ops() -> bool:
271
return _is_cuda() or _is_hip() or _is_cpu()
272
273
277
-def _install_punica() -> bool:
278
- return envs.VLLM_INSTALL_PUNICA_KERNELS
279
-
280
-
274
def get_hipcc_rocm_version():
275
# Run the hipcc --version command
276
result = subprocess.run(['hipcc', '--version'],
@@ -446,9 +439,6 @@ if _is_cuda() or _is_hip():
439
if _build_custom_ops():
440
ext_modules.append(CMakeExtension(name="vllm._C"))
441
449
- if _install_punica():
450
- ext_modules.append(CMakeExtension(name="vllm._punica_C"))
451
-
442
package_data = {
443
"vllm": ["py.typed", "model_executor/layers/fused_moe/configs/*.json"]
444
}
tests/kernels/test_sampler.py
+30
-18
@@ -1,14 +1,17 @@
1
import gc
2
+from unittest.mock import patch
3
4
import pytest
5
import torch
6
import triton
7
import triton.language as tl
8
8
-from vllm.model_executor.layers.ops.sample import (_uniform_to_exponential,
9
+from vllm.model_executor.layers.ops.sample import (_sample_triton,
10
+ _uniform_to_exponential,
11
sample)
12
from vllm.model_executor.sampling_metadata import SamplingTensors
13
from vllm.model_executor.utils import set_random_seed
14
+from vllm.triton_utils.libentry import LibEntry
15
from vllm.triton_utils.sample import (MAX_TRITON_N_COLS,
16
get_num_triton_sampler_splits)
17
@@ -76,15 +79,20 @@ def test_sample_decoding_only(random_sampling, max_best_of,
79
seeds = torch.randint(1,
80
torch.iinfo(torch.long).max, (n_splits, bs),
81
device="cuda").mul_(random_sampling_mask)
79
- sampled_tokens, sampled_logprobs, sampled_modified_probs = sample(
80
- probs=probs,
81
- logprobs=logprobs,
82
- sample_indices=sample_indices,
83
- seeds=seeds,
84
- max_best_of=max_best_of,
85
- modify_greedy_probs=modify_greedy_probs,
86
- save_logprobs=save_logprobs,
87
- _save_modified_probs=True)
82
+ #The current _sample_triton does not utilize the
83
+ # libentry decoration. The purpose of adding this patch is to test
84
+ # the correctness of libentry.
85
+ with patch("vllm.model_executor.layers.ops.sample._sample_triton",
86
+ LibEntry(_sample_triton)):
87
+ sampled_tokens, sampled_logprobs, sampled_modified_probs = sample(
88
+ probs=probs,
89
+ logprobs=logprobs,
90
+ sample_indices=sample_indices,
91
+ seeds=seeds,
92
+ max_best_of=max_best_of,
93
+ modify_greedy_probs=modify_greedy_probs,
94
+ save_logprobs=save_logprobs,
95
+ _save_modified_probs=True)
96
assert sampled_tokens.shape == (bs, max_best_of)
97
for i in range(bs):
98
assert torch.all(sampled_tokens[i] == i * (vocab_size // bs))
@@ -130,6 +138,7 @@ def test_sample_decoding_only(random_sampling, max_best_of,
138
[SINGLE_SPLIT_VOCAB_SIZE, MULTI_SPLIT_VOCAB_SIZE])
139
def test_sample_prompt_logprobs(random_sampling, max_best_of,
140
modify_greedy_probs, seed, vocab_size):
141
+
142
set_random_seed(seed)
143
prompt_sizes = [16, 32, 64, 128] * 2
144
samples = 8
@@ -157,14 +166,17 @@ def test_sample_prompt_logprobs(random_sampling, max_best_of,
166
seeds = torch.randint(1,
167
torch.iinfo(torch.long).max, (n_splits, samples),
168
device="cuda").mul_(random_sampling_mask)
160
- sampled_tokens, sampled_logprobs, _ = sample(
161
- probs=probs,
162
- logprobs=logprobs,
163
- sample_indices=sample_indices,
164
- seeds=seeds,
165
- max_best_of=max_best_of,
166
- modify_greedy_probs=modify_greedy_probs,
167
- save_logprobs=True)
169
+ #ditto
170
+ with patch("vllm.model_executor.layers.ops.sample._sample_triton",
171
+ LibEntry(_sample_triton)):
172
+ sampled_tokens, sampled_logprobs, _ = sample(
173
+ probs=probs,
174
+ logprobs=logprobs,
175
+ sample_indices=sample_indices,
176
+ seeds=seeds,
177
+ max_best_of=max_best_of,
178
+ modify_greedy_probs=modify_greedy_probs,
179
+ save_logprobs=True)
180
assert sampled_tokens.shape == (samples, max_best_of)
181
assert sampled_logprobs.shape == (samples, max_best_of)
182
for i, t in enumerate(sample_indices):
tests/lora/test_gemma.py
+1
-1
@@ -37,7 +37,7 @@ def test_gemma_lora(gemma_lora_files):
37
expected_lora_output = [
38
"more important than knowledge.\nAuthor: Albert Einstein\n",
39
"everyone else is already taken.\nAuthor: Oscar Wilde\n",
40
- "so little time\nAuthor: Frank Zappa\n",
40
+ "so little time.\nAuthor: Frank Zappa\n",
41
]
42
43
output1 = do_sample(llm, gemma_lora_files, lora_id=1)
tests/lora/test_layers.py
+81
-59
@@ -26,7 +26,8 @@ from vllm.lora.layers import (BaseLayerWithLoRA, ColumnParallelLinearWithLoRA,
26
VocabParallelEmbeddingWithLoRA)
27
# yapf: enable
28
from vllm.lora.models import (LongContextLoRAContext, LoRALayerWeights,
29
- PackedLoRALayerWeights, convert_mapping)
29
+ PackedLoRALayerWeights)
30
+from vllm.lora.punica import PunicaWrapper
31
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
32
MergedColumnParallelLinear,
33
QKVParallelLinear,
@@ -47,6 +48,9 @@ TOLERANCES = {
48
CUDA_DEVICES = [
49
f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
50
]
51
+# We will launch different triton kernels between the prefill and decode
52
+# stages, so we need to verify this. prefill stage(True) or decode stage(False)
53
+STAGES = [True, False]
54
55
56
def get_random_id_to_index(num_loras: int,
@@ -182,10 +186,12 @@ def create_random_inputs(
186
@pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
187
@pytest.mark.parametrize("device", CUDA_DEVICES)
188
@pytest.mark.parametrize("vocab_size", [512, 32000, 64000, 128000])
185
-def test_embeddings(dist_init, num_loras, device, vocab_size) -> None:
189
+@pytest.mark.parametrize("stage", STAGES)
190
+def test_embeddings(dist_init, num_loras, device, vocab_size, stage) -> None:
191
192
torch.set_default_device(device)
193
max_loras = 8
194
+ punica_wrapper = PunicaWrapper(8192, 256, device)
195
lora_config = LoRAConfig(max_loras=max_loras,
196
max_lora_rank=8,
197
lora_dtype=torch.float16)
@@ -204,7 +210,7 @@ def test_embeddings(dist_init, num_loras, device, vocab_size) -> None:
210
211
id_to_index = get_random_id_to_index(num_loras, max_loras)
212
embedding, lora_embedding = create_random_embedding_layer()
207
-
213
+ lora_embedding.set_mapping(punica_wrapper)
214
lora_dict, _ = populate_loras(
215
id_to_index,
216
layer=lora_embedding,
@@ -217,12 +223,12 @@ def test_embeddings(dist_init, num_loras, device, vocab_size) -> None:
223
input_size=(200, ),
224
input_range=(1, vocab_size),
225
)
220
- lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
221
-
222
- mapping_info = convert_mapping(lora_mapping, id_to_index, max_loras,
226
+ lora_mapping = LoRAMapping(index_mapping,
227
+ prompt_mapping,
228
+ is_prefill=stage)
229
+ punica_wrapper.update_metadata(lora_mapping, id_to_index, max_loras,
230
vocab_size,
231
lora_config.lora_extra_vocab_size)
225
- lora_embedding.set_mapping(*mapping_info)
232
233
lora_result = lora_embedding(torch.cat(inputs))
234
@@ -255,12 +261,12 @@ def test_embeddings(dist_init, num_loras, device, vocab_size) -> None:
261
input_size=(200, ),
262
input_range=(1, vocab_size),
263
)
258
- lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
259
-
260
- mapping_info = convert_mapping(lora_mapping, id_to_index, max_loras,
264
+ lora_mapping = LoRAMapping(index_mapping,
265
+ prompt_mapping,
266
+ is_prefill=stage)
267
+ punica_wrapper.update_metadata(lora_mapping, id_to_index, max_loras,
268
vocab_size,
269
lora_config.lora_extra_vocab_size)
263
- lora_embedding.set_mapping(*mapping_info, )
270
271
lora_result = lora_embedding(torch.cat(inputs))
272
expected_result = embedding(torch.cat(inputs))
@@ -278,11 +284,13 @@ def test_embeddings(dist_init, num_loras, device, vocab_size) -> None:
284
@pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
285
@pytest.mark.parametrize("device", CUDA_DEVICES)
286
@pytest.mark.parametrize("vocab_size", [512, 32000, 64000, 128000])
287
+@pytest.mark.parametrize("stage", STAGES)
288
def test_embeddings_with_new_embeddings(dist_init, num_loras, device,
282
- vocab_size) -> None:
289
+ vocab_size, stage) -> None:
290
291
torch.set_default_device(device)
292
max_loras = 8
293
+ punica_wrapper = PunicaWrapper(8192, 256, device)
294
lora_config = LoRAConfig(max_loras=max_loras,
295
max_lora_rank=8,
296
lora_dtype=torch.float16)
@@ -318,6 +326,7 @@ def test_embeddings_with_new_embeddings(dist_init, num_loras, device,
326
generate_embeddings_tensor=256,
327
)
328
329
+ lora_embedding.set_mapping(punica_wrapper)
330
# All embeddings tensors have the same shape.
331
embeddings_tensors = [
332
lora_dict[id].embeddings_tensor for id in sorted(lora_dict.keys())
@@ -334,8 +343,12 @@ def test_embeddings_with_new_embeddings(dist_init, num_loras, device,
343
input_size=(200, ),
344
input_range=(1, vocab_size),
345
)
337
- lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
338
-
346
+ lora_mapping = LoRAMapping(index_mapping,
347
+ prompt_mapping,
348
+ is_prefill=stage)
349
+ punica_wrapper.update_metadata(lora_mapping, id_to_index, max_loras,
350
+ vocab_size,
351
+ lora_config.lora_extra_vocab_size)
352
original_inputs = deepcopy(inputs)
353
354
# Force some of the inputs to be in the extended embeddings range
@@ -349,11 +362,6 @@ def test_embeddings_with_new_embeddings(dist_init, num_loras, device,
362
(embedding_id + 1) * embeddings_tensor_len - 1)
363
original_input_[-2] = vocab_size + embeddings_tensor_len - 1
364
352
- mapping_info = convert_mapping(lora_mapping, id_to_index, max_loras,
353
- vocab_size,
354
- lora_config.lora_extra_vocab_size)
355
- lora_embedding.set_mapping(*mapping_info, )
356
-
365
expanded_embedding.weight[vocab_size:vocab_size +
366
(embeddings_tensor_len *
367
max_loras)] = torch.cat(embeddings_tensors)
@@ -390,15 +398,13 @@ def test_embeddings_with_new_embeddings(dist_init, num_loras, device,
398
input_size=(200, ),
399
input_range=(1, vocab_size),
400
)
393
- lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
394
-
401
original_inputs = deepcopy(inputs)
396
-
397
- mapping_info = convert_mapping(lora_mapping, id_to_index, max_loras,
402
+ lora_mapping = LoRAMapping(index_mapping,
403
+ prompt_mapping,
404
+ is_prefill=stage)
405
+ punica_wrapper.update_metadata(lora_mapping, id_to_index, max_loras,
406
vocab_size,
407
lora_config.lora_extra_vocab_size)
400
- lora_embedding.set_mapping(*mapping_info, )
401
-
408
lora_result = lora_embedding(torch.cat(original_inputs))
409
expected_result = expanded_embedding(torch.cat(inputs))
410
@@ -413,11 +419,13 @@ def test_embeddings_with_new_embeddings(dist_init, num_loras, device,
419
@pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
420
@pytest.mark.parametrize("device", CUDA_DEVICES)
421
@pytest.mark.parametrize("vocab_size", [512, 32000, 64000, 128000])
416
-def test_lm_head_logits_processor(dist_init, num_loras, device,
417
- vocab_size) -> None:
422
+@pytest.mark.parametrize("stage", STAGES)
423
+def test_lm_head_logits_processor(dist_init, num_loras, device, vocab_size,
424
+ stage) -> None:
425
426
torch.set_default_device(device)
427
max_loras = 8
428
+ punica_wrapper = PunicaWrapper(8192, 256, device)
429
lora_config = LoRAConfig(max_loras=max_loras,
430
max_lora_rank=8,
431
lora_dtype=torch.float16)
@@ -443,7 +451,7 @@ def test_lm_head_logits_processor(dist_init, num_loras, device,
451
452
id_to_index = get_random_id_to_index(num_loras, max_loras)
453
linear, logits_processor, lora_logits_processor = _pretest()
446
-
454
+ lora_logits_processor.set_mapping(punica_wrapper)
455
# NOTE: all the generated loras share the same embeddings tensor.
456
lora_dict, _ = populate_loras(
457
id_to_index,
@@ -461,17 +469,17 @@ def test_lm_head_logits_processor(dist_init, num_loras, device,
469
input_range=(0, 1),
470
input_type=torch.float16,
471
)
464
- lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
465
-
466
- input_ = torch.rand(20, 1024)
467
- mapping_info = convert_mapping(
472
+ lora_mapping = LoRAMapping(index_mapping,
473
+ prompt_mapping,
474
+ is_prefill=stage)
475
+ punica_wrapper.update_metadata(
476
lora_mapping,
477
id_to_index,
478
max_loras,
479
vocab_size,
480
lora_config.lora_extra_vocab_size,
481
)
474
- lora_logits_processor.set_mapping(*mapping_info, )
482
+ input_ = torch.rand(20, 1024)
483
484
lora_result = lora_logits_processor._get_logits(
485
hidden_states=torch.cat(inputs),
@@ -510,12 +518,16 @@ def test_lm_head_logits_processor(dist_init, num_loras, device,
518
input_range=(0, 1),
519
input_type=torch.float16,
520
)
513
- lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
514
-
515
- mapping_info = convert_mapping(lora_mapping, id_to_index, max_loras,
516
- vocab_size,
517
- lora_config.lora_extra_vocab_size)
518
- lora_logits_processor.set_mapping(*mapping_info, )
521
+ lora_mapping = LoRAMapping(index_mapping,
522
+ prompt_mapping,
523
+ is_prefill=stage)
524
+ punica_wrapper.update_metadata(
525
+ lora_mapping,
526
+ id_to_index,
527
+ max_loras,
528
+ vocab_size,
529
+ lora_config.lora_extra_vocab_size,
530
+ )
531
532
lora_result = lora_logits_processor._get_logits(
533
hidden_states=torch.cat(inputs),
@@ -538,10 +550,12 @@ def test_lm_head_logits_processor(dist_init, num_loras, device,
550
@pytest.mark.parametrize("orientation", ["row", "column"])
551
@pytest.mark.parametrize("fully_shard", [True, False])
552
@pytest.mark.parametrize("device", CUDA_DEVICES)
553
+@pytest.mark.parametrize("stage", STAGES)
554
def test_linear_parallel(dist_init, num_loras, orientation, fully_shard,
542
- device) -> None:
555
+ device, stage) -> None:
556
557
torch.set_default_device(device)
558
+ punica_wrapper = PunicaWrapper(8192, 256, device)
559
max_loras = 8
560
lora_config = LoRAConfig(max_loras=max_loras,
561
max_lora_rank=8,
@@ -575,7 +589,7 @@ def test_linear_parallel(dist_init, num_loras, orientation, fully_shard,
589
590
id_to_index = get_random_id_to_index(num_loras, max_loras)
591
linear, lora_linear = create_random_linear_parallel_layer()
578
-
592
+ lora_linear.set_mapping(punica_wrapper)
593
lora_dict, _ = populate_loras(
594
id_to_index,
595
layer=lora_linear,
@@ -589,16 +603,16 @@ def test_linear_parallel(dist_init, num_loras, orientation, fully_shard,
603
input_range=(0, 1),
604
input_type=torch.float16,
605
)
592
- lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
593
-
594
- mapping_info = convert_mapping(
606
+ lora_mapping = LoRAMapping(index_mapping,
607
+ prompt_mapping,
608
+ is_prefill=stage)
609
+ punica_wrapper.update_metadata(
610
lora_mapping,
611
id_to_index,
612
max_loras,
613
512,
614
lora_config.lora_extra_vocab_size,
615
)
601
- lora_linear.set_mapping(*mapping_info, )
616
617
lora_result = lora_linear(torch.cat(inputs))[0]
618
@@ -628,11 +642,12 @@ def test_linear_parallel(dist_init, num_loras, orientation, fully_shard,
642
input_range=(0, 1),
643
input_type=torch.float16,
644
)
631
- lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
645
+ lora_mapping = LoRAMapping(index_mapping,
646
+ prompt_mapping,
647
+ is_prefill=stage)
648
633
- mapping_info = convert_mapping(lora_mapping, id_to_index, max_loras,
649
+ punica_wrapper.update_metadata(lora_mapping, id_to_index, max_loras,
650
512, lora_config.lora_extra_vocab_size)
635
- lora_linear.set_mapping(*mapping_info, )
651
652
lora_result = lora_linear(torch.cat(inputs))[0]
653
expected_result = linear(torch.cat(inputs))[0]
@@ -649,10 +664,12 @@ def test_linear_parallel(dist_init, num_loras, orientation, fully_shard,
664
@pytest.mark.parametrize("repeats", [1, 2, 3])
665
@pytest.mark.parametrize("fully_shard", [True, False])
666
@pytest.mark.parametrize("device", CUDA_DEVICES)
667
+@pytest.mark.parametrize("stage", STAGES)
668
def test_column_parallel_packed(dist_init, num_loras, repeats, fully_shard,
653
- device) -> None:
669
+ device, stage) -> None:
670
671
torch.set_default_device(device)
672
+ punica_wrapper = PunicaWrapper(8192, 256, device)
673
max_loras = 8
674
lora_config = LoRAConfig(max_loras=max_loras,
675
max_lora_rank=8,
@@ -707,7 +724,7 @@ def test_column_parallel_packed(dist_init, num_loras, repeats, fully_shard,
724
id_to_index = get_random_id_to_index(num_loras, max_loras)
725
726
linear, lora_linear = create_column_parallel_packed_layer()
710
-
727
+ lora_linear.set_mapping(punica_wrapper)
728
lora_dict, sublora_dict = populate_loras(
729
id_to_index,
730
layer=lora_linear,
@@ -722,16 +739,17 @@ def test_column_parallel_packed(dist_init, num_loras, repeats, fully_shard,
739
input_range=(0, 1),
740
input_type=torch.float16,
741
)
725
- lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
742
+ lora_mapping = LoRAMapping(index_mapping,
743
+ prompt_mapping,
744
+ is_prefill=stage)
745
727
- mapping_info = convert_mapping(
746
+ punica_wrapper.update_metadata(
747
lora_mapping,
748
id_to_index,
749
max_loras,
750
512,
751
lora_config.lora_extra_vocab_size,
752
)
734
- lora_linear.set_mapping(*mapping_info)
753
754
lora_result = lora_linear(torch.cat(inputs))[0]
755
@@ -762,16 +780,18 @@ def test_column_parallel_packed(dist_init, num_loras, repeats, fully_shard,
780
input_range=(0, 1),
781
input_type=torch.float16,
782
)
765
- lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
783
+ lora_mapping = LoRAMapping(index_mapping,
784
+ prompt_mapping,
785
+ is_prefill=stage)
786
767
- mapping_info = convert_mapping(
787
+ punica_wrapper.update_metadata(
788
lora_mapping,
789
id_to_index,
790
max_loras,
791
512,
792
lora_config.lora_extra_vocab_size,
793
)
774
- lora_linear.set_mapping(*mapping_info)
794
+ # lora_linear.set_mapping(*mapping_info)
795
796
lora_result = lora_linear(torch.cat(inputs))[0]
797
expected_result = linear(torch.cat(inputs))[0]
@@ -803,7 +823,7 @@ def test_rotary_embedding_long_context(dist_init, num_loras, device,
823
if torch.cuda.is_available():
824
torch.cuda.manual_seed(seed)
825
torch.set_default_device(device)
806
-
826
+ punica_wrapper = PunicaWrapper(8192, 256, device)
827
max_loras = 8
828
lora_config = LoRAConfig(max_loras=max_loras,
829
max_lora_rank=8,
@@ -825,6 +845,7 @@ def test_rotary_embedding_long_context(dist_init, num_loras, device,
845
is_neox_style,
846
)
847
lora_rope = LinearScalingRotaryEmbeddingWithLora(rope)
848
+ lora_rope.set_mapping(punica_wrapper)
849
lora_rope.create_lora_weights(max_loras, lora_config)
850
linear_rope = get_rope(head_size, rotary_dim, max_position, base,
851
is_neox_style, {
@@ -840,6 +861,7 @@ def test_rotary_embedding_long_context(dist_init, num_loras, device,
861
input_range=(0, lora_config.lora_extra_vocab_size),
862
input_type=torch.float16,
863
)
864
+
865
lora_mapping = LoRAMapping(index_mapping, prompt_mapping)
866
long_lora_context = LongContextLoRAContext(list(scaling_factors),
867
rotary_dim)
@@ -854,7 +876,7 @@ def test_rotary_embedding_long_context(dist_init, num_loras, device,
876
for i in range(len(scaling_factors)):
877
long_lora_context.offsets_by_lora_id[i] = scaling_factor_to_offset.get(
878
scaling_factors[i], 0)
857
- mapping_info = convert_mapping(
879
+ punica_wrapper.update_metadata(
880
lora_mapping,
881
id_to_index,
882
max_loras,
@@ -862,7 +884,7 @@ def test_rotary_embedding_long_context(dist_init, num_loras, device,
884
lora_config.lora_extra_vocab_size,
885
long_lora_context=long_lora_context,
886
)
865
- lora_rope.set_mapping(*mapping_info)
887
+ # lora_rope.set_mapping(*mapping_info)
888
889
positions = torch.randint(0, max_position, (batch_size, seq_len))
890
query = torch.randn(batch_size,
tests/lora/test_lora.py
deleted
-224
@@ -1,224 +0,0 @@
1
-import pytest
2
-import torch
3
-
4
-from vllm.lora.layers import _apply_lora, _apply_lora_packed_nslice
5
-
6
-from .utils import DummyLoRAManager
7
-
8
-TENSOR_SIZES = [128, 1024, 2048, 4096, 8192, 11008, 11008 // 2, 11008 // 4]
9
-QKV_TENSOR_SIZES = [
10
- (8192, 1024, 1024),
11
- (8192 // 8, 1024 // 8, 1024 // 8),
12
- (4096, 4096, 4096),
13
- (4096 // 2, 4096 // 2, 4096 // 2),
14
-]
15
-BATCH_SIZES = [8, 32, 256]
16
-RANKS = [8]
17
-DTYPES = [torch.float16]
18
-TOLERANCES = {
19
- torch.float16: (5e-3, 5e-3),
20
- torch.bfloat16: (3e-2, 2e-2),
21
-}
22
-
23
-
24
-@pytest.mark.parametrize("m", TENSOR_SIZES)
25
-@pytest.mark.parametrize("n", TENSOR_SIZES)
26
-@pytest.mark.parametrize("k", BATCH_SIZES)
27
-@pytest.mark.parametrize("rank", RANKS)
28
-@pytest.mark.parametrize("dtype", DTYPES)
29
-def test_apply_lora(m, n, k, rank, dtype) -> None:
30
- manager = DummyLoRAManager()
31
-
32
- module_name = "module"
33
- weight = torch.rand([m, n], device="cuda", dtype=dtype)
34
-
35
- manager.init_random_lora(module_name, weight, rank=rank)
36
- lora = manager.get_module_lora(module_name)
37
-
38
- input = torch.rand(k, n, device="cuda", dtype=dtype)
39
- expected = input @ lora.lora_a @ lora.lora_b * lora.scaling
40
-
41
- lora_a_stack = torch.zeros(8,
42
- 1,
43
- lora.lora_a.shape[1],
44
- lora.lora_a.shape[0],
45
- device="cuda",
46
- dtype=dtype)
47
- lora_b_stack = torch.zeros(8,
48
- 1,
49
- lora.lora_b.shape[1],
50
- lora.lora_b.shape[0],
51
- device="cuda",
52
- dtype=dtype)
53
- for i in range(lora_a_stack.shape[0]):
54
- lora_a_stack[i][0] = lora.lora_a.T
55
- lora_b_stack[i][0] = (lora.lora_b * lora.scaling).T
56
-
57
- output = torch.zeros(k, m, device="cuda", dtype=dtype)
58
- _apply_lora(
59
- input, lora_a_stack, lora_b_stack,
60
- torch.randint(0, lora_a_stack.shape[0], (len(input), ), device="cuda"),
61
- output)
62
-
63
- rtol, atol = TOLERANCES[dtype]
64
- assert torch.allclose(expected, output, rtol=rtol, atol=atol)
65
-
66
- output[:] = 0
67
- _apply_lora(input, lora_a_stack, lora_b_stack,
68
- torch.full((len(input), ), -1, device="cuda"), output)
69
- assert torch.allclose(torch.zeros_like(output), output)
70
-
71
- manager.reset_lora()
72
-
73
-
74
-@pytest.mark.parametrize("m", TENSOR_SIZES)
75
-@pytest.mark.parametrize("n", TENSOR_SIZES)
76
-@pytest.mark.parametrize("k", BATCH_SIZES)
77
-@pytest.mark.parametrize("rank", RANKS)
78
-@pytest.mark.parametrize("dtype", DTYPES)
79
-def test_apply_lora_packed_2slice(m, n, k, rank, dtype) -> None:
80
- if m % 2 != 0:
81
- pytest.skip("m must be divisible by 2")
82
- if m // 2 not in TENSOR_SIZES:
83
- pytest.skip("m//2 must be in TENSOR_SIZES")
84
-
85
- manager = DummyLoRAManager()
86
-
87
- module_name = "module"
88
- weight = torch.rand([m // 2, n], device="cuda", dtype=dtype)
89
-
90
- manager.init_random_lora(module_name + "1", weight, rank=rank)
91
- lora_1 = manager.get_module_lora(module_name + "1")
92
- manager.init_random_lora(module_name + "2", weight, rank=rank)
93
- lora_2 = manager.get_module_lora(module_name + "2")
94
-
95
- input = torch.rand(k, n, device="cuda", dtype=dtype)
96
- expected = torch.cat([
97
- input @ lora_1.lora_a @ lora_1.lora_b * lora_1.scaling,
98
- input @ lora_2.lora_a @ lora_2.lora_b * lora_2.scaling
99
- ],
100
- dim=1)
101
-
102
- lora_a_stacks = [
103
- torch.zeros(8,
104
- 1,
105
- lora_1.lora_a.shape[1],
106
- lora_1.lora_a.shape[0],
107
- device="cuda",
108
- dtype=dtype) for i in range(2)
109
- ]
110
- lora_b_stacks = [
111
- torch.zeros(8,
112
- 1,
113
- lora_1.lora_b.shape[1],
114
- lora_1.lora_b.shape[0],
115
- device="cuda",
116
- dtype=dtype) for i in range(2)
117
- ]
118
- for i in range(lora_a_stacks[0].shape[0]):
119
- lora_a_stacks[0][i][0] = lora_1.lora_a.T
120
- lora_b_stacks[0][i][0] = (lora_1.lora_b * lora_1.scaling).T
121
- lora_a_stacks[1][i][0] = lora_2.lora_a.T
122
- lora_b_stacks[1][i][0] = (lora_2.lora_b * lora_2.scaling).T
123
-
124
- output = torch.zeros(k, m, device="cuda", dtype=dtype)
125
- _apply_lora_packed_nslice(
126
- input, lora_a_stacks, lora_b_stacks,
127
- torch.randint(0,
128
- lora_a_stacks[0].shape[0], (len(input), ),
129
- device="cuda"), output, (m // 2, m // 2))
130
-
131
- rtol, atol = TOLERANCES[dtype]
132
- assert torch.allclose(expected, output, rtol=rtol, atol=atol)
133
-
134
- output[:] = 0
135
- _apply_lora_packed_nslice(input, lora_a_stacks, lora_b_stacks,
136
- torch.full((len(input), ), -1, device="cuda"),
137
- output, (m // 2, m // 2))
138
- assert torch.allclose(torch.zeros_like(output), output)
139
-
140
- manager.reset_lora()
141
-
142
-
143
-@pytest.mark.parametrize("qkv", QKV_TENSOR_SIZES)
144
-@pytest.mark.parametrize("n", TENSOR_SIZES)
145
-@pytest.mark.parametrize("k", BATCH_SIZES)
146
-@pytest.mark.parametrize("rank", RANKS)
147
-@pytest.mark.parametrize("dtype", DTYPES)
148
-def test_apply_lora_packed_3slice(qkv, n, k, rank, dtype) -> None:
149
- manager = DummyLoRAManager()
150
-
151
- module_name = "module"
152
- weight_q = torch.empty(qkv[0], n, device="cuda", dtype=dtype)
153
- weight_kv = torch.empty(qkv[1], n, device="cuda", dtype=dtype)
154
-
155
- manager.init_random_lora(module_name + "q", weight_q, rank=rank)
156
- lora_q = manager.get_module_lora(module_name + "q")
157
- manager.init_random_lora(module_name + "k", weight_kv, rank=rank)
158
- lora_k = manager.get_module_lora(module_name + "k")
159
- manager.init_random_lora(module_name + "v", weight_kv, rank=rank)
160
- lora_v = manager.get_module_lora(module_name + "v")
161
-
162
- input = torch.rand(k, n, device="cuda", dtype=dtype)
163
- expected = torch.cat([
164
- input @ lora_q.lora_a @ lora_q.lora_b * lora_q.scaling,
165
- input @ lora_k.lora_a @ lora_k.lora_b * lora_k.scaling,
166
- input @ lora_v.lora_a @ lora_v.lora_b * lora_v.scaling
167
- ],
168
- dim=1)
169
-
170
- lora_a_stacks = [
171
- torch.zeros(8,
172
- 1,
173
- lora_q.lora_a.shape[1],
174
- lora_q.lora_a.shape[0],
175
- device="cuda",
176
- dtype=dtype)
177
- ] + [
178
- torch.zeros(8,
179
- 1,
180
- lora_k.lora_a.shape[1],
181
- lora_k.lora_a.shape[0],
182
- device="cuda",
183
- dtype=dtype) for i in range(2)
184
- ]
185
- lora_b_stacks = [
186
- torch.zeros(8,
187
- 1,
188
- lora_q.lora_b.shape[1],
189
- lora_q.lora_b.shape[0],
190
- device="cuda",
191
- dtype=dtype)
192
- ] + [
193
- torch.zeros(8,
194
- 1,
195
- lora_k.lora_b.shape[1],
196
- lora_k.lora_b.shape[0],
197
- device="cuda",
198
- dtype=dtype) for i in range(2)
199
- ]
200
- for i in range(lora_a_stacks[0].shape[0]):
201
- lora_a_stacks[0][i][0] = lora_q.lora_a.T
202
- lora_b_stacks[0][i][0] = (lora_q.lora_b * lora_q.scaling).T
203
- lora_a_stacks[1][i][0] = lora_k.lora_a.T
204
- lora_b_stacks[1][i][0] = (lora_k.lora_b * lora_k.scaling).T
205
- lora_a_stacks[2][i][0] = lora_v.lora_a.T
206
- lora_b_stacks[2][i][0] = (lora_v.lora_b * lora_v.scaling).T
207
-
208
- output = torch.zeros(k, sum(qkv), device="cuda", dtype=dtype)
209
- _apply_lora_packed_nslice(
210
- input, lora_a_stacks, lora_b_stacks,
211
- torch.randint(0,
212
- lora_a_stacks[0].shape[0], (len(input), ),
213
- device="cuda"), output, (qkv[0], qkv[1], qkv[2]))
214
-
215
- rtol, atol = TOLERANCES[dtype]
216
- assert torch.allclose(expected, output, rtol=rtol, atol=atol)
217
-
218
- output[:] = 0
219
- _apply_lora_packed_nslice(input, lora_a_stacks, lora_b_stacks,
220
- torch.full((len(input), ), -1, device="cuda"),
221
- output, (qkv[0], qkv[1], qkv[2]))
222
- assert torch.allclose(torch.zeros_like(output), output)
223
-
224
- manager.reset_lora()
tests/lora/test_punica.py
deleted
-258
@@ -1,258 +0,0 @@
1
-# Based on code from https://github.com/punica-ai/punica
2
-
3
-import pytest
4
-import torch
5
-
6
-import vllm.lora.punica as punica
7
-
8
-
9
-def assert_close(a, b):
10
- rtol, atol = {
11
- torch.float16: (5e-3, 5e-3),
12
- torch.bfloat16: (3e-2, 2e-2),
13
- torch.float32: (None, None),
14
- }[a.dtype]
15
- torch.testing.assert_close(a, b, rtol=rtol, atol=atol)
16
-
17
-
18
-def _lora_ref_impl(
19
- y_final: torch.Tensor,
20
- x: torch.Tensor,
21
- wa_T_all: torch.Tensor,
22
- wb_T_all: torch.Tensor,
23
- indicies: torch.LongTensor,
24
- layer_idx: int,
25
- scale: float,
26
-):
27
- y_stage_1 = torch.empty(
28
- (x.size(0), wa_T_all.size(-2)),
29
- dtype=torch.float32,
30
- device=x.device,
31
- )
32
- bs = x.shape[0]
33
- s = torch.tensor(scale, dtype=torch.float32, device=x.device)
34
- for i, lora_idx in zip(range(bs), indicies.cpu().tolist()):
35
- xi = x[i].unsqueeze(0).to(torch.float32)
36
- wa = wa_T_all[lora_idx, layer_idx].transpose(-1, -2).to(torch.float32)
37
- if wb_T_all is not None:
38
- wb = wb_T_all[lora_idx, layer_idx].transpose(-1,
39
- -2).to(torch.float32)
40
-
41
- tmp = xi @ wa
42
- y_stage_1[i] = tmp.squeeze(0)
43
- y_final[i] += ((tmp @ wb).squeeze(0) *
44
- s if wb_T_all is not None else y_stage_1[i])
45
- return y_final, y_stage_1
46
-
47
-
48
-H1 = H2 = [
49
- 128,
50
- 256,
51
- 512,
52
- 896,
53
- 1024,
54
- 1152,
55
- 1216,
56
- 1280,
57
- 1536,
58
- 1664,
59
- 2048,
60
- 2240,
61
- 2304,
62
- 2368,
63
- 2432,
64
- 2560,
65
- 2752,
66
- 3072,
67
- 3328,
68
- 3456,
69
- 3584,
70
- 3712,
71
- 4096,
72
- 4480,
73
- 4608,
74
- 4736,
75
- 4864,
76
- 5120,
77
- 5504,
78
- 5632,
79
- 5888,
80
- 6144,
81
- 6400,
82
- 6848,
83
- 6912,
84
- 7168,
85
- 7424,
86
- 8192,
87
- 8960,
88
- 9216,
89
- 9472,
90
- 10240,
91
- 11008,
92
- 11264,
93
- 13824,
94
- 14336,
95
- 14784,
96
- 14848,
97
- 15360,
98
- 18944,
99
- 22016,
100
- 22528,
101
- 24576,
102
- 27392,
103
- 27648,
104
- 29568,
105
- 29696,
106
- 32000,
107
- 32256,
108
- 32512,
109
- 32768,
110
- 33024,
111
- 36864,
112
- 43264,
113
- 49152,
114
- 49408,
115
- 60544,
116
- 60672,
117
- 64000,
118
- 64256,
119
- 102400,
120
- 102656,
121
- 128000,
122
- 128256,
123
-]
124
-H2 = [64] + H2
125
-R = [1, 2, 4]
126
-SEED = [0xabcdabcd987]
127
-CUDA_DEVICES = [
128
- f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
129
-]
130
-
131
-
132
-@pytest.mark.parametrize("dtype_str", ["float16", "bfloat16"])
133
-@pytest.mark.parametrize("h1", H1)
134
-@pytest.mark.parametrize("r", R)
135
-@pytest.mark.parametrize("seed", SEED)
136
-@torch.inference_mode()
137
-def test_lora_a_extra_shapes(dtype_str, h1, r, seed):
138
- torch.manual_seed(seed)
139
- num_loras = 4
140
- num_layers = 1
141
- bs = 32
142
- dtype = getattr(torch, dtype_str)
143
- device = torch.device("cuda")
144
-
145
- wa_T_all = torch.randn(num_loras,
146
- num_layers,
147
- r,
148
- h1,
149
- dtype=dtype,
150
- device=device)
151
- indices = torch.randint(num_loras, (bs, ), dtype=torch.long, device=device)
152
-
153
- for layer_idx in range(num_layers):
154
- x = torch.randn(bs, h1, dtype=dtype, device=device)
155
- y = torch.randn(bs, r, dtype=dtype, device=device)
156
-
157
- y_ref = y.clone()
158
- _lora_ref_impl(
159
- y_ref,
160
- x,
161
- wa_T_all,
162
- None,
163
- indices,
164
- layer_idx,
165
- 1.0,
166
- )
167
-
168
- y_our = y.clone()
169
- punica.bgmv(y_our, x, wa_T_all, indices, layer_idx, 1.0)
170
-
171
- assert_close(y_ref, y_our)
172
-
173
-
174
-@pytest.mark.parametrize("dtype_str", ["float16", "bfloat16"])
175
-@pytest.mark.parametrize("h1", H1)
176
-@pytest.mark.parametrize("h2", H2)
177
-@pytest.mark.parametrize("seed", SEED)
178
-@pytest.mark.parametrize("device", CUDA_DEVICES)
179
-@torch.inference_mode()
180
-def test_lora_correctness(dtype_str, h1, h2, seed, device):
181
- torch.manual_seed(seed)
182
- num_loras = 4
183
- num_layers = 1
184
- r = 8
185
- bs = 32
186
- scale = 0.123
187
- dtype = getattr(torch, dtype_str)
188
- torch.set_default_device(device)
189
-
190
- wa_T_all = torch.randn(num_loras, num_layers, r, h1, dtype=dtype)
191
- wb_T_all = torch.randn(num_loras, num_layers, h2, r, dtype=dtype)
192
- indices = torch.randint(num_loras, (bs, ), dtype=torch.long)
193
-
194
- for layer_idx in range(num_layers):
195
- x = torch.randn(bs, h1, dtype=dtype)
196
- y = torch.randn(bs, h2, dtype=dtype)
197
-
198
- y_ref = y.clone()
199
- _lora_ref_impl(y_ref, x, wa_T_all, wb_T_all, indices, layer_idx, scale)
200
-
201
- y_our = y.clone()
202
- punica.add_lora(y_our, x, wa_T_all, wb_T_all, indices, layer_idx,
203
- scale)
204
-
205
- assert_close(y_ref, y_our)
206
-
207
-
208
-@pytest.mark.parametrize("dtype_str", ["float16", "bfloat16"])
209
-@pytest.mark.parametrize("h1", H1)
210
-@pytest.mark.parametrize("h2", H2)
211
-@pytest.mark.parametrize("seed", SEED)
212
-@pytest.mark.parametrize("device", CUDA_DEVICES)
213
-@torch.inference_mode()
214
-def test_lora_correctness_slice(dtype_str, h1, h2, seed, device):
215
- if h2 % 3 != 0 or h2 // 3 not in H1:
216
- pytest.skip("h2 must be divisible by 3 and in supported shapes")
217
- torch.manual_seed(seed)
218
- num_loras = 4
219
- num_layers = 1
220
- r = 8
221
- bs = 32
222
- scale = 0.123
223
- dtype = getattr(torch, dtype_str)
224
- torch.set_default_device(device)
225
-
226
- wa_T_all_0 = torch.randn(num_loras, num_layers, r, h1, dtype=dtype)
227
- wa_T_all_1 = torch.randn(num_loras, num_layers, r, h1, dtype=dtype)
228
- wa_T_all_2 = torch.randn(num_loras, num_layers, r, h1, dtype=dtype)
229
- wb_T_all_0 = torch.randn(num_loras, num_layers, h2 // 3, r, dtype=dtype)
230
- wb_T_all_1 = torch.randn(num_loras, num_layers, h2 // 3, r, dtype=dtype)
231
- wb_T_all_2 = torch.randn(num_loras, num_layers, h2 // 3, r, dtype=dtype)
232
-
233
- indices = torch.randint(num_loras, (bs, ), dtype=torch.long)
234
-
235
- for layer_idx in range(num_layers):
236
- x = torch.randn(bs, h1, dtype=dtype)
237
- y = torch.randn(bs, h2, dtype=dtype)
238
- s = h2 // 3
239
-
240
- y_ref = y.clone()
241
- _lora_ref_impl(y_ref[:, :s], x, wa_T_all_0, wb_T_all_0, indices,
242
- layer_idx, scale)
243
- _lora_ref_impl(y_ref[:, s:s * 2], x, wa_T_all_1, wb_T_all_1, indices,
244
- layer_idx, scale)
245
- _lora_ref_impl(y_ref[:, s * 2:], x, wa_T_all_2, wb_T_all_2, indices,
246
- layer_idx, scale)
247
-
248
- y_our = y.clone()
249
- punica.add_lora_slice(y_our, x, wa_T_all_0, wb_T_all_0, indices,
250
- layer_idx, scale, 0, s)
251
- punica.add_lora_slice(y_our, x, wa_T_all_1, wb_T_all_1, indices,
252
- layer_idx, scale, s, s)
253
- punica.add_lora_slice(y_our, x, wa_T_all_2, wb_T_all_2, indices,
254
- layer_idx, scale, s * 2, s)
255
-
256
- assert_close(y_ref[:, :s], y_our[:, :s])
257
- assert_close(y_ref[:, s:s * 2], y_our[:, s:s * 2])
258
- assert_close(y_ref[:, s * 2:], y_our[:, s * 2:])
tests/lora/test_punica_sizes.py
new
+408
@@ -0,0 +1,408 @@
1
+"""
2
+This script is mainly used to tests various hidden_sizes. We have collected the
3
+hidden_sizes included in the LoRA models currently supported by vLLM. It tests
4
+whether the corresponding Triton kernel can run normally when tensor parallelism
5
+is set to [1, 2, 4, 8, 16, 32, 64].
6
+"""
7
+import random
8
+from unittest.mock import patch
9
+
10
+import pytest
11
+import torch
12
+
13
+from vllm.lora.ops.bgmv_expand import bgmv_expand
14
+from vllm.lora.ops.bgmv_expand_slice import bgmv_expand_slice
15
+from vllm.lora.ops.bgmv_shrink import bgmv_shrink
16
+from vllm.lora.ops.sgmv_expand import sgmv_expand
17
+from vllm.lora.ops.sgmv_expand_slice import sgmv_expand_slice
18
+from vllm.lora.ops.sgmv_shrink import sgmv_shrink
19
+from vllm.triton_utils.libentry import LibEntry
20
+
21
+from .utils import (generate_data, generate_data_for_expand_nslices,
22
+ ref_torch_groupgemm)
23
+
24
+HIDDEN_SIZES = [
25
+ 128,
26
+ 256,
27
+ 512,
28
+ 896,
29
+ 1024,
30
+ 1152,
31
+ 1216,
32
+ 1280,
33
+ 1536,
34
+ 1664,
35
+ 2048,
36
+ 2240,
37
+ 2304,
38
+ 2368,
39
+ 2432,
40
+ 2560,
41
+ 2752,
42
+ 3072,
43
+ 3328,
44
+ 3456,
45
+ 3584,
46
+ 3712,
47
+ 4096,
48
+ 4480,
49
+ 4608,
50
+ 4736,
51
+ 4864,
52
+ 5120,
53
+ 5504,
54
+ 5632,
55
+ 5888,
56
+ 6144,
57
+ 6400,
58
+ 6848,
59
+ 6912,
60
+ 7168,
61
+ 7424,
62
+ 8192,
63
+ 8960,
64
+ 9216,
65
+ 9472,
66
+ 10240,
67
+ 11008,
68
+ 11264,
69
+ 13824,
70
+ 14336,
71
+ 14784,
72
+ 14848,
73
+ 15360,
74
+ 18944,
75
+ 22016,
76
+ 22528,
77
+ 24576,
78
+ 27392,
79
+ 27648,
80
+ 29568,
81
+ 29696,
82
+ 32000,
83
+ 32256,
84
+ 32512,
85
+ 32768,
86
+ 33024,
87
+ 36864,
88
+ 43264,
89
+ 49152,
90
+ 49408,
91
+ 60544,
92
+ 60672,
93
+ 64000,
94
+ 64256,
95
+ 102400,
96
+ 102656,
97
+ 128000,
98
+ 128256,
99
+]
100
+#The size of TP
101
+divisibility = [1, 2, 4, 8, 16, 32, 64]
102
+
103
+all_hidden_size = []
104
+for div in divisibility:
105
+ for hidden_size in HIDDEN_SIZES:
106
+ all_hidden_size.append(hidden_size // div)
107
+
108
+HIDDEN_SIZES = list(set(all_hidden_size))
109
+
110
+BATCHES = [4]
111
+NUM_LORA = [4]
112
+DTYPES = [torch.float16, torch.bfloat16]
113
+MAX_RANKS = [32]
114
+SCALES = [0.5]
115
+SEED = [0]
116
+CUDA_DEVICES = [f"cuda:{0}"]
117
+
118
+
119
+def assert_close(a, b):
120
+ rtol, atol = {
121
+ torch.float16: (6e-2, 6e-2),
122
+ torch.bfloat16: (6e-2, 6e-2),
123
+ torch.float32: (1e-2, 1e-2),
124
+ }[a.dtype]
125
+ torch.testing.assert_close(a, b, rtol=rtol, atol=atol)
126
+
127
+
128
+@pytest.mark.parametrize("batches", BATCHES)
129
+@pytest.mark.parametrize("num_loras", NUM_LORA)
130
+@pytest.mark.parametrize("rank", MAX_RANKS)
131
+@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
132
+@pytest.mark.parametrize("scaling", SCALES)
133
+@pytest.mark.parametrize("dtype", DTYPES)
134
+@pytest.mark.parametrize("op_type", ["shrink", "expand"])
135
+@pytest.mark.parametrize("seed", SEED)
136
+@pytest.mark.parametrize("device", CUDA_DEVICES)
137
+def test_punica_sgmv(
138
+ batches: int,
139
+ num_loras: int,
140
+ rank: int,
141
+ hidden_size: int,
142
+ scaling: float,
143
+ dtype: torch.dtype,
144
+ op_type: str,
145
+ seed: int,
146
+ device: str,
147
+):
148
+ random.seed(seed)
149
+ torch.set_default_device(device)
150
+ torch.random.manual_seed(seed)
151
+ if torch.cuda.is_available():
152
+ torch.cuda.manual_seed(seed)
153
+
154
+ seq_length = 128
155
+ (
156
+ inputs_tensor,
157
+ lora_weights,
158
+ our_out_tensor,
159
+ ref_out_tensor,
160
+ b_seq_start_loc,
161
+ lora_indices_tensor,
162
+ seq_len_tensor,
163
+ indices,
164
+ ) = generate_data(
165
+ batches,
166
+ hidden_size,
167
+ num_loras,
168
+ rank,
169
+ seq_length,
170
+ dtype,
171
+ op_type,
172
+ device,
173
+ )
174
+ max_seq_length = seq_len_tensor.max()
175
+ if isinstance(max_seq_length, tuple):
176
+ max_seq_length = max_seq_length[0].item()
177
+ else:
178
+ max_seq_length = max_seq_length.item()
179
+ if op_type == "shrink":
180
+ sgmv_shrink(
181
+ inputs_tensor,
182
+ lora_weights,
183
+ our_out_tensor,
184
+ b_seq_start_loc,
185
+ seq_len_tensor,
186
+ lora_indices_tensor,
187
+ batches,
188
+ max_seq_length,
189
+ scaling,
190
+ )
191
+ else:
192
+ sgmv_expand(
193
+ inputs_tensor,
194
+ lora_weights,
195
+ our_out_tensor,
196
+ b_seq_start_loc,
197
+ seq_len_tensor,
198
+ lora_indices_tensor,
199
+ batches,
200
+ max_seq_length,
201
+ add_inputs=True,
202
+ )
203
+ ref_torch_groupgemm(
204
+ ref_out_tensor,
205
+ inputs_tensor,
206
+ lora_weights,
207
+ lora_indices_tensor,
208
+ seq_len_tensor,
209
+ batches,
210
+ scaling if op_type == "shrink" else 1.0,
211
+ op_type,
212
+ )
213
+ if op_type == "shrink":
214
+ ref_out_tensor = ref_out_tensor.to(torch.float32)
215
+ assert_close(our_out_tensor, ref_out_tensor)
216
+
217
+
218
+@pytest.mark.parametrize("batches", BATCHES)
219
+@pytest.mark.parametrize("num_loras", NUM_LORA)
220
+@pytest.mark.parametrize("rank", MAX_RANKS)
221
+@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
222
+@pytest.mark.parametrize("scaling", SCALES)
223
+@pytest.mark.parametrize("dtype", DTYPES)
224
+@pytest.mark.parametrize("op_type", ["shrink", "expand"])
225
+@pytest.mark.parametrize("seed", SEED)
226
+@pytest.mark.parametrize("device", CUDA_DEVICES)
227
+def test_punica_bgmv(
228
+ batches: int,
229
+ num_loras: int,
230
+ rank: int,
231
+ hidden_size: int,
232
+ scaling: float,
233
+ dtype: torch.dtype,
234
+ op_type: str,
235
+ seed: int,
236
+ device: str,
237
+):
238
+ from vllm.lora.ops.bgmv_expand import _bgmv_expand_kernel
239
+ from vllm.lora.ops.bgmv_shrink import _bgmv_shrink_kernel
240
+
241
+ random.seed(seed)
242
+ torch.set_default_device(device)
243
+ torch.random.manual_seed(seed)
244
+ if torch.cuda.is_available():
245
+ torch.cuda.manual_seed(seed)
246
+
247
+ seq_length = 1
248
+ (
249
+ inputs_tensor,
250
+ lora_weights,
251
+ our_out_tensor,
252
+ ref_out_tensor,
253
+ b_seq_start_loc,
254
+ lora_indices_tensor,
255
+ seq_len_tensor,
256
+ indices,
257
+ ) = generate_data(
258
+ batches,
259
+ hidden_size,
260
+ num_loras,
261
+ rank,
262
+ seq_length,
263
+ dtype,
264
+ op_type,
265
+ device,
266
+ )
267
+ if op_type == "shrink":
268
+ # The current _bgmv_shrink_kernel does not require the libentry
269
+ # decoration. The purpose of adding this patch is to test the
270
+ # correctness of libentry.
271
+ with patch(
272
+ "vllm.lora.ops.bgmv_shrink._bgmv_shrink_kernel",
273
+ LibEntry(_bgmv_shrink_kernel),
274
+ ):
275
+ bgmv_shrink(
276
+ inputs_tensor,
277
+ lora_weights,
278
+ our_out_tensor,
279
+ indices,
280
+ scaling,
281
+ )
282
+ else:
283
+ # ditto
284
+ with patch(
285
+ "vllm.lora.ops.bgmv_expand._bgmv_expand_kernel",
286
+ LibEntry(_bgmv_expand_kernel),
287
+ ):
288
+ bgmv_expand(
289
+ inputs_tensor,
290
+ lora_weights,
291
+ our_out_tensor,
292
+ indices,
293
+ add_inputs=True,
294
+ )
295
+ ref_torch_groupgemm(
296
+ ref_out_tensor,
297
+ inputs_tensor,
298
+ lora_weights,
299
+ lora_indices_tensor,
300
+ seq_len_tensor,
301
+ batches,
302
+ scaling if op_type == "shrink" else 1.0,
303
+ op_type,
304
+ )
305
+ if op_type == "shrink":
306
+ ref_out_tensor = ref_out_tensor.to(torch.float32)
307
+ assert_close(our_out_tensor, ref_out_tensor)
308
+
309
+
310
+@pytest.mark.parametrize("batches", BATCHES)
311
+@pytest.mark.parametrize("num_loras", NUM_LORA)
312
+@pytest.mark.parametrize("rank", MAX_RANKS)
313
+@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
314
+@pytest.mark.parametrize("nslices", [2, 3])
315
+@pytest.mark.parametrize("dtype", DTYPES)
316
+@pytest.mark.parametrize("op_type", ["sgmv", "bgmv"])
317
+@pytest.mark.parametrize("seed", SEED)
318
+@pytest.mark.parametrize("device", CUDA_DEVICES)
319
+def test_punica_expand_nslices(
320
+ batches: int,
321
+ num_loras: int,
322
+ rank: int,
323
+ hidden_size: int,
324
+ nslices: int,
325
+ dtype: torch.dtype,
326
+ op_type: str,
327
+ seed: int,
328
+ device: str,
329
+):
330
+ from vllm.lora.ops.bgmv_expand_slice import _bgmv_expand_slice_kernel
331
+
332
+ random.seed(seed)
333
+ torch.set_default_device(device)
334
+ torch.random.manual_seed(seed)
335
+ if torch.cuda.is_available():
336
+ torch.cuda.manual_seed(seed)
337
+ seq_length = 128 if op_type == "sgmv" else 1
338
+ (
339
+ inputs_tensor,
340
+ lora_weights_lst,
341
+ our_outputs,
342
+ ref_outputs,
343
+ b_seq_start_loc,
344
+ lora_indices_tensor,
345
+ seq_len_tensor,
346
+ indices,
347
+ ) = generate_data_for_expand_nslices(
348
+ batches,
349
+ hidden_size,
350
+ num_loras,
351
+ rank,
352
+ seq_length,
353
+ dtype,
354
+ nslices,
355
+ device,
356
+ )
357
+ max_seq_length = seq_len_tensor.max()
358
+ if isinstance(max_seq_length, tuple):
359
+ max_seq_length = max_seq_length[0].item()
360
+ else:
361
+ max_seq_length = max_seq_length.item()
362
+ slice_offset = 0
363
+ for index in range(nslices):
364
+ lora_weights = lora_weights_lst[index]
365
+ if op_type == "sgmv":
366
+ sgmv_expand_slice(
367
+ inputs_tensor,
368
+ lora_weights,
369
+ our_outputs,
370
+ b_seq_start_loc,
371
+ seq_len_tensor,
372
+ lora_indices_tensor,
373
+ batches,
374
+ max_seq_length,
375
+ slice_offset,
376
+ hidden_size,
377
+ add_inputs=True,
378
+ )
379
+ else:
380
+ # The current _bgmv_expand_slice_kernel does not require the
381
+ # libentry decoration. The purpose of adding this patch is to test
382
+ # the correctness of libentry.
383
+ with patch(
384
+ "vllm.lora.ops.bgmv_expand_slice._bgmv_expand_slice_kernel",
385
+ LibEntry(_bgmv_expand_slice_kernel),
386
+ ):
387
+ bgmv_expand_slice(
388
+ inputs_tensor,
389
+ lora_weights,
390
+ our_outputs,
391
+ indices,
392
+ slice_offset,
393
+ slice_size=hidden_size,
394
+ add_inputs=True,
395
+ )
396
+ ref_torch_groupgemm(
397
+ ref_outputs[:, slice_offset:slice_offset + hidden_size],
398
+ inputs_tensor,
399
+ lora_weights,
400
+ lora_indices_tensor,
401
+ seq_len_tensor,
402
+ batches,
403
+ 1.0,
404
+ op_type="expand",
405
+ )
406
+
407
+ slice_offset += hidden_size
408
+ assert_close(our_outputs, ref_outputs)
tests/lora/test_punica_variation.py
new
+342
@@ -0,0 +1,342 @@
1
+"""
2
+This script is mainly used to test whether trtion kernels can run normally
3
+under different conditions, including various batches, numbers of LoRA , and
4
+maximum ranks.
5
+"""
6
+import random
7
+from unittest.mock import patch
8
+
9
+import pytest
10
+import torch
11
+
12
+from vllm.lora.ops.bgmv_expand import bgmv_expand
13
+from vllm.lora.ops.bgmv_expand_slice import bgmv_expand_slice
14
+from vllm.lora.ops.bgmv_shrink import bgmv_shrink
15
+from vllm.lora.ops.sgmv_expand import sgmv_expand
16
+from vllm.lora.ops.sgmv_expand_slice import sgmv_expand_slice
17
+from vllm.lora.ops.sgmv_shrink import sgmv_shrink
18
+from vllm.triton_utils.libentry import LibEntry
19
+
20
+from .utils import (generate_data, generate_data_for_expand_nslices,
21
+ ref_torch_groupgemm)
22
+
23
+HIDDEN_SIZES = [3424, 4096, 4097]
24
+
25
+BATCHES = [1, 4, 16, 32]
26
+NUM_LORA = [1, 4, 8, 16, 32, 64, 128]
27
+DTYPES = [torch.float16, torch.bfloat16]
28
+MAX_RANKS = [1, 4, 8, 16, 32, 64, 128]
29
+SCALES = [0.5]
30
+SEED = [0]
31
+CUDA_DEVICES = [f"cuda:{0}"]
32
+
33
+
34
+def assert_close(a, b):
35
+ rtol, atol = {
36
+ torch.float16: (6e-2, 6e-2),
37
+ torch.bfloat16: (6e-2, 6e-2),
38
+ torch.float32: (1e-2, 1e-2),
39
+ }[a.dtype]
40
+ torch.testing.assert_close(a, b, rtol=rtol, atol=atol)
41
+
42
+
43
+@pytest.mark.parametrize("batches", BATCHES)
44
+@pytest.mark.parametrize("num_loras", NUM_LORA)
45
+@pytest.mark.parametrize("rank", MAX_RANKS)
46
+@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
47
+@pytest.mark.parametrize("scaling", SCALES)
48
+@pytest.mark.parametrize("dtype", DTYPES)
49
+@pytest.mark.parametrize("op_type", ["shrink", "expand"])
50
+@pytest.mark.parametrize("seed", SEED)
51
+@pytest.mark.parametrize("device", CUDA_DEVICES)
52
+def test_punica_sgmv(
53
+ batches: int,
54
+ num_loras: int,
55
+ rank: int,
56
+ hidden_size: int,
57
+ scaling: float,
58
+ dtype: torch.dtype,
59
+ op_type: str,
60
+ seed: int,
61
+ device: str,
62
+):
63
+ random.seed(seed)
64
+ torch.set_default_device(device)
65
+ torch.random.manual_seed(seed)
66
+ if torch.cuda.is_available():
67
+ torch.cuda.manual_seed(seed)
68
+
69
+ seq_length = 128
70
+ (
71
+ inputs_tensor,
72
+ lora_weights,
73
+ our_out_tensor,
74
+ ref_out_tensor,
75
+ b_seq_start_loc,
76
+ lora_indices_tensor,
77
+ seq_len_tensor,
78
+ indices,
79
+ ) = generate_data(
80
+ batches,
81
+ hidden_size,
82
+ num_loras,
83
+ rank,
84
+ seq_length,
85
+ dtype,
86
+ op_type,
87
+ device,
88
+ )
89
+ max_seq_length = seq_len_tensor.max()
90
+ if isinstance(max_seq_length, tuple):
91
+ max_seq_length = max_seq_length[0].item()
92
+ else:
93
+ max_seq_length = max_seq_length.item()
94
+ if op_type == "shrink":
95
+ sgmv_shrink(
96
+ inputs_tensor,
97
+ lora_weights,
98
+ our_out_tensor,
99
+ b_seq_start_loc,
100
+ seq_len_tensor,
101
+ lora_indices_tensor,
102
+ batches,
103
+ max_seq_length,
104
+ scaling,
105
+ )
106
+ else:
107
+ sgmv_expand(
108
+ inputs_tensor,
109
+ lora_weights,
110
+ our_out_tensor,
111
+ b_seq_start_loc,
112
+ seq_len_tensor,
113
+ lora_indices_tensor,
114
+ batches,
115
+ max_seq_length,
116
+ add_inputs=True,
117
+ )
118
+ ref_torch_groupgemm(
119
+ ref_out_tensor,
120
+ inputs_tensor,
121
+ lora_weights,
122
+ lora_indices_tensor,
123
+ seq_len_tensor,
124
+ batches,
125
+ scaling if op_type == "shrink" else 1.0,
126
+ op_type,
127
+ )
128
+ if op_type == "shrink":
129
+ ref_out_tensor = ref_out_tensor.to(torch.float32)
130
+ assert_close(our_out_tensor, ref_out_tensor)
131
+
132
+
133
+@pytest.mark.parametrize("batches", BATCHES)
134
+@pytest.mark.parametrize("num_loras", NUM_LORA)
135
+@pytest.mark.parametrize("rank", MAX_RANKS)
136
+@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
137
+@pytest.mark.parametrize("scaling", SCALES)
138
+@pytest.mark.parametrize("dtype", DTYPES)
139
+@pytest.mark.parametrize("op_type", ["shrink", "expand"])
140
+@pytest.mark.parametrize("seed", SEED)
141
+@pytest.mark.parametrize("device", CUDA_DEVICES)
142
+def test_punica_bgmv(
143
+ batches: int,
144
+ num_loras: int,
145
+ rank: int,
146
+ hidden_size: int,
147
+ scaling: float,
148
+ dtype: torch.dtype,
149
+ op_type: str,
150
+ seed: int,
151
+ device: str,
152
+):
153
+ from vllm.lora.ops.bgmv_expand import _bgmv_expand_kernel
154
+ from vllm.lora.ops.bgmv_shrink import _bgmv_shrink_kernel
155
+
156
+ random.seed(seed)
157
+ torch.set_default_device(device)
158
+ torch.random.manual_seed(seed)
159
+ if torch.cuda.is_available():
160
+ torch.cuda.manual_seed(seed)
161
+
162
+ seq_length = 1
163
+ (
164
+ inputs_tensor,
165
+ lora_weights,
166
+ our_out_tensor,
167
+ ref_out_tensor,
168
+ b_seq_start_loc,
169
+ lora_indices_tensor,
170
+ seq_len_tensor,
171
+ indices,
172
+ ) = generate_data(
173
+ batches,
174
+ hidden_size,
175
+ num_loras,
176
+ rank,
177
+ seq_length,
178
+ dtype,
179
+ op_type,
180
+ device,
181
+ )
182
+ if op_type == "shrink":
183
+ # The current _bgmv_shrink_kernel does not require the libentry
184
+ # decoration. The purpose of adding this patch is to test the
185
+ # correctness of libentry.
186
+ with patch(
187
+ "vllm.lora.ops.bgmv_shrink._bgmv_shrink_kernel",
188
+ LibEntry(_bgmv_shrink_kernel),
189
+ ):
190
+ bgmv_shrink(
191
+ inputs_tensor,
192
+ lora_weights,
193
+ our_out_tensor,
194
+ indices,
195
+ scaling,
196
+ )
197
+ else:
198
+ # ditto
199
+ with patch(
200
+ "vllm.lora.ops.bgmv_expand._bgmv_expand_kernel",
201
+ LibEntry(_bgmv_expand_kernel),
202
+ ):
203
+ bgmv_expand(
204
+ inputs_tensor,
205
+ lora_weights,
206
+ our_out_tensor,
207
+ indices,
208
+ add_inputs=True,
209
+ )
210
+ ref_torch_groupgemm(
211
+ ref_out_tensor,
212
+ inputs_tensor,
213
+ lora_weights,
214
+ lora_indices_tensor,
215
+ seq_len_tensor,
216
+ batches,
217
+ scaling if op_type == "shrink" else 1.0,
218
+ op_type,
219
+ )
220
+ if op_type == "shrink":
221
+ ref_out_tensor = ref_out_tensor.to(torch.float32)
222
+ assert_close(our_out_tensor, ref_out_tensor)
223
+
224
+
225
+@pytest.mark.parametrize("batches", BATCHES)
226
+@pytest.mark.parametrize("num_loras", NUM_LORA)
227
+@pytest.mark.parametrize("rank", MAX_RANKS)
228
+@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
229
+@pytest.mark.parametrize("nslices", [2, 3])
230
+@pytest.mark.parametrize("dtype", DTYPES)
231
+@pytest.mark.parametrize("op_type", ["sgmv", "bgmv"])
232
+@pytest.mark.parametrize("seed", SEED)
233
+@pytest.mark.parametrize("device", CUDA_DEVICES)
234
+def test_punica_expand_nslices(
235
+ batches: int,
236
+ num_loras: int,
237
+ rank: int,
238
+ hidden_size: int,
239
+ nslices: int,
240
+ dtype: torch.dtype,
241
+ op_type: str,
242
+ seed: int,
243
+ device: str,
244
+):
245
+ from vllm.lora.ops.bgmv_expand_slice import _bgmv_expand_slice_kernel
246
+
247
+ random.seed(seed)
248
+ torch.set_default_device(device)
249
+ torch.random.manual_seed(seed)
250
+ if torch.cuda.is_available():
251
+ torch.cuda.manual_seed(seed)
252
+ seq_length = 128 if op_type == "sgmv" else 1
253
+ (
254
+ inputs_tensor,
255
+ lora_weights_lst,
256
+ our_outputs,
257
+ ref_outputs,
258
+ b_seq_start_loc,
259
+ lora_indices_tensor,
260
+ seq_len_tensor,
261
+ indices,
262
+ ) = generate_data_for_expand_nslices(
263
+ batches,
264
+ hidden_size,
265
+ num_loras,
266
+ rank,
267
+ seq_length,
268
+ dtype,
269
+ nslices,
270
+ device,
271
+ )
272
+ max_seq_length = seq_len_tensor.max()
273
+ if isinstance(max_seq_length, tuple):
274
+ max_seq_length = max_seq_length[0].item()
275
+ else:
276
+ max_seq_length = max_seq_length.item()
277
+ slice_offset = 0
278
+ for index in range(nslices):
279
+ lora_weights = lora_weights_lst[index]
280
+ if op_type == "sgmv":
281
+ sgmv_expand_slice(
282
+ inputs_tensor,
283
+ lora_weights,
284
+ our_outputs,
285
+ b_seq_start_loc,
286
+ seq_len_tensor,
287
+ lora_indices_tensor,
288
+ batches,
289
+ max_seq_length,
290
+ slice_offset,
291
+ hidden_size,
292
+ add_inputs=True,
293
+ )
294
+ else:
295
+ # The current _bgmv_expand_slice_kernel does not require the
296
+ # libentry decoration. The purpose of adding this patch is to test
297
+ # the correctness of libentry.
298
+ with patch(
299
+ "vllm.lora.ops.bgmv_expand_slice._bgmv_expand_slice_kernel",
300
+ LibEntry(_bgmv_expand_slice_kernel),
301
+ ):
302
+ bgmv_expand_slice(
303
+ inputs_tensor,
304
+ lora_weights,
305
+ our_outputs,
306
+ indices,
307
+ slice_offset,
308
+ slice_size=hidden_size,
309
+ add_inputs=True,
310
+ )
311
+ ref_torch_groupgemm(
312
+ ref_outputs[:, slice_offset:slice_offset + hidden_size],
313
+ inputs_tensor,
314
+ lora_weights,
315
+ lora_indices_tensor,
316
+ seq_len_tensor,
317
+ batches,
318
+ 1.0,
319
+ op_type="expand",
320
+ )
321
+
322
+ slice_offset += hidden_size
323
+ assert_close(our_outputs, ref_outputs)
324
+
325
+
326
+if __name__ == "__main__":
327
+ from itertools import product
328
+
329
+ lst = list(
330
+ product(
331
+ BATCHES,
332
+ NUM_LORA,
333
+ MAX_RANKS,
334
+ [1.0],
335
+ [torch.float16],
336
+ ["expand"],
337
+ SEED,
338
+ CUDA_DEVICES,
339
+ ))
340
+ for ele in lst:
341
+ test_punica_bgmv(*ele)
342
+ print(f"{ele},pass")
tests/lora/test_quant_model.py
+27
-21
@@ -64,14 +64,16 @@ def test_quant_model_lora(tinyllama_lora_files, model, tp_size):
64
# if torch.cuda.device_count() < tp_size:
65
# pytest.skip(f"Not enough GPUs for tensor parallelism {tp_size}")
66
67
- llm = vllm.LLM(model=model.model_path,
68
- enable_lora=True,
69
- max_num_seqs=16,
70
- max_loras=4,
71
- max_model_len=400,
72
- tensor_parallel_size=tp_size,
73
- quantization=model.quantization,
74
- trust_remote_code=True)
67
+ llm = vllm.LLM(
68
+ model=model.model_path,
69
+ enable_lora=True,
70
+ max_num_seqs=16,
71
+ max_loras=4,
72
+ max_model_len=400,
73
+ tensor_parallel_size=tp_size,
74
+ gpu_memory_utilization=0.2, #avoid OOM
75
+ quantization=model.quantization,
76
+ trust_remote_code=True)
77
78
if model.quantization is None:
79
expected_no_lora_output = [
@@ -156,24 +158,28 @@ def test_quant_model_tp_equality(tinyllama_lora_files, model):
158
# if torch.cuda.device_count() < 2:
159
# pytest.skip(f"Not enough GPUs for tensor parallelism {2}")
160
159
- llm_tp1 = vllm.LLM(model=model.model_path,
160
- enable_lora=True,
161
- max_num_seqs=16,
162
- max_loras=4,
163
- tensor_parallel_size=1,
164
- quantization=model.quantization,
165
- trust_remote_code=True)
161
+ llm_tp1 = vllm.LLM(
162
+ model=model.model_path,
163
+ enable_lora=True,
164
+ max_num_seqs=16,
165
+ max_loras=4,
166
+ tensor_parallel_size=1,
167
+ gpu_memory_utilization=0.2, #avoid OOM
168
+ quantization=model.quantization,
169
+ trust_remote_code=True)
170
output_tp1 = do_sample(llm_tp1, tinyllama_lora_files, lora_id=1)
171
172
del llm_tp1
173
cleanup()
174
171
- llm_tp2 = vllm.LLM(model=model.model_path,
172
- enable_lora=True,
173
- max_num_seqs=16,
174
- max_loras=4,
175
- tensor_parallel_size=2,
176
- quantization=model.quantization)
175
+ llm_tp2 = vllm.LLM(
176
+ model=model.model_path,
177
+ enable_lora=True,
178
+ max_num_seqs=16,
179
+ max_loras=4,
180
+ tensor_parallel_size=2,
181
+ gpu_memory_utilization=0.2, #avoid OOM
182
+ quantization=model.quantization)
183
output_tp2 = do_sample(llm_tp2, tinyllama_lora_files, lora_id=1)
184
185
del llm_tp2
tests/lora/utils.py
+148
@@ -86,3 +86,151 @@ class DummyLoRAManager:
86
packed_lora = PackedLoRALayerWeights.pack(base_loras)
87
self.set_module_lora(module_name, packed_lora)
88
return packed_lora
89
+
90
+
91
+def assert_close(a, b):
92
+ rtol, atol = {
93
+ torch.float16: (6e-2, 6e-2),
94
+ torch.bfloat16: (6e-2, 6e-2),
95
+ torch.float32: (1e-2, 1e-2),
96
+ }[a.dtype]
97
+ torch.testing.assert_close(a, b, rtol=rtol, atol=atol)
98
+
99
+
100
+def ref_torch_groupgemm(
101
+ out_tensor,
102
+ inputs,
103
+ lora_weights,
104
+ lora_indices_tensor,
105
+ seq_len_tensor,
106
+ batches,
107
+ scaling,
108
+ op_type,
109
+) -> torch.Tensor:
110
+ out_list = []
111
+ current_offset = 0
112
+ for lora_index, b_length in zip(range(batches), seq_len_tensor):
113
+ input_weight = inputs[current_offset:b_length + current_offset, :]
114
+ current_offset += b_length
115
+ lora_weight = lora_weights[lora_indices_tensor[lora_index]]
116
+ result = torch.nn.functional.linear(input_weight, lora_weight)
117
+ result *= scaling
118
+ out_list.append(result)
119
+ cat_result = torch.cat(out_list, dim=0)
120
+ if op_type == "expand":
121
+ out_tensor += cat_result
122
+ else:
123
+ out_tensor.copy_(cat_result)
124
+ return
125
+
126
+
127
+def generate_data(batches, hidden_size, lora_nums, max_rank, seq_length, dtype,
128
+ op_type, device):
129
+ seq_len_tensor = torch.randint(seq_length, seq_length + 1,
130
+ (batches, )).to(device)
131
+ b_seq_start_loc = torch.cumsum(
132
+ torch.tensor([0] + seq_len_tensor[:-1].tolist(), dtype=torch.long),
133
+ dim=0,
134
+ ).to(device)
135
+ total_tokens = seq_len_tensor.sum()
136
+ if op_type == "shrink":
137
+ inputs_tensor = torch.rand((total_tokens, hidden_size),
138
+ dtype=dtype).to(device)
139
+ lora_weights = torch.rand(
140
+ (lora_nums, max_rank, hidden_size), # col-major
141
+ dtype=dtype,
142
+ ).to(device)
143
+ # shrink op need atomic_add, so output is initinized by 0
144
+ ref_out_tensor = torch.zeros((total_tokens, max_rank),
145
+ dtype=dtype,
146
+ device=inputs_tensor.device)
147
+ # NOTE shrink kernel using torch.float32 as output type
148
+ our_out_tensor = torch.zeros((total_tokens, max_rank),
149
+ dtype=torch.float32).to(device)
150
+ else:
151
+ inputs_tensor = torch.rand(
152
+ (total_tokens, max_rank),
153
+ dtype=dtype,
154
+ ).to(device)
155
+ lora_weights = torch.rand(
156
+ (lora_nums, hidden_size, max_rank), # col-major
157
+ dtype=dtype,
158
+ ).to(device)
159
+ # expand op needs to complete y+=a@lora_b, so output is
160
+ # initinized randomly
161
+ ref_out_tensor = torch.rand(
162
+ (total_tokens, hidden_size),
163
+ dtype=dtype,
164
+ ).to(device)
165
+ # Ensure the same input.
166
+ our_out_tensor = ref_out_tensor.clone()
167
+ lora_indices_tensor = torch.randint(0,
168
+ lora_nums - 1 if lora_nums > 1 else 1,
169
+ (batches, )).to(device)
170
+ indices = torch.zeros((total_tokens), dtype=torch.long).to(device)
171
+ current_offset = 0
172
+ for b_id in range(batches):
173
+ lora_index = lora_indices_tensor[b_id]
174
+ indices[current_offset:current_offset +
175
+ seq_len_tensor[b_id]].copy_(lora_index)
176
+ current_offset += seq_len_tensor[b_id].item()
177
+ return (
178
+ inputs_tensor,
179
+ lora_weights,
180
+ our_out_tensor,
181
+ ref_out_tensor,
182
+ b_seq_start_loc,
183
+ lora_indices_tensor,
184
+ seq_len_tensor,
185
+ indices,
186
+ )
187
+
188
+
189
+def generate_data_for_expand_nslices(batches, hidden_size, lora_nums, max_rank,
190
+ seq_length, dtype, nslices, device):
191
+ seq_len_tensor = torch.randint(seq_length, seq_length + 1,
192
+ (batches, )).to(device)
193
+ b_seq_start_loc = torch.cumsum(
194
+ torch.tensor([0] + seq_len_tensor[:-1].tolist(), dtype=torch.long),
195
+ dim=0,
196
+ ).to(device)
197
+ total_tokens = seq_len_tensor.sum()
198
+ inputs_tensor = torch.rand(
199
+ (total_tokens, max_rank),
200
+ dtype=dtype,
201
+ ).to(device)
202
+ lora_weights_lst = []
203
+ for _ in range(nslices):
204
+ lora_weights_lst.append(
205
+ torch.rand(
206
+ (lora_nums, hidden_size, max_rank), # col-major
207
+ dtype=dtype,
208
+ ).to(device))
209
+ # expand op needs to complete y+=a@lora_b, so output is
210
+ # initinized randomly
211
+ ref_out_tensor = torch.rand((total_tokens, hidden_size * nslices),
212
+ dtype=dtype).to(device)
213
+ # Ensure the same input.
214
+ our_out_tensor = ref_out_tensor.clone()
215
+ lora_indices_tensor = torch.randint(0,
216
+ lora_nums - 1 if lora_nums > 1 else 1,
217
+ (batches, ))
218
+ indices = torch.zeros((total_tokens), dtype=torch.long).to(device)
219
+ current_offset = 0
220
+ for b_id in range(batches):
221
+ lora_index = lora_indices_tensor[b_id]
222
+ indices[current_offset:current_offset +
223
+ seq_len_tensor[b_id]] = lora_index.item()
224
+ current_offset += seq_len_tensor[b_id].item()
225
+
226
+ lora_indices_tensor = lora_indices_tensor.to(device)
227
+ return (
228
+ inputs_tensor,
229
+ lora_weights_lst,
230
+ our_out_tensor,
231
+ ref_out_tensor,
232
+ b_seq_start_loc,
233
+ lora_indices_tensor,
234
+ seq_len_tensor,
235
+ indices,
236
+ )
vllm/_custom_ops.py
+1
-41
@@ -13,12 +13,9 @@ try:
13
except ImportError as e:
14
logger.warning("Failed to import from vllm._C with %r", e)
15
16
-with contextlib.suppress(ImportError):
17
- import vllm._moe_C
18
-
16
with contextlib.suppress(ImportError):
17
# ruff: noqa: F401
21
- import vllm._punica_C
18
+ import vllm._moe_C
19
20
21
def is_custom_op_supported(op_name: str) -> bool:
@@ -519,43 +516,6 @@ def register_graph_buffers(fa: int, handles: List[str],
516
torch.ops._C_custom_ar.register_graph_buffers(fa, handles, offsets)
517
518
522
-# punica
523
-def dispatch_bgmv(
524
- y: torch.Tensor,
525
- x: torch.Tensor,
526
- w_t_all: torch.Tensor,
527
- indicies: torch.Tensor,
528
- layer_idx: int,
529
- scale: float,
530
-) -> None:
531
- torch.ops._punica_C.dispatch_bgmv(y, x, w_t_all, indicies, layer_idx,
532
- scale)
533
-
534
-
535
-def dispatch_bgmv_low_level(
536
- y: torch.Tensor,
537
- x: torch.Tensor,
538
- w_t_all: torch.Tensor,
539
- indicies: torch.Tensor,
540
- layer_idx: int,
541
- scale: float,
542
- h_in: int,
543
- h_out: int,
544
- y_offset: int,
545
-) -> None:
546
- torch.ops._punica_C.dispatch_bgmv_low_level(
547
- y,
548
- x,
549
- w_t_all,
550
- indicies,
551
- layer_idx,
552
- scale,
553
- h_in,
554
- h_out,
555
- y_offset,
556
- )
557
-
558
-
519
# temporary fix for https://github.com/vllm-project/vllm/issues/5456
520
# TODO: remove this in v0.6.0
521
names_and_values = globals()
vllm/envs.py
-5
@@ -45,7 +45,6 @@ if TYPE_CHECKING:
45
MAX_JOBS: Optional[str] = None
46
NVCC_THREADS: Optional[str] = None
47
VLLM_USE_PRECOMPILED: bool = False
48
- VLLM_INSTALL_PUNICA_KERNELS: bool = False
48
VLLM_NO_DEPRECATION_WARNING: bool = False
49
CMAKE_BUILD_TYPE: Optional[str] = None
50
VERBOSE: bool = False
@@ -94,10 +93,6 @@ environment_variables: Dict[str, Callable[[], Any]] = {
93
"VLLM_USE_PRECOMPILED":
94
lambda: bool(os.environ.get("VLLM_USE_PRECOMPILED")),
95
97
- # If set, vllm will install Punica kernels
98
- "VLLM_INSTALL_PUNICA_KERNELS":
99
- lambda: bool(int(os.getenv("VLLM_INSTALL_PUNICA_KERNELS", "0"))),
100
-
96
# CMake build type
97
# If not set, defaults to "Debug" or "RelWithDebInfo"
98
# Available options: "Debug", "Release", "RelWithDebInfo"
vllm/lora/fully_sharded_layers.py
+80
-57
@@ -14,7 +14,6 @@ from vllm.lora.layers import (ColumnParallelLinearWithLoRA,
14
MergedQKVParallelLinearWithLora,
15
QKVParallelLinearWithLora,
16
RowParallelLinearWithLoRA)
17
-from vllm.lora.punica import bgmv, dispatch_bgmv_low_level
17
18
if TYPE_CHECKING:
19
pass
@@ -28,7 +27,7 @@ def _fully_sharded_can_replace(can_replace):
27
28
def dec(*args, **kwargs):
29
return (can_replace(*args, **kwargs)
31
- and kwargs['lora_config'].fully_sharded_loras)
30
+ and kwargs["lora_config"].fully_sharded_loras)
31
32
return dec
33
@@ -59,25 +58,30 @@ class ColumnParallelLinearWithShardedLoRA(ColumnParallelLinearWithLoRA):
58
x = x.view(-1, x.shape[-1])
59
output, out_orig_shape = output.view(-1,
60
output.shape[-1]), output.shape
62
- buffer = torch.zeros((x.shape[0], self.lora_a_stacked.shape[2]),
63
- dtype=torch.float32,
64
- device=x.device)
65
-
66
- bgmv(buffer, x, self.lora_a_stacked,
67
- self.indices[:self.indices_len[0]], 0, 1.0)
61
+ buffer = torch.zeros(
62
+ (x.shape[0], self.lora_a_stacked.shape[2]),
63
+ dtype=torch.float32,
64
+ device=x.device,
65
+ )
66
+ self.punica_wrapper.add_shrink(buffer, x, self.lora_a_stacked, 1.0)
67
buffer = tensor_model_parallel_all_gather(buffer)
69
- bgmv(output, buffer, self.lora_b_stacked,
70
- self.indices[:self.indices_len[0]], 0, 1.0)
68
+ self.punica_wrapper.add_expand(output,
69
+ buffer,
70
+ self.lora_b_stacked,
71
+ add_input=True)
72
# now have column partitioned output
72
-
73
output = output.view(*out_orig_shape)
74
return output
75
76
@classmethod
77
@_fully_sharded_can_replace
78
- def can_replace_layer(cls, source_layer: nn.Module,
79
- lora_config: LoRAConfig, packed_modules_list: List,
80
- model_config: Optional[PretrainedConfig]) -> bool:
78
+ def can_replace_layer(
79
+ cls,
80
+ source_layer: nn.Module,
81
+ lora_config: LoRAConfig,
82
+ packed_modules_list: List,
83
+ model_config: Optional[PretrainedConfig],
84
+ ) -> bool:
85
# specifying kwargs so they can be easily accessed in decorator
86
return super().can_replace_layer(
87
source_layer=source_layer,
@@ -88,14 +92,14 @@ class ColumnParallelLinearWithShardedLoRA(ColumnParallelLinearWithLoRA):
92
)
93
94
91
-def _mcp_apply(x, bias, layer):
95
+def _mcp_apply(x, bias, layer: QKVParallelLinearWithLora):
96
"""
93
- MergedColumnParallelLinearWithShardedLoRA and
94
- MergedQKVParallelLinearWithShardedLora share the same
97
+ MergedColumnParallelLinearWithShardedLoRA and
98
+ MergedQKVParallelLinearWithShardedLora share the same
99
LoRa weight application method.
100
101
The main difference is the step by shard_size for lora_b which can
98
- vary for MergedQKVParallelLinearWithShardedLora but is constant for
102
+ vary for MergedQKVParallelLinearWithShardedLora but is constant for
103
MergedColumnParallelLinearWithShardedLoRA.
104
"""
105
# expecting 2 for column parallel and 3 for qkv
@@ -104,21 +108,27 @@ def _mcp_apply(x, bias, layer):
108
109
x = x.view(-1, x.shape[-1])
110
output, out_orig_shape = output.view(-1, output.shape[-1]), output.shape
107
- buffers = torch.zeros((n, x.shape[0], layer.lora_a_stacked[0].shape[2]),
108
- dtype=torch.float32,
109
- device=x.device)
111
+ buffers = torch.zeros(
112
+ (n, x.shape[0], layer.lora_a_stacked[0].shape[2]),
113
+ dtype=torch.float32,
114
+ device=x.device,
115
+ )
116
for idx in range(n):
111
- bgmv(buffers[idx], x, layer.lora_a_stacked[idx],
112
- layer.indices[:layer.indices_len[0]], 0, 1.0)
117
+ layer.punica_wrapper.add_shrink(buffers[idx], x,
118
+ layer.lora_a_stacked[idx], 1.0)
119
120
buffers = tensor_model_parallel_all_gather(buffers)
121
left_offset = 0
122
for idx in range(n):
123
shard_size = layer.lora_b_stacked[idx].shape[2]
118
- dispatch_bgmv_low_level(output, buffers[idx],
119
- layer.lora_b_stacked[idx],
120
- layer.indices[:layer.indices_len[0]], 0, 1.0,
121
- left_offset, shard_size)
124
+ layer.punica_wrapper.add_expand_slice(
125
+ output,
126
+ buffers[idx],
127
+ layer.lora_b_stacked[idx],
128
+ left_offset,
129
+ shard_size,
130
+ add_input=True,
131
+ )
132
left_offset += shard_size
133
134
output = output.view(*out_orig_shape)
@@ -129,7 +139,7 @@ def _mcp_apply(x, bias, layer):
139
class MergedColumnParallelLinearWithShardedLoRA(
140
MergedColumnParallelLinearWithLoRA):
141
"""
132
- Differs from MergedColumnParallelLinearWithLoRA by slicing the
142
+ Differs from MergedColumnParallelLinearWithLoRA by slicing the
143
LoRA A's also.
144
145
Based on S-LoRA, slicing happens along the rank dim.
@@ -145,7 +155,8 @@ class MergedColumnParallelLinearWithShardedLoRA(
155
lora_a = [
156
lora_a[0][:,
157
output_start_idx:output_start_idx + output_shard_size],
148
- lora_a[1][:, output_start_idx:output_start_idx + output_shard_size]
158
+ lora_a[1][:,
159
+ output_start_idx:output_start_idx + output_shard_size],
160
]
161
return lora_a
162
@@ -155,9 +166,13 @@ class MergedColumnParallelLinearWithShardedLoRA(
166
167
@classmethod
168
@_fully_sharded_can_replace
158
- def can_replace_layer(cls, source_layer: nn.Module,
159
- lora_config: LoRAConfig, packed_modules_list: List,
160
- model_config: Optional[PretrainedConfig]) -> bool:
169
+ def can_replace_layer(
170
+ cls,
171
+ source_layer: nn.Module,
172
+ lora_config: LoRAConfig,
173
+ packed_modules_list: List,
174
+ model_config: Optional[PretrainedConfig],
175
+ ) -> bool:
176
# specifying kwargs so they can be easily accessed in decorator
177
return super().can_replace_layer(
178
source_layer=source_layer,
@@ -170,7 +185,7 @@ class MergedColumnParallelLinearWithShardedLoRA(
185
186
class QKVParallelLinearWithShardedLora(QKVParallelLinearWithLora):
187
"""
173
- Differs from QKVParallelLinearWithLora by slicing the
188
+ Differs from QKVParallelLinearWithLora by slicing the
189
LoRA A's also.
190
191
Based on S-LoRA, slicing happens along the rank dim.
@@ -193,14 +208,13 @@ class QKVParallelLinearWithShardedLora(QKVParallelLinearWithLora):
208
buffer = torch.zeros((x.shape[0], self.lora_a_stacked.shape[2]),
209
dtype=torch.float32,
210
device=x.device)
196
-
197
- bgmv(buffer, x, self.lora_a_stacked,
198
- self.indices[:self.indices_len[0]], 0, 1.0)
211
+ self.punica_wrapper.add_shrink(buffer, x, self.lora_a_stacked, 1.0)
212
buffer = tensor_model_parallel_all_gather(buffer)
200
- bgmv(output, buffer, self.lora_b_stacked,
201
- self.indices[:self.indices_len[0]], 0, 1.0)
213
+ self.punica_wrapper.add_expand(output,
214
+ buffer,
215
+ self.lora_b_stacked,
216
+ add_input=True)
217
# now have column partitioned output
203
-
218
output = output.view(*out_orig_shape)
219
return output
220
@@ -237,7 +251,7 @@ class MergedQKVParallelLinearWithShardedLora(MergedQKVParallelLinearWithLora):
251
lora_a = [
252
lora_a[0][:, start_idx[0]:start_idx[0] + shard_size[0]],
253
lora_a[1][:, start_idx[1]:start_idx[1] + shard_size[1]],
240
- lora_a[2][:, start_idx[2]:start_idx[2] + shard_size[2]]
254
+ lora_a[2][:, start_idx[2]:start_idx[2] + shard_size[2]],
255
]
256
return lora_a
257
@@ -247,9 +261,13 @@ class MergedQKVParallelLinearWithShardedLora(MergedQKVParallelLinearWithLora):
261
262
@classmethod
263
@_fully_sharded_can_replace
250
- def can_replace_layer(cls, source_layer: nn.Module,
251
- lora_config: LoRAConfig, packed_modules_list: List,
252
- model_config: Optional[PretrainedConfig]) -> bool:
264
+ def can_replace_layer(
265
+ cls,
266
+ source_layer: nn.Module,
267
+ lora_config: LoRAConfig,
268
+ packed_modules_list: List,
269
+ model_config: Optional[PretrainedConfig],
270
+ ) -> bool:
271
# specifying kwargs so they can be easily accessed in decorator
272
return super().can_replace_layer(
273
source_layer=source_layer,
@@ -262,11 +280,11 @@ class MergedQKVParallelLinearWithShardedLora(MergedQKVParallelLinearWithLora):
280
281
class RowParallelLinearWithShardedLoRA(RowParallelLinearWithLoRA):
282
"""
265
- Differs from RowParallelLinearWithLoRA by slicing the
283
+ Differs from RowParallelLinearWithLoRA by slicing the
284
LoRA B's also.
285
286
Based on S-LoRA, slicing happens along the output dim.
269
- This yields a combined partial sum from the row parallel base
287
+ This yields a combined partial sum from the row parallel base
288
layer and column partitioned output from the LoRA.
289
"""
290
@@ -283,11 +301,13 @@ class RowParallelLinearWithShardedLoRA(RowParallelLinearWithLoRA):
301
x = x.view(-1, x.shape[-1])
302
output, out_orig_shape = output.view(-1,
303
output.shape[-1]), output.shape
286
- buffer = torch.zeros((x.shape[0], self.lora_a_stacked.shape[2]),
287
- dtype=torch.float32,
288
- device=x.device)
289
- bgmv(buffer, x, self.lora_a_stacked,
290
- self.indices[:self.indices_len[0]], 0, 1.0)
304
+ buffer = torch.zeros(
305
+ (x.shape[0], self.lora_a_stacked.shape[2]),
306
+ dtype=torch.float32,
307
+ device=x.device,
308
+ )
309
+
310
+ self.punica_wrapper.add_shrink(buffer, x, self.lora_a_stacked, 1.0)
311
buffer = tensor_model_parallel_all_reduce(buffer)
312
313
# following S-LoRA, allows the fusing of all_gather and all_reduce
@@ -298,18 +318,21 @@ class RowParallelLinearWithShardedLoRA(RowParallelLinearWithLoRA):
318
# reduced before being used
319
shard_size = self.lora_b_stacked.shape[2]
320
start_idx = self.tp_rank * shard_size
301
- dispatch_bgmv_low_level(output, buffer, self.lora_b_stacked,
302
- self.indices[:self.indices_len[0]], 0, 1.0,
303
- start_idx, shard_size)
304
-
321
+ self.punica_wrapper.add_expand_slice(output, buffer,
322
+ self.lora_b_stacked, start_idx,
323
+ shard_size)
324
output = output.view(*out_orig_shape)
325
return output
326
327
@classmethod
328
@_fully_sharded_can_replace
310
- def can_replace_layer(cls, source_layer: nn.Module,
311
- lora_config: LoRAConfig, packed_modules_list: List,
312
- model_config: Optional[PretrainedConfig]) -> bool:
329
+ def can_replace_layer(
330
+ cls,
331
+ source_layer: nn.Module,
332
+ lora_config: LoRAConfig,
333
+ packed_modules_list: List,
334
+ model_config: Optional[PretrainedConfig],
335
+ ) -> bool:
336
# specifying kwargs so they can be easily accessed in decorator
337
return super().can_replace_layer(
338
source_layer=source_layer,
vllm/lora/layers.py
+151
-286
@@ -17,7 +17,7 @@ from vllm.distributed import (get_tensor_model_parallel_rank,
17
tensor_model_parallel_all_reduce,
18
tensor_model_parallel_gather)
19
from vllm.distributed.utils import divide
20
-from vllm.lora.punica import add_lora, add_lora_slice, bgmv
20
+from vllm.lora.punica import PunicaWrapper
21
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
22
MergedColumnParallelLinear,
23
QKVParallelLinear,
@@ -55,88 +55,17 @@ def _not_fully_sharded_can_replace(can_replace):
55
"""
56
57
def dec(*args, **kwargs):
58
- decorate = kwargs.pop('decorate') if 'decorate' in kwargs else True
59
- condition = (not kwargs['lora_config'].fully_sharded_loras
58
+ decorate = kwargs.pop("decorate") if "decorate" in kwargs else True
59
+ condition = (not kwargs["lora_config"].fully_sharded_loras
60
if decorate else True)
61
return can_replace(*args, **kwargs) and condition
62
63
return dec
64
65
66
-def _apply_lora(
67
- x: torch.Tensor,
68
- lora_a_stacked: torch.Tensor,
69
- lora_b_stacked: torch.Tensor,
70
- indices: torch.Tensor,
71
- output: torch.Tensor,
72
-):
73
- """Applies lora to each input.
74
-
75
- This method applies all loras to each input. It uses the
76
- indices vector to determine which lora yields the
77
- correct output. An index of -1 means no lora should be
78
- applied. This method adds the final lora results to the
79
- output.
80
-
81
- Input shapes:
82
- x: (batch_size, hidden_dim)
83
- lora_a_stacked: (num_loras, lora_rank, hidden_dim)
84
- lora_b_stacked: (num_loras, output_dim, lora_rank)
85
- indices: (batch_size)
86
- output: (batch_size, output_dim)
87
- """
88
- org_output = output
89
- x = x.view(-1, x.shape[-1])
90
- output = output.view(-1, output.shape[-1])
91
- indices = indices.view(-1)
92
- add_lora(output, x, lora_a_stacked, lora_b_stacked, indices, 0, 1.0)
93
- return output.view_as(org_output)
94
-
95
-
96
-def _apply_lora_packed_nslice(
97
- x: torch.Tensor,
98
- lora_a_stacked: Tuple[torch.Tensor, torch.Tensor, torch.Tensor],
99
- lora_b_stacked: Tuple[torch.Tensor, torch.Tensor, torch.Tensor],
100
- indices: torch.Tensor,
101
- output: torch.Tensor,
102
- output_slices: Tuple[int, ...],
103
-):
104
- """Applies lora to each input.
105
-
106
- This method applies all loras to each input. It uses the
107
- indices vector to determine which lora yields the
108
- correct output. An index of -1 means no lora should be
109
- applied. This method adds the final lora results to the
110
- output.
111
-
112
- This method is used for layers that are composed of multiple sublayers
113
- (slices) packed together.
114
-
115
- Input shapes:
116
- x: (batch_size, hidden_dim)
117
- lora_a_stacked: 3 element tuple of (num_loras, lora_rank, hidden_dim)
118
- lora_b_stacked: 3 element tuple of (num_loras, output_dim, lora_rank)
119
- indices: (batch_size)
120
- output: (batch_size, q_slice_size + 2*kv_slice_size)
121
- output_slices: n-1 element tuple of (slice_size...),
122
- where n is number of slices
123
- """
124
- org_output = output
125
- x = x.view(-1, x.shape[-1])
126
- output = output.view(-1, output.shape[-1])
127
- indices = indices.view(-1)
128
- offset_left = 0
129
- for slice_idx in range(len(output_slices)):
130
- add_lora_slice(output, x, lora_a_stacked[slice_idx],
131
- lora_b_stacked[slice_idx], indices, 0, 1.0, offset_left,
132
- output_slices[slice_idx])
133
- offset_left += output_slices[slice_idx]
134
- return output.view_as(org_output)
135
-
136
-
66
@dataclass
67
class LoRAMapping(AdapterMapping):
139
- pass
68
+ is_prefill: bool = False
69
70
71
class BaseLayerWithLoRA(nn.Module):
@@ -154,10 +83,11 @@ class BaseLayerWithLoRA(nn.Module):
83
...
84
85
def create_lora_weights(
157
- self,
158
- max_loras: int,
159
- lora_config: LoRAConfig,
160
- model_config: Optional[PretrainedConfig] = None) -> None:
86
+ self,
87
+ max_loras: int,
88
+ lora_config: LoRAConfig,
89
+ model_config: Optional[PretrainedConfig] = None,
90
+ ) -> None:
91
"""Initializes lora matrices."""
92
...
93
@@ -177,20 +107,18 @@ class BaseLayerWithLoRA(nn.Module):
107
108
def set_mapping(
109
self,
180
- base_indices: torch.Tensor,
181
- sampler_indices: torch.Tensor,
182
- sampler_indices_padded: torch.Tensor,
183
- embeddings_indices: torch.Tensor,
184
- long_lora_indices: torch.Tensor,
185
- indices_len: List[int],
110
+ punica_wrapper: PunicaWrapper,
111
):
187
- """Sets the mapping indices."""
188
- ...
112
+ self.punica_wrapper: PunicaWrapper = punica_wrapper
113
114
@classmethod
191
- def can_replace_layer(cls, source_layer: nn.Module,
192
- lora_config: LoRAConfig, packed_modules_list: List,
193
- model_config: Optional[PretrainedConfig]) -> bool:
115
+ def can_replace_layer(
116
+ cls,
117
+ source_layer: nn.Module,
118
+ lora_config: LoRAConfig,
119
+ packed_modules_list: List,
120
+ model_config: Optional[PretrainedConfig],
121
+ ) -> bool:
122
"""Returns True if the layer can be replaced by this LoRA layer."""
123
raise NotImplementedError
124
@@ -259,10 +187,6 @@ class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA):
187
self.lora_a_stacked.shape[0] * self.lora_a_stacked.shape[1],
188
self.lora_a_stacked.shape[2],
189
)
262
- # Lazily initialized.
263
- self.indices: torch.Tensor
264
- self.indices_len: List[int]
265
- self.embeddings_indices: torch.Tensor
190
191
def reset_lora(self, index: int):
192
self.lora_a_stacked[index] = 0
@@ -285,40 +209,27 @@ class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA):
209
if embeddings_tensor is not None:
210
self.embeddings_tensors[
211
index, :embeddings_tensor.shape[0], :embeddings_tensor.
288
- shape[1]].copy_(embeddings_tensor, non_blocking=True)
212
+ shape[1], ].copy_(embeddings_tensor, non_blocking=True)
213
if self.embeddings_slice is not None:
214
# TODO(yard1): Optimize this copy, we don't need to copy
215
# everything, just the modified part
216
embeddings = self.embeddings_tensors.view(
217
self.embeddings_tensors.shape[0] *
218
self.embeddings_tensors.shape[1],
295
- self.embeddings_tensors.shape[2]
219
+ self.embeddings_tensors.shape[2],
220
)[self.embeddings_slice[0]:self.embeddings_slice[1]]
221
assert self.embeddings_weights is not None
222
self.embeddings_weights[:embeddings.shape[0]].copy_(embeddings)
223
300
- def set_mapping(
301
- self,
302
- base_indices: torch.Tensor,
303
- sampler_indices: torch.Tensor,
304
- sampler_indices_padded: torch.Tensor,
305
- embeddings_indices: torch.Tensor,
306
- long_lora_indices: torch.Tensor,
307
- indices_len: List[int],
308
- ):
309
- self.indices = base_indices
310
- self.embeddings_indices = embeddings_indices
311
- self.indices_len = indices_len
312
-
224
def forward(self, x: torch.Tensor) -> torch.Tensor:
225
added_tokens_mask = x > self.base_layer.org_vocab_size - 1
315
- embedding_len = self.indices_len[3]
316
- indices = self.embeddings_indices[1][:embedding_len].view_as(x)
226
+ embeddings_indices = self.punica_wrapper.embeddings_indices
227
+ indices = embeddings_indices[1].view_as(x)
228
full_lora_a_embeddings = F.embedding(
229
x + indices,
230
self.lora_a_stacked_2d,
231
)
321
- indices = self.embeddings_indices[0][:embedding_len].view_as(x)
232
+ indices = embeddings_indices[0].view_as(x)
233
full_output = self.base_layer.forward(
234
x.add_(indices * added_tokens_mask))
235
@@ -329,22 +240,32 @@ class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA):
240
if full_lora_a_embeddings.ndim == 3:
241
full_lora_a_embeddings = full_lora_a_embeddings.view(
242
full_lora_a_embeddings.shape[0] *
332
- full_lora_a_embeddings.shape[1], -1)
333
- bgmv(full_output, full_lora_a_embeddings, self.lora_b_stacked,
334
- self.indices[:self.indices_len[0]], 0, 1.0)
243
+ full_lora_a_embeddings.shape[1],
244
+ -1,
245
+ )
246
+
247
+ # Embedding layer only need expand op
248
+ self.punica_wrapper.add_expand(full_output,
249
+ full_lora_a_embeddings,
250
+ self.lora_b_stacked,
251
+ add_input=True)
252
return full_output.view_as(full_output_org)
253
254
@classmethod
338
- def can_replace_layer(cls, source_layer: nn.Module,
339
- lora_config: LoRAConfig, packed_modules_list: List,
340
- model_config: Optional[PretrainedConfig]) -> bool:
255
+ def can_replace_layer(
256
+ cls,
257
+ source_layer: nn.Module,
258
+ lora_config: LoRAConfig,
259
+ packed_modules_list: List,
260
+ model_config: Optional[PretrainedConfig],
261
+ ) -> bool:
262
return type(source_layer) is VocabParallelEmbedding
263
264
265
class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
266
"""
267
LoRA on top of ColumnParallelLinear layer.
347
-
268
+
269
LoRA B is sliced for tensor parallelism.
270
"""
271
@@ -357,10 +278,11 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
278
self.device = _get_lora_device(self.base_layer)
279
280
def create_lora_weights(
360
- self,
361
- max_loras: int,
362
- lora_config: LoRAConfig,
363
- model_config: Optional[PretrainedConfig] = None) -> None:
281
+ self,
282
+ max_loras: int,
283
+ lora_config: LoRAConfig,
284
+ model_config: Optional[PretrainedConfig] = None,
285
+ ) -> None:
286
self.lora_config = lora_config
287
self.tp_size = get_tensor_model_parallel_world_size()
288
lora_a_output_size_per_partition = (
@@ -384,10 +306,6 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
306
)
307
self.output_dim = self.lora_b_stacked.shape[2]
308
387
- # lazily initialized.
388
- self.indices: torch.Tensor
389
- self.indices_len: List[int]
390
-
309
def reset_lora(self, index: int):
310
self.lora_a_stacked[index] = 0
311
self.lora_b_stacked[index] = 0
@@ -423,28 +341,11 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
341
0, :lora_b.shape[1], :lora_b.shape[0]].copy_(
342
lora_b.T, non_blocking=True)
343
426
- def set_mapping(
427
- self,
428
- base_indices: torch.Tensor,
429
- sampler_indices: torch.Tensor,
430
- sampler_indices_padded: torch.Tensor,
431
- embeddings_indices: torch.Tensor,
432
- long_lora_indices: torch.Tensor,
433
- indices_len: List[int],
434
- ):
435
- self.indices = base_indices
436
- self.indices_len = indices_len
437
-
344
def apply(self, x: torch.Tensor,
345
bias: Optional[torch.Tensor]) -> torch.Tensor:
346
output = self.base_layer.quant_method.apply(self.base_layer, x, bias)
441
- _apply_lora(
442
- x,
443
- self.lora_a_stacked,
444
- self.lora_b_stacked,
445
- self.indices[:self.indices_len[0]],
446
- output,
447
- )
347
+ self.punica_wrapper.add_lora(output, x, self.lora_a_stacked,
348
+ self.lora_b_stacked, 1.0)
349
return output
350
351
def forward(self, input_):
@@ -473,9 +374,13 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
374
375
@classmethod
376
@_not_fully_sharded_can_replace
476
- def can_replace_layer(cls, source_layer: nn.Module,
477
- lora_config: LoRAConfig, packed_modules_list: List,
478
- model_config: Optional[PretrainedConfig]) -> bool:
377
+ def can_replace_layer(
378
+ cls,
379
+ source_layer: nn.Module,
380
+ lora_config: LoRAConfig,
381
+ packed_modules_list: List,
382
+ model_config: Optional[PretrainedConfig],
383
+ ) -> bool:
384
return type(source_layer) is ColumnParallelLinear or (
385
type(source_layer) is MergedColumnParallelLinear
386
and len(packed_modules_list) == 1)
@@ -494,10 +399,11 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
399
super().__init__(base_layer)
400
401
def create_lora_weights(
497
- self,
498
- max_loras: int,
499
- lora_config: LoRAConfig,
500
- model_config: Optional[PretrainedConfig] = None) -> None:
402
+ self,
403
+ max_loras: int,
404
+ lora_config: LoRAConfig,
405
+ model_config: Optional[PretrainedConfig] = None,
406
+ ) -> None:
407
self.lora_config = lora_config
408
n_slices = 2
409
if not (len(self.base_layer.output_sizes) == n_slices
@@ -533,8 +439,6 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
439
) for _ in range(n_slices))
440
441
self.output_dim = self.lora_b_stacked[0].shape[2]
536
- # Lazily initialized.
537
- self.indices: torch.Tensor
442
443
def reset_lora(self, index: int):
444
self.lora_a_stacked[0][index] = 0
@@ -556,7 +460,8 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
460
start_idx = self.tp_rank * shard_size
461
end_idx = (self.tp_rank + 1) * shard_size
462
lora_b = [
559
- lora_b[0][:, start_idx:end_idx], lora_b[1][:, start_idx:end_idx]
463
+ lora_b[0][:, start_idx:end_idx],
464
+ lora_b[1][:, start_idx:end_idx],
465
]
466
return lora_b
467
@@ -591,34 +496,33 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
496
def apply(self, x: torch.Tensor,
497
bias: Optional[torch.Tensor]) -> torch.Tensor:
498
output = self.base_layer.quant_method.apply(self.base_layer, x, bias)
594
- _apply_lora_packed_nslice(
595
- x,
596
- self.lora_a_stacked,
597
- self.lora_b_stacked,
598
- self.indices[:self.indices_len[0]],
599
- output,
600
- (self.output_dim, self.output_dim),
601
- )
499
+ self.punica_wrapper.add_lora_packed_nslice(
500
+ output, x, self.lora_a_stacked, self.lora_b_stacked, 1.0,
501
+ (self.output_dim, self.output_dim))
502
return output
503
504
@classmethod
505
@_not_fully_sharded_can_replace
606
- def can_replace_layer(cls, source_layer: nn.Module,
607
- lora_config: LoRAConfig, packed_modules_list: List,
608
- model_config: Optional[PretrainedConfig]) -> bool:
609
- return type(source_layer) is MergedColumnParallelLinear and len(
610
- packed_modules_list) == 2
506
+ def can_replace_layer(
507
+ cls,
508
+ source_layer: nn.Module,
509
+ lora_config: LoRAConfig,
510
+ packed_modules_list: List,
511
+ model_config: Optional[PretrainedConfig],
512
+ ) -> bool:
513
+ return (type(source_layer) is MergedColumnParallelLinear
514
+ and len(packed_modules_list) == 2)
515
516
517
class QKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
518
"""
615
- ColumnParallelLinear layer that is specifically designed for
616
- qkv_proj. Certain models, such as chtglm3 and baichuan-7b,
617
- only contains a single LoRA within their qkv_proj layer.
519
+ ColumnParallelLinear layer that is specifically designed for
520
+ qkv_proj. Certain models, such as chtglm3 and baichuan-7b,
521
+ only contains a single LoRA within their qkv_proj layer.
522
619
- During inference with Tensor Parallel, the weights of lora_b
523
+ During inference with Tensor Parallel, the weights of lora_b
524
must be accurately partitioned according to the respective ranks.
621
-
525
+
526
Q slice may have different shape than K and V slices (which both have
527
the same shape).
528
"""
@@ -696,10 +600,11 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
600
super().__init__(base_layer)
601
602
def create_lora_weights(
699
- self,
700
- max_loras: int,
701
- lora_config: LoRAConfig,
702
- model_config: Optional[PretrainedConfig] = None) -> None:
603
+ self,
604
+ max_loras: int,
605
+ lora_config: LoRAConfig,
606
+ model_config: Optional[PretrainedConfig] = None,
607
+ ) -> None:
608
self.lora_config = lora_config
609
self.tp_size = get_tensor_model_parallel_world_size()
610
self.tp_rank = get_tensor_model_parallel_rank()
@@ -767,11 +672,15 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
672
),
673
)
674
770
- self.output_slices = (self.q_proj_shard_size, self.kv_proj_shard_size,
771
- self.kv_proj_shard_size)
675
+ self.output_slices = (
676
+ self.q_proj_shard_size,
677
+ self.kv_proj_shard_size,
678
+ self.kv_proj_shard_size,
679
+ )
680
self.packed_indices: Optional[torch.Tensor] = None
681
self.standard_indices: Optional[torch.Tensor] = None
682
# lazily initialized.
683
+ self.indices: torch.Tensor
684
self.indices_len: List[int]
685
686
def reset_lora(self, index: int):
@@ -794,15 +703,15 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
703
if lora_b[0] is not None:
704
lora_b_q = lora_b[0][:, self.q_proj_shard_size *
705
self.q_shard_id:self.q_proj_shard_size *
797
- (self.q_shard_id + 1)]
706
+ (self.q_shard_id + 1), ]
707
if lora_b[1] is not None:
708
lora_b_k = lora_b[1][:, self.kv_proj_shard_size *
709
self.kv_shard_id:self.kv_proj_shard_size *
801
- (self.kv_shard_id + 1)]
710
+ (self.kv_shard_id + 1), ]
711
if lora_b[2] is not None:
712
lora_b_v = lora_b[2][:, self.kv_proj_shard_size *
713
self.kv_shard_id:self.kv_proj_shard_size *
805
- (self.kv_shard_id + 1)]
714
+ (self.kv_shard_id + 1), ]
715
lora_b = [lora_b_q, lora_b_k, lora_b_v]
716
return lora_b
717
@@ -851,23 +760,23 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
760
def apply(self, x: torch.Tensor,
761
bias: Optional[torch.Tensor]) -> torch.Tensor:
762
output = self.base_layer.quant_method.apply(self.base_layer, x, bias)
854
- _apply_lora_packed_nslice(
855
- x,
856
- self.lora_a_stacked,
857
- self.lora_b_stacked,
858
- self.indices[:self.indices_len[0]],
859
- output,
860
- self.output_slices,
861
- )
763
+ self.punica_wrapper.add_lora_packed_nslice(output, x,
764
+ self.lora_a_stacked,
765
+ self.lora_b_stacked, 1.0,
766
+ self.output_slices)
767
return output
768
769
@classmethod
770
@_not_fully_sharded_can_replace
866
- def can_replace_layer(cls, source_layer: nn.Module,
867
- lora_config: LoRAConfig, packed_modules_list: List,
868
- model_config: Optional[PretrainedConfig]) -> bool:
869
- return type(source_layer) is QKVParallelLinear and len(
870
- packed_modules_list) == 3
771
+ def can_replace_layer(
772
+ cls,
773
+ source_layer: nn.Module,
774
+ lora_config: LoRAConfig,
775
+ packed_modules_list: List,
776
+ model_config: Optional[PretrainedConfig],
777
+ ) -> bool:
778
+ return (type(source_layer) is QKVParallelLinear
779
+ and len(packed_modules_list) == 3)
780
781
782
class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
@@ -880,10 +789,11 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
789
self.device = _get_lora_device(self.base_layer)
790
791
def create_lora_weights(
883
- self,
884
- max_loras: int,
885
- lora_config: LoRAConfig,
886
- model_config: Optional[PretrainedConfig] = None) -> None:
792
+ self,
793
+ max_loras: int,
794
+ lora_config: LoRAConfig,
795
+ model_config: Optional[PretrainedConfig] = None,
796
+ ) -> None:
797
self.lora_config = lora_config
798
self.tp_rank = get_tensor_model_parallel_rank()
799
self.lora_a_stacked = torch.zeros(
@@ -911,9 +821,6 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
821
dtype=lora_config.lora_dtype,
822
device=self.device,
823
)
914
- # Lazily initialized
915
- self.indices: torch.Tensor
916
- self.indices_len: List[int]
824
825
def reset_lora(self, index: int):
826
self.lora_a_stacked[index] = 0
@@ -950,27 +857,10 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
857
0, :lora_b.shape[1], :lora_b.shape[0]].copy_(
858
lora_b.T, non_blocking=True)
859
953
- def set_mapping(
954
- self,
955
- base_indices: torch.Tensor,
956
- sampler_indices: torch.Tensor,
957
- sampler_indices_padded: torch.Tensor,
958
- embeddings_indices: torch.Tensor,
959
- long_lora_indices: torch.Tensor,
960
- indices_len: List[int],
961
- ):
962
- self.indices = base_indices
963
- self.indices_len = indices_len
964
-
860
def apply(self, x: torch.Tensor) -> torch.Tensor:
861
output = self.base_layer.quant_method.apply(self.base_layer, x)
967
- _apply_lora(
968
- x,
969
- self.lora_a_stacked,
970
- self.lora_b_stacked,
971
- self.indices[:self.indices_len[0]],
972
- output,
973
- )
862
+ self.punica_wrapper.add_lora(output, x, self.lora_a_stacked,
863
+ self.lora_b_stacked, 1.0)
864
return output
865
866
def forward(self, input_):
@@ -1013,14 +903,18 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
903
904
@property
905
def weight(self):
1016
- return self.base_layer.weight if hasattr(
1017
- self.base_layer, "weight") else self.base_layer.qweight
906
+ return (self.base_layer.weight if hasattr(self.base_layer, "weight")
907
+ else self.base_layer.qweight)
908
909
@classmethod
910
@_not_fully_sharded_can_replace
1021
- def can_replace_layer(cls, source_layer: nn.Module,
1022
- lora_config: LoRAConfig, packed_modules_list: List,
1023
- model_config: Optional[PretrainedConfig]) -> bool:
911
+ def can_replace_layer(
912
+ cls,
913
+ source_layer: nn.Module,
914
+ lora_config: LoRAConfig,
915
+ packed_modules_list: List,
916
+ model_config: Optional[PretrainedConfig],
917
+ ) -> bool:
918
return type(source_layer) is RowParallelLinear
919
920
@@ -1125,10 +1019,6 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
1019
dtype=torch.long)
1020
else:
1021
self.sharded_to_full_mapping_gpu = None
1128
- # Lazily initialized.
1129
- self.indices: torch.Tensor
1130
- self.indices_len: List[int]
1131
- self.indices_padded: torch.Tensor
1022
1023
def reset_lora(self, index: int):
1024
self.lora_a_stacked[index] = 0
@@ -1154,19 +1044,6 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
1044
index, :embeddings_tensor.shape[0], :embeddings_tensor.
1045
shape[1], ] = embeddings_tensor
1046
1157
- def set_mapping(
1158
- self,
1159
- base_indices: torch.Tensor,
1160
- sampler_indices: torch.Tensor,
1161
- sampler_indices_padded: torch.Tensor,
1162
- embeddings_indices: torch.Tensor,
1163
- long_lora_indices: torch.Tensor,
1164
- indices_len: List[int],
1165
- ):
1166
- self.indices = sampler_indices
1167
- self.indices_padded = sampler_indices_padded
1168
- self.indices_len = indices_len
1169
-
1047
def _get_logits(
1048
self,
1049
hidden_states: torch.Tensor,
@@ -1212,38 +1089,37 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
1089
out=lora_logits[:-1])
1090
lora_logits[-1] = float("-inf")
1091
lora_logits = lora_logits.mT
1092
+ indices_padded = self.punica_wrapper.sampler_indices_padded
1093
lora_logits = (lora_logits.reshape(
1094
lora_logits.shape[0] * lora_logits.shape[1],
1095
lora_logits.shape[2],
1218
- ).index_select(0,
1219
- self.indices_padded[:self.indices_len[2]]).nan_to_num_(
1220
- nan=float("-inf"),
1221
- posinf=float("inf"),
1222
- neginf=float("-inf")))
1096
+ ).index_select(0, indices_padded).nan_to_num_(nan=float("-inf"),
1097
+ posinf=float("inf"),
1098
+ neginf=float("-inf")))
1099
logits[:,
1100
self.base_layer.org_vocab_size:self.base_layer.org_vocab_size +
1225
- lora_logits.shape[1]] = lora_logits
1226
-
1227
- _apply_lora(
1228
- hidden_states,
1229
- self.lora_a_stacked,
1230
- self.lora_b_stacked,
1231
- self.indices[:self.indices_len[1]],
1232
- logits,
1233
- )
1101
+ lora_logits.shape[1], ] = lora_logits
1102
+
1103
+ # LogitsProcessorWithLoRA always using bgmv
1104
+ self.punica_wrapper.add_lora_logits(logits, hidden_states,
1105
+ self.lora_a_stacked,
1106
+ self.lora_b_stacked, 1.0)
1107
1108
# Remove paddings in vocab (if any).
1109
logits = logits[:, :self.base_layer.vocab_size]
1237
-
1110
return logits
1111
1112
def forward(self, *args, **kwargs):
1113
return type(self.base_layer).forward(self, *args, **kwargs)
1114
1115
@classmethod
1244
- def can_replace_layer(cls, source_layer: nn.Module,
1245
- lora_config: LoRAConfig, packed_modules_list: List,
1246
- model_config: Optional[PretrainedConfig]) -> bool:
1116
+ def can_replace_layer(
1117
+ cls,
1118
+ source_layer: nn.Module,
1119
+ lora_config: LoRAConfig,
1120
+ packed_modules_list: List,
1121
+ model_config: Optional[PretrainedConfig],
1122
+ ) -> bool:
1123
# Special handling for the LogitsProcessor.
1124
return False
1125
@@ -1259,9 +1135,6 @@ class LinearScalingRotaryEmbeddingWithLora(BaseLayerWithLoRA):
1135
def __init__(self, base_layer: RotaryEmbedding) -> None:
1136
super().__init__()
1137
self.base_layer = base_layer
1262
- # Lazily initialized
1263
- self.long_lora_indices: torch.Tensor
1264
- self.indices_len: List[int]
1138
1139
@property
1140
def scaling_factors(self):
@@ -1277,9 +1150,8 @@ class LinearScalingRotaryEmbeddingWithLora(BaseLayerWithLoRA):
1150
lora_config: LoRAConfig,
1151
model_config: Optional[PretrainedConfig] = None,
1152
) -> None:
1280
- scaling_factors = list(
1281
- lora_config.long_lora_scaling_factors
1282
- ) if lora_config.long_lora_scaling_factors else []
1153
+ scaling_factors = (list(lora_config.long_lora_scaling_factors)
1154
+ if lora_config.long_lora_scaling_factors else [])
1155
base_scaling_factor = (self.base_layer.scaling_factor if isinstance(
1156
self.base_layer, LinearScalingRotaryEmbedding) else 1.0)
1157
scaling_factors = sorted(
@@ -1306,18 +1178,6 @@ class LinearScalingRotaryEmbeddingWithLora(BaseLayerWithLoRA):
1178
):
1179
...
1180
1309
- def set_mapping(
1310
- self,
1311
- base_indices: torch.Tensor,
1312
- sampler_indices: torch.Tensor,
1313
- sampler_indices_padded: torch.Tensor,
1314
- embeddings_indices: torch.Tensor,
1315
- long_lora_indices: torch.Tensor,
1316
- indices_len: List[int],
1317
- ):
1318
- self.long_lora_indices = long_lora_indices
1319
- self.indices_len = indices_len
1320
-
1181
def forward(
1182
self,
1183
positions: torch.Tensor,
@@ -1328,19 +1188,24 @@ class LinearScalingRotaryEmbeddingWithLora(BaseLayerWithLoRA):
1188
positions,
1189
query,
1190
key,
1331
- offsets=self.long_lora_indices[:self.indices_len[4]])
1191
+ offsets=self.punica_wrapper.long_lora_indices,
1192
+ )
1193
1194
@property
1195
def scaling_factor_to_offset(self) -> Dict[float, int]:
1196
return self.base_layer.scaling_factor_to_offset
1197
1198
@classmethod
1338
- def can_replace_layer(cls, source_layer: nn.Module,
1339
- lora_config: LoRAConfig, packed_modules_list: List,
1340
- model_config: Optional[PretrainedConfig]) -> bool:
1199
+ def can_replace_layer(
1200
+ cls,
1201
+ source_layer: nn.Module,
1202
+ lora_config: LoRAConfig,
1203
+ packed_modules_list: List,
1204
+ model_config: Optional[PretrainedConfig],
1205
+ ) -> bool:
1206
"""Returns True if the layer can be replaced by this LoRA layer."""
1342
- return type(source_layer) is LinearScalingRotaryEmbedding or type(
1343
- source_layer) is RotaryEmbedding
1207
+ return (type(source_layer) is LinearScalingRotaryEmbedding
1208
+ or type(source_layer) is RotaryEmbedding)
1209
1210
def extra_repr(self) -> str:
1211
return self.base_layer.extra_repr()
vllm/lora/models.py
+16
-155
@@ -4,7 +4,7 @@ import math
4
import os
5
import re
6
from dataclasses import dataclass, field
7
-from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union
7
+from typing import Any, Callable, Dict, List, Optional, Type
8
9
import safetensors.torch
10
import torch
@@ -21,6 +21,7 @@ from vllm.lora.layers import (BaseLayerWithLoRA,
21
LinearScalingRotaryEmbeddingWithLora,
22
LoRAMapping)
23
from vllm.lora.lora import LoRALayerWeights, PackedLoRALayerWeights
24
+from vllm.lora.punica import PunicaWrapper
25
from vllm.lora.utils import (from_layer, from_layer_logits_processor,
26
parse_fine_tuned_lora_name, replace_submodule)
27
from vllm.model_executor.models.interfaces import SupportsLoRA
@@ -43,115 +44,6 @@ class LongContextLoRAContext:
44
offsets_by_lora_id: Dict[int, int] = field(default_factory=dict)
45
46
46
-def convert_mapping(
47
- mapping: LoRAMapping,
48
- lora_index_to_id: List[Optional[int]],
49
- max_loras: int,
50
- vocab_size: int,
51
- extra_vocab_size: int,
52
- long_lora_context: Optional[LongContextLoRAContext] = None,
53
-) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor,
54
- Optional[torch.Tensor], List[int]]:
55
- """Converts LoRAMapping to index tensors.
56
-
57
- Args:
58
- mapping: LoRAMapping mapping rows in a batch to LoRA ids.
59
- lora_index_to_id: List mapping LoRA ids to LoRA indices.
60
- max_loras: Maximum number of LoRAs.
61
- vocab_size: Model vocab size.
62
- extra_vocab_size: Extra vocab size each LoRA can have.
63
- long_lora_context: Passed if there are long context lora in a batch.
64
-
65
- Returns:
66
- A tuple of tensors:
67
- base_indices: Tensor of shape [batch_size] mapping batch rows to
68
- LoRA indices.
69
- sampler_indices: Tensor of shape [batch_size] mapping requests to
70
- LoRA indices for sampler. For generation, this will be the
71
- same as base_indicies. For prefill, this will map requests
72
- to LoRA indices.
73
- sampler_indices_padded: Tensor of shape [batch_size] mapping
74
- requests to LoRA indices for sampler with padding.
75
- Same as sampler_indicies, but -1 is replaced with
76
- max_loras.
77
- embeddings_indices: Tensor of shape [2, batch_size] mapping
78
- requests to embedding indices. First row is for embeddings
79
- added by the LoRAs, second row is for the LoRA.lora_a
80
- embeddings.
81
- long_lora_indices: Tensor of shape [batch_size] mapping
82
- requests to RoPE offsets and rot dims for long LoRAs.
83
- None if long context lora doesn't exist.
84
- indices_len: List of lengths of the above tensors.
85
- Used to index into each tensor. It contains length for
86
- (base_indices, sampler_indices, sampler_indices_padded,
87
- embeddings_indices, long_lora_indices). If long_lora doesn't
88
- exist, it only contains first 4 entries.
89
- """
90
- index_mapping_indices: List[int] = list(mapping.index_mapping).copy()
91
- embedding_indices = index_mapping_indices.copy()
92
- lora_indices = index_mapping_indices.copy()
93
- long_lora_offsets: Optional[torch.Tensor] = None
94
- if long_lora_context:
95
- long_lora_offsets = torch.zeros(len(index_mapping_indices),
96
- device="cuda",
97
- dtype=torch.long)
98
- prompt_mapping: List[int] = [
99
- lora_index_to_id.index(x) if x > 0 else -1
100
- for x in mapping.prompt_mapping
101
- ]
102
- lora_idx = None
103
- for i in range(len(index_mapping_indices)):
104
- # TODO index can be slow. optimize
105
- lora_idx = (lora_index_to_id.index(index_mapping_indices[i])
106
- if index_mapping_indices[i] > 0 else -1)
107
- embedding_indices[i] = lora_idx if index_mapping_indices[i] > 0 else 0
108
- lora_indices[i] = lora_idx
109
- if long_lora_context:
110
- assert long_lora_offsets is not None
111
- lora_offset: int = long_lora_context.offsets_by_lora_id.get(
112
- index_mapping_indices[i], 0)
113
- long_lora_offsets[i] = lora_offset
114
-
115
- indices_list: List[Union[List[int], torch.Tensor]] = [
116
- index_mapping_indices, lora_indices, embedding_indices
117
- ]
118
- if long_lora_context:
119
- assert long_lora_offsets is not None
120
- indices_list.append(long_lora_offsets)
121
- indices = torch.tensor(indices_list, dtype=torch.long, device="cuda")
122
- prompt_mapping_tensor = torch.tensor(prompt_mapping,
123
- device="cuda",
124
- dtype=torch.long)
125
- embeddings_indices = torch.stack([
126
- indices[2] * extra_vocab_size,
127
- indices[2] * (vocab_size + extra_vocab_size)
128
- ])
129
- embeddings_indices[embeddings_indices == -1] = max_loras - 1
130
- base_indices = indices[1]
131
- sampler_indices = prompt_mapping_tensor
132
- sampler_indices_padded = sampler_indices.clone()
133
- sampler_indices_padded[sampler_indices_padded == -1] = max_loras - 1
134
- sampler_indices_padded = (
135
- torch.arange(
136
- 0, len(sampler_indices_padded), device="cuda", dtype=torch.long) +
137
- (sampler_indices_padded * len(sampler_indices_padded)))
138
- long_lora_indices = None
139
- long_lora_indices_len: Optional[int] = None
140
- if long_lora_context:
141
- long_lora_indices = indices[3]
142
- long_lora_indices_len = long_lora_indices.shape[-1]
143
- # Contain length of indices tensors. Used to index into each tensor.
144
- indices_len = [
145
- base_indices.shape[-1], sampler_indices.shape[-1],
146
- sampler_indices_padded.shape[-1], embeddings_indices.shape[-1]
147
- ]
148
- if long_lora_indices_len is not None:
149
- indices_len.append(long_lora_indices_len)
150
-
151
- return (base_indices, sampler_indices, sampler_indices_padded,
152
- embeddings_indices, long_lora_indices, indices_len)
153
-
154
-
47
def get_lora_id():
48
global _GLOBAL_LORA_ID
49
_GLOBAL_LORA_ID += 1
@@ -422,29 +314,12 @@ class LoRAModelManager(AdapterModelManager):
314
self.lora_index_to_id: List[Optional[int]] = [None] * self.lora_slots
315
self.vocab_size = vocab_size
316
self.long_lora_context: Optional[LongContextLoRAContext] = None
425
- self.base_indices = torch.empty(self.max_num_batched_tokens,
426
- dtype=torch.long,
427
- device="cuda")
428
- self.sampler_indices = torch.empty(self.max_num_batched_tokens,
429
- dtype=torch.long,
430
- device="cuda")
431
- self.sampler_indices_padded = torch.empty(self.max_num_batched_tokens,
432
- dtype=torch.long,
433
- device="cuda")
434
- self.embeddings_indices = torch.empty(2,
435
- self.max_num_batched_tokens,
436
- dtype=torch.long,
437
- device="cuda")
438
- self.long_lora_indices = torch.empty(self.max_num_batched_tokens,
439
- dtype=torch.long,
440
- device="cuda")
317
+ self.punica_wrapper = PunicaWrapper(max_num_batched_tokens,
318
+ max_batches=self.max_num_seqs,
319
+ device="cuda")
320
# Scaling factor -> offset to the sin_cos_cache to it.
321
# Used for long context lora.
322
self.scaling_factor_to_offset: Dict[float, int] = {}
444
- # 4 is the number of indicies tensors defined above
445
- # base_indices, sampler_indices, sampler_indices_padded,
446
- # embeddings_indices
447
- self.indices_len: List[Optional[int]] = [None] * 4
323
super().__init__(model)
324
if hasattr(self.model, "supported_lora_modules"):
325
self.supported_lora_modules = copy.deepcopy(
@@ -536,28 +411,16 @@ class LoRAModelManager(AdapterModelManager):
411
"Pinning is not supported in LoRAModelManager."
412
"Use LRUCacheLoRAModelManager for pinning") # type: ignore
413
539
- # TODO see if this can be vectorized
414
def _set_adapter_mapping(self, mapping: LoRAMapping) -> None:
541
- (base_indices, sampler_indices, sampler_indices_padded,
542
- embeddings_indices, long_lora_offsets_tensor,
543
- indices_len) = convert_mapping(mapping, self.lora_index_to_id,
544
- self.lora_slots + 1, self.vocab_size,
545
- self.lora_config.lora_extra_vocab_size,
546
- self.long_lora_context)
547
- self.base_indices[:base_indices.shape[0]].copy_(base_indices)
548
- self.sampler_indices[:sampler_indices.shape[0]].copy_(sampler_indices)
549
- self.sampler_indices_padded[:sampler_indices_padded.shape[0]].copy_(
550
- sampler_indices_padded)
551
- self.embeddings_indices[:embeddings_indices.
552
- shape[0], :embeddings_indices.shape[1]].copy_(
553
- embeddings_indices)
554
- if long_lora_offsets_tensor is not None:
555
- self.long_lora_indices[:long_lora_offsets_tensor.shape[0]].copy_(
556
- long_lora_offsets_tensor)
557
- else:
558
- self.long_lora_indices.zero_()
559
- # Maintain the reference
560
- self.indices_len[:] = indices_len
415
+ # update lora states
416
+ self.punica_wrapper.update_metadata(
417
+ mapping,
418
+ self.lora_index_to_id,
419
+ self.lora_slots + 1,
420
+ self.vocab_size,
421
+ self.lora_config.lora_extra_vocab_size,
422
+ self.long_lora_context,
423
+ )
424
425
def remove_all_adapters(self):
426
"""Remove all LoRAModels from the manager."""
@@ -595,10 +458,8 @@ class LoRAModelManager(AdapterModelManager):
458
self.model.config))
459
self.register_module(module_name, new_module)
460
self._register_packed_modules(module_name)
598
- new_module.set_mapping(self.base_indices, self.sampler_indices,
599
- self.sampler_indices_padded,
600
- self.embeddings_indices,
601
- self.long_lora_indices, self.indices_len)
461
+ # All lora layers share the same punica_wrapper based on reference.
462
+ new_module.set_mapping(self.punica_wrapper)
463
464
def register_module(self, module_name: str, module: "BaseLayerWithLoRA"):
465
assert isinstance(module, BaseLayerWithLoRA)
vllm/lora/ops/__init__.py
vllm/lora/ops/bgmv_expand.py
new
+169
@@ -0,0 +1,169 @@
1
+"""
2
+Based on:
3
+Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023).
4
+Punica: Multi-Tenant LoRA Serving.
5
+https://arxiv.org/abs/2310.18547
6
+"""
7
+
8
+from typing import Dict, Optional
9
+
10
+import torch
11
+import triton
12
+import triton.language as tl
13
+
14
+from .utils import get_lora_op_configs
15
+
16
+
17
+@triton.jit
18
+def _bgmv_expand_kernel(
19
+ input_ptr,
20
+ lora_ptr,
21
+ out_ptr,
22
+ N,
23
+ K,
24
+ lora_indices,
25
+ xm_stride,
26
+ xk_stride,
27
+ l0_stride,
28
+ lora_k_stride,
29
+ lora_n_stride,
30
+ cm_stride,
31
+ cn_stride,
32
+ BLOCK_N: tl.constexpr,
33
+ BLOCK_K: tl.constexpr,
34
+ SPLIT_N: tl.constexpr,
35
+ EVEN_K: tl.constexpr,
36
+ ADD_INPUTS: tl.constexpr,
37
+ CAST_TYPE: tl.constexpr,
38
+):
39
+ """
40
+ GroupGEMV, additionally, introducing SPLIT_N can improve large hidden_size's
41
+ performance
42
+ """
43
+ pid_sn = tl.program_id(axis=0)
44
+ cur_batch = tl.program_id(axis=1)
45
+ lora_index = tl.load(lora_indices + cur_batch)
46
+ if lora_index == -1:
47
+ return
48
+ offset_k = tl.arange(0, BLOCK_K)
49
+ offset_n = tl.arange(0, BLOCK_N)
50
+ if EVEN_K:
51
+ tiled_a = tl.load(input_ptr + cur_batch * xm_stride +
52
+ offset_k * xk_stride, ) # [BLOCK_K]
53
+ else:
54
+ tiled_a = tl.load(
55
+ input_ptr + cur_batch * xm_stride + offset_k * xk_stride,
56
+ mask=offset_k < K,
57
+ other=0,
58
+ ) # [BLOCK_K]
59
+ # N must be divisible by SPLIT_N
60
+ split_n_length = tl.cdiv(N, SPLIT_N)
61
+ if CAST_TYPE:
62
+ tiled_a = tiled_a.to(lora_ptr.dtype.element_ty)
63
+ # sliding to next row-block
64
+ b_ptr = (lora_ptr + l0_stride * lora_index +
65
+ pid_sn * split_n_length * lora_k_stride)
66
+ c_ptr = out_ptr + cur_batch * cm_stride + pid_sn * split_n_length
67
+ for n in range(0, split_n_length, BLOCK_N):
68
+ current_n = n + offset_n
69
+ current_n_c = tl.max_contiguous(current_n, BLOCK_N)
70
+ b_ptr_mask = (current_n[:, None] < split_n_length) & (offset_k[None, :]
71
+ < K)
72
+ c_mask = current_n < split_n_length
73
+ tiled_b = tl.load(
74
+ b_ptr + current_n_c[:, None] * lora_k_stride +
75
+ offset_k[None, :] * lora_n_stride,
76
+ mask=b_ptr_mask,
77
+ other=0.0,
78
+ ) # [BLOCK_N,BLOCK_K]
79
+ if ADD_INPUTS:
80
+ tiled_out = tl.load(c_ptr + current_n * cn_stride, mask=c_mask)
81
+ accumulator = tl.sum(tiled_a * tiled_b, 1) + tiled_out
82
+ else:
83
+ accumulator = tl.sum(tiled_a * tiled_b, 1)
84
+
85
+ tl.store(c_ptr + current_n * cn_stride, accumulator, mask=c_mask)
86
+
87
+
88
+@torch.inference_mode()
89
+def bgmv_expand(
90
+ inputs: torch.Tensor,
91
+ lora_b_weights: torch.Tensor,
92
+ output_tensor: torch.Tensor,
93
+ lora_indices_tensor: torch.Tensor,
94
+ add_inputs: bool = True,
95
+ override_config: Optional[Dict[str, int]] = None,
96
+):
97
+ """
98
+ Args:
99
+ inputs (torch.Tensor): input tensor
100
+ lora_b_weights (torch.Tensor): lora'a weight
101
+ output_tensor (torch.Tensor): output tensor
102
+ lora_indices_tensor (torch.Tensor): (batch_size,). The LoRA index
103
+ corresponding to each batch, An index of -1 means no lora should be
104
+ applied.
105
+ batches (int): batch size
106
+ add_inputs (bool, optional): Defaults to False. adds the final lora
107
+ results to the output.
108
+ override_config (Optional[Dict[str, int]], optional): Defaults to None.
109
+ Triton grid config
110
+ """
111
+
112
+ assert inputs.dtype in [torch.float16, torch.bfloat16, torch.float32]
113
+ assert lora_b_weights.dtype in [
114
+ torch.float16,
115
+ torch.bfloat16,
116
+ ]
117
+ assert inputs.size(1) == lora_b_weights.size(-1)
118
+
119
+ assert inputs.is_contiguous()
120
+ assert output_tensor.is_contiguous()
121
+
122
+ if lora_b_weights.ndim == 4: # shape:(lora_num,1,size,rank)
123
+ assert lora_b_weights.size(1) == 1
124
+ lora_b_weights = lora_b_weights.squeeze(dim=1)
125
+ else:
126
+ assert lora_b_weights.ndim == 3 # shape:(lora_num,size,rank)
127
+ assert lora_b_weights.is_contiguous()
128
+
129
+ # TODO tuning this config
130
+ N, K = lora_b_weights.shape[-2:] # K= rank,N=hidden_size
131
+ BLOCK_K = triton.next_power_of_2(K)
132
+ EVEN_K = K % BLOCK_K == 0
133
+ ADD_INPUTS = add_inputs
134
+ CAST_TYPE = False
135
+ if inputs.dtype == torch.float32 and lora_b_weights.dtype in [
136
+ torch.float16,
137
+ torch.bfloat16,
138
+ ]:
139
+ CAST_TYPE = True
140
+ batches = lora_indices_tensor.size(0)
141
+ if override_config:
142
+ config = override_config
143
+ else:
144
+ config = get_lora_op_configs("expand", batches, N)
145
+ grid = lambda META: (
146
+ META["SPLIT_N"],
147
+ batches,
148
+ )
149
+ _bgmv_expand_kernel[grid](
150
+ inputs,
151
+ lora_b_weights,
152
+ output_tensor,
153
+ N,
154
+ K,
155
+ lora_indices_tensor,
156
+ inputs.stride(0),
157
+ inputs.stride(1),
158
+ lora_b_weights.stride(0),
159
+ lora_b_weights.stride(1),
160
+ lora_b_weights.stride(2),
161
+ output_tensor.stride(0),
162
+ output_tensor.stride(1),
163
+ BLOCK_K=BLOCK_K,
164
+ EVEN_K=EVEN_K,
165
+ ADD_INPUTS=ADD_INPUTS,
166
+ CAST_TYPE=CAST_TYPE,
167
+ **config,
168
+ )
169
+ return
vllm/lora/ops/bgmv_expand_slice.py
new
+182
@@ -0,0 +1,182 @@
1
+"""
2
+Based on:
3
+Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023).
4
+Punica: Multi-Tenant LoRA Serving.
5
+https://arxiv.org/abs/2310.18547
6
+"""
7
+
8
+from typing import Dict, Optional
9
+
10
+import torch
11
+import triton
12
+import triton.language as tl
13
+
14
+from .utils import get_lora_op_configs
15
+
16
+
17
+@triton.jit
18
+def _bgmv_expand_slice_kernel(
19
+ input_ptr,
20
+ lora_ptr,
21
+ out_ptr,
22
+ N,
23
+ K,
24
+ lora_indices,
25
+ xm_stride,
26
+ xk_stride,
27
+ l0_stride,
28
+ lora_k_stride,
29
+ lora_n_stride,
30
+ cm_stride,
31
+ cn_stride,
32
+ slice_offset,
33
+ BLOCK_N: tl.constexpr,
34
+ BLOCK_K: tl.constexpr,
35
+ SPLIT_N: tl.constexpr,
36
+ EVEN_K: tl.constexpr,
37
+ ADD_INPUTS: tl.constexpr,
38
+ CAST_TYPE: tl.constexpr,
39
+):
40
+ """
41
+ GroupGEMV, additionally, introducing SPLIT_N can improve large hidden_size's
42
+ performance
43
+ """
44
+ pid_sn = tl.program_id(axis=0)
45
+ cur_batch = tl.program_id(axis=1)
46
+ lora_index = tl.load(lora_indices + cur_batch)
47
+ if lora_index == -1:
48
+ return
49
+ offset_k = tl.arange(0, BLOCK_K)
50
+ offset_n = tl.arange(0, BLOCK_N)
51
+ if EVEN_K:
52
+ tiled_a = tl.load(input_ptr + cur_batch * xm_stride +
53
+ offset_k * xk_stride, ) # [BLOCK_K]
54
+ else:
55
+ tiled_a = tl.load(
56
+ input_ptr + cur_batch * xm_stride + offset_k * xk_stride,
57
+ mask=offset_k < K,
58
+ other=0,
59
+ ) # [BLOCK_K]
60
+ # N must be divisible by SPLIT_N
61
+ split_n_length = tl.cdiv(N, SPLIT_N)
62
+ if CAST_TYPE:
63
+ tiled_a = tiled_a.to(lora_ptr.dtype.element_ty)
64
+ # sliding to next row-block
65
+ b_ptr = (lora_ptr + l0_stride * lora_index +
66
+ pid_sn * split_n_length * lora_k_stride)
67
+ c_ptr = (out_ptr + cur_batch * cm_stride + pid_sn * split_n_length +
68
+ slice_offset * cn_stride)
69
+
70
+ for n in range(0, split_n_length, BLOCK_N):
71
+ current_n = n + offset_n
72
+ b_ptr_mask = (current_n[:, None] < split_n_length) & (offset_k[None, :]
73
+ < K)
74
+ c_mask = current_n < split_n_length
75
+ tiled_b = tl.load(
76
+ b_ptr + current_n[:, None] * lora_k_stride +
77
+ offset_k[None, :] * lora_n_stride,
78
+ mask=b_ptr_mask,
79
+ other=0.0,
80
+ ) # [BLOCK_N,BLOCK_K]
81
+
82
+ if ADD_INPUTS:
83
+ tiled_out = tl.load(c_ptr + current_n * cn_stride, mask=c_mask)
84
+ accumulator = tl.sum(tiled_a * tiled_b, 1) + tiled_out
85
+ else:
86
+ accumulator = tl.sum(tiled_a * tiled_b, 1)
87
+
88
+ tl.store(c_ptr + current_n * cn_stride, accumulator, mask=c_mask)
89
+
90
+
91
+@torch.inference_mode()
92
+def bgmv_expand_slice(
93
+ inputs: torch.Tensor,
94
+ lora_b_weights: torch.Tensor,
95
+ output_tensor: torch.Tensor,
96
+ lora_indices_tensor: torch.Tensor,
97
+ slice_offset: int,
98
+ slice_size: int,
99
+ add_inputs: bool = True,
100
+ override_config: Optional[Dict[str, int]] = None,
101
+):
102
+ """
103
+ Args:
104
+ inputs (torch.Tensor): input tensor
105
+ lora_b_weights (torch.Tensor): lora'b weight
106
+ output_tensor (torch.Tensor): output tensor
107
+ lora_indices_tensor (torch.Tensor): (batch_size,). The LoRA index
108
+ corresponding to each batch, An index of -1 means no lora should be
109
+ applied.
110
+ slice_offst (int): output_tensor's offst
111
+ slice_size (int): current output_tensor's size
112
+ batches (int): batch size
113
+ add_inputs (bool, optional): Defaults to False.
114
+ override_config (Optional[Dict[str, int]], optional): Defaults to None.
115
+ Triton grid config
116
+ """
117
+
118
+ assert inputs.dtype in [torch.float16, torch.bfloat16, torch.float32]
119
+ assert lora_b_weights.dtype in [
120
+ torch.float16,
121
+ torch.bfloat16,
122
+ ]
123
+ assert inputs.size(1) == lora_b_weights.size(-1)
124
+
125
+ assert slice_size == lora_b_weights.size(-2)
126
+ assert inputs.is_contiguous()
127
+ assert output_tensor.is_contiguous()
128
+
129
+ if lora_b_weights.ndim == 4: # shape:(lora_num,1,size,rank)
130
+ assert lora_b_weights.size(1) == 1
131
+ lora_b_weights = lora_b_weights.squeeze(dim=1)
132
+ else:
133
+ assert lora_b_weights.ndim == 3 # shape:(lora_num,size,rank)
134
+
135
+ assert lora_b_weights.is_contiguous()
136
+
137
+ # TODO tuning this config
138
+
139
+ N, K = lora_b_weights.shape[-2:] # K= rank,N=hidden_size
140
+ BLOCK_K = triton.next_power_of_2(K)
141
+ EVEN_K = K % BLOCK_K == 0
142
+ ADD_INPUTS = add_inputs
143
+ CAST_TYPE = False
144
+ if inputs.dtype == torch.float32 and lora_b_weights.dtype in [
145
+ torch.float16,
146
+ torch.bfloat16,
147
+ ]:
148
+ CAST_TYPE = True
149
+
150
+ batches = lora_indices_tensor.size(0)
151
+
152
+ if override_config:
153
+ config = override_config
154
+ else:
155
+ config = get_lora_op_configs("expand", batches, N)
156
+
157
+ grid = lambda META: (
158
+ META["SPLIT_N"],
159
+ batches,
160
+ )
161
+ _bgmv_expand_slice_kernel[grid](
162
+ inputs,
163
+ lora_b_weights,
164
+ output_tensor,
165
+ N,
166
+ K,
167
+ lora_indices_tensor,
168
+ inputs.stride(0),
169
+ inputs.stride(1),
170
+ lora_b_weights.stride(0),
171
+ lora_b_weights.stride(1),
172
+ lora_b_weights.stride(2),
173
+ output_tensor.stride(0),
174
+ output_tensor.stride(1),
175
+ slice_offset,
176
+ BLOCK_K=BLOCK_K,
177
+ EVEN_K=EVEN_K,
178
+ ADD_INPUTS=ADD_INPUTS,
179
+ CAST_TYPE=CAST_TYPE,
180
+ **config,
181
+ )
182
+ return
vllm/lora/ops/bgmv_shrink.py
new
+150
@@ -0,0 +1,150 @@
1
+"""
2
+Based on:
3
+Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023).
4
+Punica: Multi-Tenant LoRA Serving.
5
+https://arxiv.org/abs/2310.18547
6
+"""
7
+
8
+from typing import Dict, Optional
9
+
10
+import torch
11
+import triton
12
+import triton.language as tl
13
+
14
+from .utils import get_lora_op_configs
15
+
16
+
17
+@triton.jit
18
+def _bgmv_shrink_kernel(
19
+ input_ptr,
20
+ lora_ptr,
21
+ out_ptr,
22
+ N,
23
+ K,
24
+ lora_indices,
25
+ scaling,
26
+ xm_stride,
27
+ xk_stride,
28
+ l0_stride,
29
+ lora_k_stride,
30
+ lora_n_stride,
31
+ cm_stride,
32
+ cn_stride,
33
+ BLOCK_N: tl.constexpr,
34
+ BLOCK_K: tl.constexpr,
35
+ SPLIT_K: tl.constexpr,
36
+):
37
+ """
38
+ GroupGEMV, additionally, introducing SPLIT-K can improve large hidden_size's
39
+ performance
40
+ """
41
+ pid_sk = tl.program_id(axis=0)
42
+ cur_batch = tl.program_id(axis=1)
43
+ lora_index = tl.load(lora_indices + cur_batch)
44
+ if lora_index == -1:
45
+ return
46
+
47
+ offset_n = tl.arange(0, BLOCK_N)
48
+ offset_k = tl.arange(0, BLOCK_K) + pid_sk * BLOCK_K
49
+ a_ptr = input_ptr + cur_batch * xm_stride
50
+ b_ptr = lora_ptr + l0_stride * lora_index
51
+ accumulator = tl.zeros((BLOCK_N, ), dtype=tl.float32)
52
+ for k in range(0, K, BLOCK_K * SPLIT_K):
53
+ current_k = k + offset_k
54
+ current_k_c = tl.max_contiguous(current_k, BLOCK_K)
55
+ tiled_a = tl.load(
56
+ a_ptr + current_k_c,
57
+ mask=current_k < K,
58
+ other=0.0,
59
+ ) # [BLOCK_K]
60
+ b_ptr_mask = (offset_n[:, None] < N) & (current_k[None, :] < K)
61
+
62
+ tiled_b = tl.load(
63
+ b_ptr + offset_n[:, None] * lora_k_stride +
64
+ current_k[None, :] * lora_n_stride,
65
+ mask=b_ptr_mask,
66
+ other=0.0,
67
+ ) # [BLOCK_N,BLOCK_K]
68
+
69
+ accumulator += tl.sum(tiled_a * tiled_b, 1)
70
+ accumulator *= scaling
71
+ offset_cn = tl.arange(0, BLOCK_N)
72
+ c_ptr = out_ptr + cur_batch * cm_stride + offset_cn * cn_stride
73
+ c_mask = offset_cn < N
74
+ if SPLIT_K == 1:
75
+ tl.store(c_ptr, accumulator, mask=c_mask)
76
+ else:
77
+ tl.atomic_add(c_ptr, accumulator, mask=c_mask)
78
+
79
+
80
+@torch.inference_mode()
81
+def bgmv_shrink(
82
+ inputs: torch.Tensor,
83
+ lora_a_weights: torch.Tensor,
84
+ output_tensor: torch.Tensor,
85
+ lora_indices_tensor: torch.Tensor,
86
+ scaling: float = 1.0,
87
+ override_config: Optional[Dict[str, int]] = None,
88
+):
89
+ """
90
+ Args:
91
+ inputs (torch.Tensor): input tensor
92
+ lora_a_weights (torch.Tensor): lora'a weight
93
+ output_tensor (torch.Tensor): output tensor
94
+ lora_indices_tensor (torch.Tensor): (batch_size,). The LoRA index
95
+ corresponding to each batch. An index of -1 means no lora should be
96
+ applied.
97
+ batches (int): batch size
98
+ scaling (float): Scaling factor.
99
+ override_config (Optional[Dict[str, int]], optional): Defaults to None.
100
+ Triton grid config
101
+ """
102
+ assert inputs.dtype == lora_a_weights.dtype
103
+ assert inputs.dtype in [torch.float16, torch.bfloat16]
104
+ assert lora_a_weights.dtype in [
105
+ torch.float16,
106
+ torch.bfloat16,
107
+ ]
108
+ assert inputs.size(1) == lora_a_weights.size(-1)
109
+ assert inputs.is_contiguous()
110
+
111
+ if lora_a_weights.ndim == 4: # shape:(lora_num,1,rank, size)
112
+ assert lora_a_weights.size(1) == 1
113
+ lora_a_weights = lora_a_weights.squeeze(dim=1)
114
+ else:
115
+ assert lora_a_weights.ndim == 3 # shape:(lora_num,rank, size)
116
+ assert lora_a_weights.is_contiguous()
117
+ assert output_tensor.is_contiguous()
118
+ # TODO tuning this config
119
+ batches = lora_indices_tensor.size(0)
120
+ N, K = lora_a_weights.shape[-2:] # K=hidden_size,N=rank
121
+ BLOCK_N = triton.next_power_of_2(N)
122
+ if override_config:
123
+ config = override_config
124
+ else:
125
+ # First try to load optimal config from the file
126
+ config = get_lora_op_configs("bgmv_shrink", batches, K)
127
+
128
+ grid = lambda META: (
129
+ META["SPLIT_K"],
130
+ batches,
131
+ )
132
+ _bgmv_shrink_kernel[grid](
133
+ inputs,
134
+ lora_a_weights,
135
+ output_tensor,
136
+ N,
137
+ K,
138
+ lora_indices_tensor,
139
+ scaling,
140
+ inputs.stride(0),
141
+ inputs.stride(1),
142
+ lora_a_weights.stride(0),
143
+ lora_a_weights.stride(1),
144
+ lora_a_weights.stride(2),
145
+ output_tensor.stride(0),
146
+ output_tensor.stride(1),
147
+ BLOCK_N=BLOCK_N,
148
+ **config,
149
+ )
150
+ return
vllm/lora/ops/sgmv_expand.py
new
+192
@@ -0,0 +1,192 @@
1
+"""
2
+Based on:
3
+Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023).
4
+Punica: Multi-Tenant LoRA Serving.
5
+https://arxiv.org/abs/2310.18547
6
+"""
7
+
8
+import torch
9
+import triton
10
+import triton.language as tl
11
+
12
+from vllm.triton_utils import libentry
13
+
14
+
15
+@libentry()
16
+@triton.jit
17
+def _sgmv_expand_kernel(
18
+ input_ptr,
19
+ lora_ptr,
20
+ out_ptr,
21
+ N,
22
+ K,
23
+ b_seq_start_loc,
24
+ seq_lens,
25
+ lora_indices,
26
+ xm_stride,
27
+ xk_stride, # 1
28
+ l0_stride, # hidden_size*max_rank
29
+ lora_k_stride,
30
+ lora_n_stride,
31
+ cm_stride,
32
+ cn_stride,
33
+ BLOCK_M: tl.constexpr,
34
+ BLOCK_N: tl.constexpr,
35
+ BLOCK_K: tl.constexpr,
36
+ EVEN_K: tl.constexpr,
37
+ ADD_INPUTS: tl.constexpr,
38
+ CAST_TYPE: tl.constexpr,
39
+):
40
+ """
41
+ The sgmv's expand triton kernel is based on GroupGEMM.
42
+ """
43
+ pid = tl.program_id(axis=0)
44
+ cur_batch = tl.program_id(axis=1)
45
+ cta_n_num = tl.cdiv(N, BLOCK_N)
46
+ pid_m = pid // cta_n_num
47
+ pid_n = pid % cta_n_num
48
+ M = tl.load(seq_lens + cur_batch)
49
+ if pid_m * BLOCK_M > M:
50
+ return
51
+ lora_index = tl.load(lora_indices + cur_batch)
52
+ if lora_index == -1:
53
+ return
54
+ cur_seq_start = tl.load(b_seq_start_loc + cur_batch)
55
+ offset_m = tl.arange(0, BLOCK_M) + pid_m * BLOCK_M
56
+ offset_n = tl.arange(0, BLOCK_N) + pid_n * BLOCK_N
57
+ offset_k = tl.arange(0, BLOCK_K)
58
+ ram = tl.max_contiguous(tl.multiple_of(offset_m % M, BLOCK_M), BLOCK_M)
59
+ rbn = tl.max_contiguous(tl.multiple_of(offset_n % N, BLOCK_N), BLOCK_N)
60
+
61
+ a_ptr = (input_ptr + cur_seq_start * xm_stride + ram[:, None] * xm_stride +
62
+ offset_k[None, :] * xk_stride, )
63
+ b_ptr = (lora_ptr + l0_stride * lora_index +
64
+ offset_k[:, None] * lora_n_stride + rbn[None, :] * lora_k_stride)
65
+ accumulator = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
66
+ for k in range(tl.cdiv(K, BLOCK_K)):
67
+ if EVEN_K:
68
+ tiled_a = tl.load(a_ptr)
69
+ tiled_b = tl.load(b_ptr)
70
+ else:
71
+ tiled_a = tl.load(a_ptr,
72
+ mask=offset_k[None, :] < K - k * BLOCK_K,
73
+ other=0)
74
+ tiled_b = tl.load(b_ptr,
75
+ mask=offset_k[:, None] < K - k * BLOCK_K,
76
+ other=0)
77
+ if CAST_TYPE:
78
+ tiled_a = tiled_a.to(lora_ptr.dtype.element_ty)
79
+ accumulator += tl.dot(
80
+ tiled_a,
81
+ tiled_b,
82
+ )
83
+ a_ptr += BLOCK_K * xk_stride
84
+ b_ptr += BLOCK_K * lora_n_stride
85
+ tiled_c = accumulator.to(lora_ptr.dtype.element_ty)
86
+ offset_cm = cur_seq_start + tl.arange(0, BLOCK_M) + pid_m * BLOCK_M
87
+ offset_cn = tl.arange(0, BLOCK_N) + pid_n * BLOCK_N
88
+ c_ptr = (out_ptr + offset_cm[:, None] * cm_stride +
89
+ offset_cn[None, :] * cn_stride)
90
+ M = tl.load(seq_lens + cur_batch)
91
+ c_mask = (offset_cm[:, None] <
92
+ (cur_seq_start + M)) & (offset_cn[None, :] < N)
93
+ if ADD_INPUTS:
94
+ tiled_out = tl.load(c_ptr, mask=c_mask)
95
+ tiled_c += tiled_out
96
+ tl.store(c_ptr, tiled_c, mask=c_mask)
97
+
98
+
99
+@torch.inference_mode()
100
+def sgmv_expand(
101
+ inputs: torch.Tensor,
102
+ lora_b_weights: torch.Tensor,
103
+ output_tensor: torch.Tensor,
104
+ b_seq_start_loc: torch.Tensor,
105
+ seq_len_tensor: torch.Tensor,
106
+ lora_indices_tensor: torch.Tensor,
107
+ batches: int,
108
+ max_seq_length: int,
109
+ add_inputs: bool = False,
110
+):
111
+ """
112
+ Args:
113
+ inputs (torch.Tensor): input tensor
114
+ lora_b_weights (torch.Tensor): lora'a weight
115
+ output_tensor (torch.Tensor): output tensor
116
+ b_seq_start_loc (torch.Tensor): (batch_size,). The cumulative
117
+ sequence lengths of the sequences in the batch, used to index
118
+ into sequence. E.g.,if the sequence length is [4, 6], it is
119
+ [0, 4, 10].
120
+ seq_len_tensor (torch.Tensor): (batch_size,). record the sequence
121
+ length of the sequences in the batch
122
+ lora_indices_tensor (torch.Tensor): (batch_size,). The LoRA index
123
+ corresponding to each batch. An index of -1 means no lora should be
124
+ applied.
125
+ batches (int): batch size
126
+ max_seq_length (int): The max sequence lengths of the sequences
127
+ in the batch
128
+ add_inputs (bool, optional): Defaults to False. adds the final lora
129
+ results to the output.
130
+ """
131
+
132
+ assert inputs.dtype in [torch.float16, torch.bfloat16, torch.float32]
133
+ assert lora_b_weights.dtype in [
134
+ torch.float16,
135
+ torch.bfloat16,
136
+ ]
137
+ assert inputs.size(1) == lora_b_weights.size(-1)
138
+ assert b_seq_start_loc.size(0) == batches
139
+ assert lora_indices_tensor.size(0) == batches
140
+ assert inputs.is_contiguous()
141
+ assert output_tensor.is_contiguous()
142
+
143
+ if lora_b_weights.ndim == 4: # shape:(lora_num,1,size,rank)
144
+ assert lora_b_weights.size(1) == 1
145
+ lora_b_weights = lora_b_weights.squeeze(dim=1)
146
+ else:
147
+ assert lora_b_weights.ndim == 3 # shape:(lora_num,size,rank)
148
+
149
+ assert lora_b_weights.is_contiguous()
150
+
151
+ # TODO tuning this config
152
+
153
+ N, K = lora_b_weights.shape[-2:] # K= rank,N=hidden_size
154
+ BLOCK_M = 32
155
+ BLOCK_N = 32
156
+ BLOCK_K = 16
157
+ EVEN_K = K % BLOCK_K == 0
158
+ ADD_INPUTS = add_inputs
159
+ CAST_TYPE = False
160
+ if inputs.dtype == torch.float32 and lora_b_weights.dtype in [
161
+ torch.float16,
162
+ torch.bfloat16,
163
+ ]:
164
+ CAST_TYPE = True
165
+ grid = (
166
+ triton.cdiv(max_seq_length, BLOCK_M) * triton.cdiv(N, BLOCK_N),
167
+ batches,
168
+ )
169
+ _sgmv_expand_kernel[grid](
170
+ inputs,
171
+ lora_b_weights,
172
+ output_tensor,
173
+ N,
174
+ K,
175
+ b_seq_start_loc,
176
+ seq_len_tensor,
177
+ lora_indices_tensor,
178
+ inputs.stride(0),
179
+ inputs.stride(1),
180
+ lora_b_weights.stride(0),
181
+ lora_b_weights.stride(1),
182
+ lora_b_weights.stride(2),
183
+ output_tensor.stride(0),
184
+ output_tensor.stride(1),
185
+ BLOCK_M,
186
+ BLOCK_N,
187
+ BLOCK_K,
188
+ EVEN_K,
189
+ ADD_INPUTS,
190
+ CAST_TYPE,
191
+ )
192
+ return
vllm/lora/ops/sgmv_expand_slice.py
new
+205
@@ -0,0 +1,205 @@
1
+"""
2
+Based on:
3
+Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023).
4
+Punica: Multi-Tenant LoRA Serving.
5
+https://arxiv.org/abs/2310.18547
6
+"""
7
+
8
+import torch
9
+import triton
10
+import triton.language as tl
11
+
12
+from vllm.triton_utils import libentry
13
+
14
+
15
+@libentry()
16
+@triton.jit
17
+def _sgmv_expand_slice_kernel(
18
+ input_ptr,
19
+ lora_ptr,
20
+ out_ptr,
21
+ N,
22
+ K,
23
+ b_seq_start_loc,
24
+ seq_lens,
25
+ lora_indices,
26
+ xm_stride,
27
+ xk_stride, # 1
28
+ l0_stride, # hidden_size*max_rank
29
+ lora_k_stride,
30
+ lora_n_stride,
31
+ cm_stride,
32
+ cn_stride,
33
+ slice_offset,
34
+ BLOCK_M: tl.constexpr,
35
+ BLOCK_N: tl.constexpr,
36
+ BLOCK_K: tl.constexpr,
37
+ EVEN_K: tl.constexpr,
38
+ ADD_INPUTS: tl.constexpr,
39
+ CAST_TYPE: tl.constexpr,
40
+):
41
+ """
42
+
43
+ Similar to the 'sgmv_expand' operator, but with an added parameter
44
+ 'slice_offset'. The reason for not reusing the 'sgmv_expand' operator
45
+ might be that in the future, we could implement a fusion operator to
46
+ achieve the current functionality instead of having to call it multiple
47
+ times.
48
+ """
49
+ pid = tl.program_id(axis=0)
50
+ cur_batch = tl.program_id(axis=1)
51
+ cta_n_num = tl.cdiv(N, BLOCK_N)
52
+ pid_m = pid // cta_n_num
53
+ pid_n = pid % cta_n_num
54
+ M = tl.load(seq_lens + cur_batch)
55
+ if pid_m * BLOCK_M > M:
56
+ return
57
+ lora_index = tl.load(lora_indices + cur_batch)
58
+ if lora_index == -1:
59
+ return
60
+ cur_seq_start = tl.load(b_seq_start_loc + cur_batch)
61
+ offset_m = tl.arange(0, BLOCK_M) + pid_m * BLOCK_M
62
+ offset_n = tl.arange(0, BLOCK_N) + pid_n * BLOCK_N
63
+ offset_k = tl.arange(0, BLOCK_K)
64
+ ram = tl.max_contiguous(tl.multiple_of(offset_m % M, BLOCK_M), BLOCK_M)
65
+ rbn = tl.max_contiguous(tl.multiple_of(offset_n % N, BLOCK_N), BLOCK_N)
66
+
67
+ a_ptr = (input_ptr + cur_seq_start * xm_stride + ram[:, None] * xm_stride +
68
+ offset_k[None, :] * xk_stride, )
69
+ b_ptr = (lora_ptr + l0_stride * lora_index +
70
+ offset_k[:, None] * lora_n_stride + rbn[None, :] * lora_k_stride)
71
+ accumulator = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
72
+ for k in range(tl.cdiv(K, BLOCK_K)):
73
+ if EVEN_K:
74
+ tiled_a = tl.load(a_ptr)
75
+ tiled_b = tl.load(b_ptr)
76
+ else:
77
+ tiled_a = tl.load(a_ptr,
78
+ mask=offset_k[None, :] < K - k * BLOCK_K,
79
+ other=0)
80
+ tiled_b = tl.load(b_ptr,
81
+ mask=offset_k[:, None] < K - k * BLOCK_K,
82
+ other=0)
83
+ if CAST_TYPE:
84
+ tiled_a = tiled_a.to(lora_ptr.dtype.element_ty)
85
+ accumulator += tl.dot(
86
+ tiled_a,
87
+ tiled_b,
88
+ )
89
+ a_ptr += BLOCK_K * xk_stride
90
+ b_ptr += BLOCK_K * lora_n_stride
91
+ tiled_c = accumulator.to(lora_ptr.dtype.element_ty)
92
+ offset_cm = cur_seq_start + tl.arange(0, BLOCK_M) + pid_m * BLOCK_M
93
+ offset_cn = tl.arange(0, BLOCK_N) + pid_n * BLOCK_N + slice_offset
94
+ c_ptr = (out_ptr + offset_cm[:, None] * cm_stride +
95
+ offset_cn[None, :] * cn_stride)
96
+ M = tl.load(seq_lens + cur_batch)
97
+ c_mask = (offset_cm[:, None] < (cur_seq_start + M)) & (offset_cn[None, :] <
98
+ (slice_offset + N))
99
+ if ADD_INPUTS:
100
+ tiled_out = tl.load(c_ptr, mask=c_mask)
101
+ tiled_c += tiled_out
102
+ tl.store(c_ptr, tiled_c, mask=c_mask)
103
+
104
+
105
+@torch.inference_mode()
106
+def sgmv_expand_slice(
107
+ inputs: torch.Tensor,
108
+ lora_b_weights: torch.Tensor,
109
+ output_tensor: torch.Tensor,
110
+ b_seq_start_loc: torch.Tensor,
111
+ seq_len_tensor: torch.Tensor,
112
+ lora_indices_tensor: torch.Tensor,
113
+ batches: int,
114
+ max_seq_length: int,
115
+ slice_offset: int,
116
+ slice_size: int,
117
+ add_inputs: bool = False,
118
+):
119
+ """_summary_
120
+
121
+ Args:
122
+ inputs (torch.Tensor): input tensor
123
+ lora_b_weights (torch.Tensor): lora'a weight
124
+ output_tensor (torch.Tensor): output tensor
125
+ b_seq_start_loc (torch.Tensor): (batch_size,). The cumulative
126
+ sequence lengths of the sequences in the batch, used to index
127
+ into sequence. E.g.,if the sequence length is [4, 6], it is
128
+ [0, 4, 10].
129
+ seq_len_tensor (torch.Tensor): (batch_size,). record the sequence
130
+ length of the sequences in the batch
131
+ lora_indices_tensor (torch.Tensor): (batch_size,). The LoRA index
132
+ corresponding to each batch. An index of -1 means no lora should be
133
+ applied.
134
+ batches (int): batch size
135
+ max_seq_length (int): The max sequence lengths of the sequences
136
+ in the batch
137
+ slice_offst (int): output_tensor's offst
138
+ slice_size (int): current output_tensor's size
139
+ add_inputs (bool, optional): Defaults to False. adds the final lora
140
+ results to the output..
141
+ """
142
+
143
+ assert inputs.dtype in [torch.float16, torch.bfloat16, torch.float32]
144
+ assert lora_b_weights.dtype in [
145
+ torch.float16,
146
+ torch.bfloat16,
147
+ ]
148
+ assert inputs.size(1) == lora_b_weights.size(-1)
149
+ assert b_seq_start_loc.size(0) == batches
150
+ assert lora_indices_tensor.size(0) == batches
151
+ assert slice_size == lora_b_weights.size(-2)
152
+ assert inputs.is_contiguous()
153
+ assert output_tensor.is_contiguous()
154
+
155
+ if lora_b_weights.ndim == 4: # shape:(lora_num,1,size,rank)
156
+ assert lora_b_weights.size(1) == 1
157
+ lora_b_weights = lora_b_weights.squeeze(dim=1)
158
+ else:
159
+ assert lora_b_weights.ndim == 3 # shape:(lora_num,size,rank)
160
+
161
+ assert lora_b_weights.is_contiguous()
162
+
163
+ # TODO tuning this config
164
+ N, K = lora_b_weights.shape[-2:] # K= rank,N=hidden_size
165
+
166
+ BLOCK_M = 32
167
+ BLOCK_N = 32
168
+ BLOCK_K = 16
169
+ EVEN_K = K % BLOCK_K == 0
170
+ ADD_INPUTS = add_inputs
171
+ CAST_TYPE = False
172
+ if inputs.dtype == torch.float32 and lora_b_weights.dtype in [
173
+ torch.float16,
174
+ torch.bfloat16,
175
+ ]:
176
+ CAST_TYPE = True
177
+ grid = (
178
+ triton.cdiv(max_seq_length, BLOCK_M) * triton.cdiv(N, BLOCK_N),
179
+ batches,
180
+ )
181
+ _sgmv_expand_slice_kernel[grid](
182
+ inputs,
183
+ lora_b_weights,
184
+ output_tensor,
185
+ N,
186
+ K,
187
+ b_seq_start_loc,
188
+ seq_len_tensor,
189
+ lora_indices_tensor,
190
+ inputs.stride(0),
191
+ inputs.stride(1),
192
+ lora_b_weights.stride(0),
193
+ lora_b_weights.stride(1),
194
+ lora_b_weights.stride(2),
195
+ output_tensor.stride(0),
196
+ output_tensor.stride(1),
197
+ slice_offset,
198
+ BLOCK_M,
199
+ BLOCK_N,
200
+ BLOCK_K,
201
+ EVEN_K,
202
+ ADD_INPUTS,
203
+ CAST_TYPE,
204
+ )
205
+ return
vllm/lora/ops/sgmv_shrink.py
new
+189
@@ -0,0 +1,189 @@
1
+"""
2
+Based on:
3
+Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023).
4
+Punica: Multi-Tenant LoRA Serving.
5
+https://arxiv.org/abs/2310.18547
6
+"""
7
+
8
+import torch
9
+import triton
10
+import triton.language as tl
11
+
12
+from vllm.triton_utils import libentry
13
+
14
+
15
+@libentry()
16
+@triton.jit
17
+def _sgmv_shrink_kernel(
18
+ input_ptr,
19
+ lora_ptr,
20
+ out_ptr,
21
+ N,
22
+ K,
23
+ b_seq_start_loc,
24
+ seq_lens,
25
+ lora_indices,
26
+ scaling,
27
+ xm_stride, # hidden_size
28
+ xk_stride, # 1
29
+ l0_stride, # hidden_size*max_rank
30
+ lora_k_stride,
31
+ lora_n_stride,
32
+ cm_stride,
33
+ cn_stride,
34
+ BLOCK_M: tl.constexpr,
35
+ BLOCK_N: tl.constexpr,
36
+ BLOCK_K: tl.constexpr,
37
+ EVEN_K: tl.constexpr,
38
+ SPLIT_K: tl.constexpr,
39
+):
40
+ """
41
+ The sgmv's shrink triton kernel is based on GroupGEMM+SPLIT-K.
42
+ The GEMM of Multi-LoRA can be considered as GroupGEMM. Additionally,
43
+ introducing SPLIT-K can improve performance
44
+ """
45
+ pid = tl.program_id(axis=0)
46
+ pid_sk = tl.program_id(axis=1)
47
+ cur_batch = tl.program_id(axis=2)
48
+ cta_n_num = tl.cdiv(N, BLOCK_N)
49
+ pid_m = pid // cta_n_num
50
+ pid_n = pid % cta_n_num
51
+
52
+ M = tl.load(seq_lens + cur_batch)
53
+ if pid_m * BLOCK_M > M:
54
+ return
55
+ lora_index = tl.load(lora_indices + cur_batch)
56
+ if lora_index == -1:
57
+ return
58
+ cur_seq_start = tl.load(b_seq_start_loc + cur_batch)
59
+ offset_m = tl.arange(0, BLOCK_M) + pid_m * BLOCK_M
60
+ offset_n = tl.arange(0, BLOCK_N) + pid_n * BLOCK_N
61
+ offset_k = pid_sk * BLOCK_K + tl.arange(0, BLOCK_K)
62
+
63
+ ram = tl.max_contiguous(tl.multiple_of(offset_m % M, BLOCK_M), BLOCK_M)
64
+ rbn = tl.max_contiguous(tl.multiple_of(offset_n % N, BLOCK_N), BLOCK_N)
65
+
66
+ a_ptr = (input_ptr + cur_seq_start * xm_stride + ram[:, None] * xm_stride +
67
+ offset_k[None, :] * xk_stride)
68
+ b_ptr = (lora_ptr + l0_stride * lora_index + rbn[None, :] * lora_k_stride +
69
+ offset_k[:, None] * lora_n_stride)
70
+
71
+ accumulator = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
72
+ for k in range(0, tl.cdiv(K, BLOCK_K * SPLIT_K)):
73
+ if EVEN_K:
74
+ tiled_a = tl.load(a_ptr)
75
+ tiled_b = tl.load(b_ptr)
76
+ else:
77
+ k_remaining = K - k * (BLOCK_K * SPLIT_K)
78
+ tiled_a = tl.load(a_ptr,
79
+ mask=offset_k[None, :] < k_remaining,
80
+ other=0.0)
81
+ tiled_b = tl.load(b_ptr,
82
+ mask=offset_k[:, None] < k_remaining,
83
+ other=0.0)
84
+ accumulator += tl.dot(tiled_a, tiled_b)
85
+
86
+ a_ptr += BLOCK_K * SPLIT_K * xk_stride
87
+ b_ptr += BLOCK_K * SPLIT_K * lora_n_stride
88
+ offset_cm = cur_seq_start + tl.arange(0, BLOCK_M) + pid_m * BLOCK_M
89
+
90
+ offset_cn = tl.arange(0, BLOCK_N) + pid_n * BLOCK_N
91
+ c_ptr = (out_ptr + offset_cm[:, None] * cm_stride +
92
+ offset_cn[None, :] * cn_stride)
93
+ c_mask = (offset_cm[:, None] <
94
+ (cur_seq_start + M)) & (offset_cn[None, :] < N)
95
+ accumulator *= scaling
96
+ # handles write-back with reduction-splitting
97
+ if SPLIT_K == 1:
98
+ tl.store(c_ptr, accumulator, mask=c_mask)
99
+ else:
100
+ tl.atomic_add(c_ptr, accumulator, mask=c_mask)
101
+
102
+
103
+@torch.inference_mode()
104
+def sgmv_shrink(
105
+ inputs: torch.Tensor,
106
+ lora_a_weights: torch.Tensor,
107
+ output_tensor: torch.Tensor,
108
+ b_seq_start_loc: torch.Tensor,
109
+ seq_len_tensor: torch.Tensor,
110
+ lora_indices_tensor: torch.Tensor,
111
+ batches: int,
112
+ max_seq_length: int,
113
+ scaling: float,
114
+):
115
+ """
116
+
117
+ Args:
118
+ inputs (torch.Tensor): input tensor
119
+ lora_a_weights (torch.Tensor): lora'a weight
120
+ output_tensor (torch.Tensor): output tensor
121
+ b_seq_start_loc (torch.Tensor): (batch_size,). The cumulative
122
+ sequence lengths of the sequences in the batch, used to index
123
+ into sequence. E.g.,if the sequence length is [4, 6], it is
124
+ [0, 4].
125
+ seq_len_tensor (torch.Tensor): (batch_size,). record the sequence
126
+ length of the sequences in the batch
127
+ lora_indices_tensor (torch.Tensor): (batch_size,). The LoRA index
128
+ corresponding to each batch. An index of -1 means no lora should be
129
+ applied.
130
+ batches (int): batch size
131
+ max_seq_length (int): The max sequence lengths of the sequences
132
+ in the batch
133
+ scaling (float): Scaling factor.
134
+ """
135
+ assert inputs.dtype == lora_a_weights.dtype
136
+ assert inputs.dtype in [torch.float16, torch.bfloat16]
137
+ assert lora_a_weights.dtype in [
138
+ torch.float16,
139
+ torch.bfloat16,
140
+ ]
141
+ assert inputs.size(1) == lora_a_weights.size(-1)
142
+ assert b_seq_start_loc.size(0) == batches
143
+ assert lora_indices_tensor.size(0) == batches
144
+ assert inputs.is_contiguous()
145
+
146
+ if lora_a_weights.ndim == 4: # shape:(lora_num,1,rank, size)
147
+ assert lora_a_weights.size(1) == 1
148
+ lora_a_weights = lora_a_weights.squeeze(dim=1)
149
+ else:
150
+ assert lora_a_weights.ndim == 3 # shape:(lora_num,rank, size)
151
+ assert lora_a_weights.is_contiguous()
152
+ assert output_tensor.is_contiguous()
153
+ # TODO tuning this config
154
+ N, K = lora_a_weights.shape[-2:] # K=hidden_size,N=rank
155
+ BLOCK_M = 32
156
+ BLOCK_N = 16
157
+ BLOCK_K = 32
158
+ SPLIT_K = 8
159
+ EVEN_K = K % (BLOCK_K * SPLIT_K) == 0
160
+ grid = (
161
+ triton.cdiv(max_seq_length, BLOCK_M) * triton.cdiv(N, BLOCK_N),
162
+ SPLIT_K,
163
+ batches,
164
+ )
165
+
166
+ _sgmv_shrink_kernel[grid](
167
+ inputs,
168
+ lora_a_weights,
169
+ output_tensor,
170
+ N,
171
+ K,
172
+ b_seq_start_loc,
173
+ seq_len_tensor,
174
+ lora_indices_tensor,
175
+ scaling,
176
+ inputs.stride(0),
177
+ inputs.stride(1),
178
+ lora_a_weights.stride(0),
179
+ lora_a_weights.stride(1),
180
+ lora_a_weights.stride(2),
181
+ output_tensor.stride(0),
182
+ output_tensor.stride(1),
183
+ BLOCK_M,
184
+ BLOCK_N,
185
+ BLOCK_K,
186
+ EVEN_K,
187
+ SPLIT_K,
188
+ )
189
+ return
vllm/lora/ops/utils.py
new
+46
@@ -0,0 +1,46 @@
1
+import functools
2
+from typing import Dict
3
+
4
+
5
+@functools.lru_cache
6
+def _get_op_configs(op_type: str, batch: int, hidden_size: int):
7
+ # TODO: add optimal configurations
8
+ return None
9
+
10
+
11
+def _check_divisibility(hidden_size: int):
12
+ # The bgmv_expand kernel requires that the hidden_size be divisible by
13
+ # the number below.
14
+ divisibility = [2, 4, 8, 16, 32, 64]
15
+ divisibility.sort(reverse=True)
16
+ for div in divisibility:
17
+ if hidden_size % div == 0:
18
+ return div
19
+ # hidden_size is an odd number
20
+ return 1
21
+
22
+
23
+def _get_default_config(op_type: str, batch: int, hidden_size: int):
24
+ if op_type == "expand":
25
+ return {
26
+ "BLOCK_N": 256,
27
+ "SPLIT_N": _check_divisibility(hidden_size),
28
+ "num_warps": 8
29
+ }
30
+ else:
31
+ return {"BLOCK_K": 256, "SPLIT_K": 64, "num_warps": 8}
32
+
33
+
34
+def get_lora_op_configs(op_type: str, batch: int,
35
+ hidden_size: int) -> Dict[str, int]:
36
+ """Inspired by `fused_moe_kernel`
37
+ The return value will be a dictionary mapping an irregular grid of batch
38
+ sizes and hidden_size to configurations of the bgmv-related kernel.
39
+ NOTE: It currently only supports the default configuration. We plan to
40
+ generate optimal configurations for different hardware in the future using
41
+ scripts similar to `benchmark_moe.py`.
42
+ """
43
+ config = _get_op_configs(op_type, batch, hidden_size)
44
+ if not config:
45
+ config = _get_default_config(op_type, batch, hidden_size)
46
+ return config
vllm/lora/punica.py
+581
-184
@@ -1,207 +1,604 @@
1
-# Based on code from https://github.com/punica-ai/punica
1
+"""
2
+Based on:
3
+Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023).
4
+Punica: Multi-Tenant LoRA Serving.
5
+https://arxiv.org/abs/2310.18547
6
+"""
7
3
-from typing import Optional
8
+from typing import TYPE_CHECKING, Callable, List, Optional, Tuple, Union
9
10
import torch
11
7
-from vllm import _custom_ops as ops
8
-from vllm.platforms import current_platform
12
+from vllm.triton_utils import HAS_TRITON
13
14
+if HAS_TRITON:
15
+ from vllm.lora.ops.bgmv_expand import bgmv_expand
16
+ from vllm.lora.ops.bgmv_expand_slice import bgmv_expand_slice
17
+ from vllm.lora.ops.bgmv_shrink import bgmv_shrink
18
+ from vllm.lora.ops.sgmv_expand import sgmv_expand
19
+ from vllm.lora.ops.sgmv_expand_slice import sgmv_expand_slice
20
+ from vllm.lora.ops.sgmv_shrink import sgmv_shrink
21
11
-def _check_punica_support():
12
- if ops.is_custom_op_supported("_punica_C::dispatch_bgmv"):
13
- return
22
+if TYPE_CHECKING:
23
+ # avoid circuit import
24
+ from vllm.lora.layers import LoRAMapping
25
+ from vllm.lora.models import LongContextLoRAContext
26
15
- if current_platform.get_device_capability() < (8, 0):
16
- raise ImportError(
17
- "punica LoRA kernels require compute capability >= 8.0")
18
- else:
19
- raise ImportError(
20
- "punica LoRA kernels could not be imported. If you built vLLM "
21
- "from source, make sure VLLM_INSTALL_PUNICA_KERNELS=1 env var "
22
- "was set.")
23
-
24
-
25
-def bgmv(
26
- y: torch.Tensor,
27
- x: torch.Tensor,
28
- w_t_all: torch.Tensor,
29
- indicies: torch.LongTensor,
30
- layer_idx: int,
31
- scale: float,
32
-):
33
- """
34
- Semantics:
35
- y[i] += (
36
- x[i].unsqueeze(0)
37
- @ w_t_all[indices[i], layer_idx, :, :].transpose(-1, -2)
38
- * scale
39
- ).squeeze(0)
27
41
- Args:
42
- y: Shape: `[B, H2]`. Output vectors. Will be changed in-place.
43
- x: Shape: `[B, H1]`. Input vectors.
44
- w_t_all: Shape: `[None, L, H2, H1]`. All of the transposed weight
45
- matrices.
46
- indicies: Shape: `[B]`. Indices of the weight matrices.
47
- layer_idx: Layer index of the weight matrices.
48
- scale: Scaling factor.
28
+def compute_meta(
29
+ token_lora_tensor: torch.Tensor
30
+) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, int, int, bool]:
31
+ """
32
+ Get the information required for the sgmv kernel. With the features:
33
+ 1. If consecutive requests in the batch use the same LoRA, this function
34
+ will combine them into a single request, improving sgmv kernel inference
35
+ performance.
36
+ 2. At the beginning of each prefill stage inference, recalculations are
37
+ needed based on the input, but only once.
38
"""
50
- _check_punica_support()
39
52
- ops.dispatch_bgmv(y, x, w_t_all, indicies, layer_idx, scale)
40
+ lora_indices_tensor, seq_length_tensor = torch.unique_consecutive(
41
+ token_lora_tensor, return_counts=True)
42
+ cum_result = torch.cumsum(seq_length_tensor, dim=0)
43
+ b_seq_start_tensor = torch.zeros_like(seq_length_tensor)
44
+ b_seq_start_tensor[1:].copy_(cum_result[:-1])
45
+ max_length = seq_length_tensor.max().item()
46
47
+ batch_size = lora_indices_tensor.size(0)
48
+ no_lora = False
49
+ # -1 means no lora should be applied. Use `no_lora` to determine whether
50
+ # the current step requires LoRA. If LoRA is not needed, the prefill stage
51
+ # does not need to launch the triton kernel, which can improve performance
52
+ if batch_size == 1 and lora_indices_tensor == -1:
53
+ no_lora = True
54
+ return (b_seq_start_tensor, seq_length_tensor, lora_indices_tensor,
55
+ batch_size, max_length, no_lora)
56
55
-def dispatch_bgmv_low_level(y: torch.Tensor, x: torch.Tensor,
56
- w_t_all: torch.Tensor, indicies: torch.LongTensor,
57
- layer_idx: int, scale: float, y_offset: int,
58
- y_slice_size: int):
59
- """
60
- Same as `bgmv` but you can operate on slices of y.
61
- Pass whole y, define y_offset and y_slice_size.
57
63
- Semantics:
64
- y[i] += (
65
- x[i].unsqueeze(0)
66
- @ w_t_all[indices[i], layer_idx, :, :].transpose(-1, -2)
67
- * scale
68
- ).squeeze(0)
58
+# TODO see if this can be vectorized
59
+def convert_mapping(
60
+ mapping: "LoRAMapping",
61
+ lora_index_to_id: List[Optional[int]],
62
+ max_loras: int,
63
+ vocab_size: int,
64
+ extra_vocab_size: int,
65
+ long_lora_context: Optional["LongContextLoRAContext"] = None,
66
+) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor,
67
+ Optional[torch.Tensor], List[int]]:
68
+ """Converts LoRAMapping to index tensors.
69
70
Args:
71
- y: Shape: `[B, H2]`. Output vectors. Will be changed in-place.
72
- x: Shape: `[B, H1]`. Input vectors.
73
- w_t_all: Shape: `[None, L, y_slice_size, H1]`. Column partition of
74
- all of the transposed LoRA matrices.
75
- indicies: Shape: `[B]`. Indices of the LoRA weights.
76
- layer_idx: Layer index of LoRA weights.
77
- scale: Scaling factor.
78
- y_offset: Offset to apply to the starting column of y.
79
- y_slice_size: Size of the y column slice.
71
+ mapping: LoRAMapping mapping rows in a batch to LoRA ids.
72
+ lora_index_to_id: List mapping LoRA ids to LoRA indices.
73
+ max_loras: Maximum number of LoRAs.
74
+ vocab_size: Model vocab size.
75
+ extra_vocab_size: Extra vocab size each LoRA can have.
76
+ long_lora_context: Passed if there are long context lora in a batch.
77
+
78
+ Returns:
79
+ A tuple of tensors:
80
+ base_indices: Tensor of shape [batch_size] mapping batch rows to
81
+ LoRA indices.
82
+ sampler_indices: Tensor of shape [batch_size] mapping requests to
83
+ LoRA indices for sampler. For generation, this will be the
84
+ same as base_indicies. For prefill, this will map requests
85
+ to LoRA indices.
86
+ sampler_indices_padded: Tensor of shape [batch_size] mapping
87
+ requests to LoRA indices for sampler with padding.
88
+ Same as sampler_indicies, but -1 is replaced with
89
+ max_loras.
90
+ embeddings_indices: Tensor of shape [2, batch_size] mapping
91
+ requests to embedding indices. First row is for embeddings
92
+ added by the LoRAs, second row is for the LoRA.lora_a
93
+ embeddings.
94
+ long_lora_indices: Tensor of shape [batch_size] mapping
95
+ requests to RoPE offsets and rot dims for long LoRAs.
96
+ None if long context lora doesn't exist.
97
+ indices_len: List of lengths of the above tensors. It contains
98
+ (base_indices, sampler_indices, sampler_indices_padded,
99
+ embeddings_indices, long_lora_indices).
100
"""
81
- _check_punica_support()
82
-
83
- ops.dispatch_bgmv_low_level(
84
- y,
85
- x,
86
- w_t_all,
87
- indicies,
88
- layer_idx,
89
- scale,
90
- x.size(1),
91
- y_slice_size,
92
- y_offset,
93
- )
101
+ index_mapping_indices: List[int] = list(mapping.index_mapping).copy()
102
+ embedding_indices = index_mapping_indices.copy()
103
+ lora_indices = index_mapping_indices.copy()
104
+ long_lora_offsets: Optional[torch.Tensor] = None
105
+ if long_lora_context:
106
+ long_lora_offsets = torch.zeros(len(index_mapping_indices),
107
+ device="cuda",
108
+ dtype=torch.long)
109
+ prompt_mapping: List[int] = [
110
+ lora_index_to_id.index(x) if x > 0 else -1
111
+ for x in mapping.prompt_mapping
112
+ ]
113
+ lora_idx = None
114
+ for i in range(len(index_mapping_indices)):
115
+ # TODO index can be slow. optimize
116
+ lora_idx = (lora_index_to_id.index(index_mapping_indices[i])
117
+ if index_mapping_indices[i] > 0 else -1)
118
+ embedding_indices[i] = lora_idx if index_mapping_indices[i] > 0 else 0
119
+ lora_indices[i] = lora_idx
120
+ if long_lora_context:
121
+ assert long_lora_offsets is not None
122
+ lora_offset: int = long_lora_context.offsets_by_lora_id.get(
123
+ index_mapping_indices[i], 0)
124
+ long_lora_offsets[i] = lora_offset
125
126
+ indices_list: List[Union[List[int], torch.Tensor]] = [
127
+ index_mapping_indices,
128
+ lora_indices,
129
+ embedding_indices,
130
+ ]
131
+ if long_lora_context:
132
+ assert long_lora_offsets is not None
133
+ indices_list.append(long_lora_offsets)
134
+ indices = torch.tensor(indices_list, dtype=torch.long, device="cuda")
135
+ prompt_mapping_tensor = torch.tensor(prompt_mapping,
136
+ device="cuda",
137
+ dtype=torch.long)
138
+ embeddings_indices = torch.stack([
139
+ indices[2] * extra_vocab_size,
140
+ indices[2] * (vocab_size + extra_vocab_size),
141
+ ])
142
+ embeddings_indices[embeddings_indices == -1] = max_loras - 1
143
+ base_indices = indices[1]
144
+ sampler_indices = prompt_mapping_tensor
145
+ sampler_indices_padded = sampler_indices.clone()
146
+ sampler_indices_padded[sampler_indices_padded == -1] = max_loras - 1
147
+ sampler_indices_padded = torch.arange(
148
+ 0, len(sampler_indices_padded), device="cuda", dtype=torch.long) + (
149
+ sampler_indices_padded * len(sampler_indices_padded))
150
+ long_lora_indices = None
151
+ long_lora_indices_len: Optional[int] = None
152
+ if long_lora_context:
153
+ long_lora_indices = indices[3]
154
+ long_lora_indices_len = long_lora_indices.shape[-1]
155
+ # Contain length of indices tensors. Used to index into each tensor.
156
+ indices_len = [
157
+ base_indices.shape[-1],
158
+ sampler_indices.shape[-1],
159
+ sampler_indices_padded.shape[-1],
160
+ embeddings_indices.shape[-1],
161
+ ]
162
+ if long_lora_indices_len is not None:
163
+ indices_len.append(long_lora_indices_len)
164
+ else:
165
+ # If long_lora doesn't exist,append None
166
+ indices_len.append(None)
167
96
-def add_lora(y: torch.Tensor,
97
- x: torch.Tensor,
98
- wa_t_all: torch.Tensor,
99
- wb_t_all: torch.Tensor,
100
- indicies: torch.LongTensor,
101
- layer_idx: int,
102
- scale: float,
103
- *,
104
- buffer: Optional[torch.Tensor] = None):
105
- """
106
- Semantics:
107
- y[i] += (
108
- x[i].unsqueeze(0)
109
- @ wa_t_all[indices[i], layer_idx, :, :].transpose(-1, -2)
110
- @ wb_t_all[indices[i], layer_idx, :, :].transpose(-1, -2)
111
- * scale
112
- ).squeeze(0)
168
+ return (
169
+ base_indices,
170
+ sampler_indices,
171
+ sampler_indices_padded,
172
+ embeddings_indices,
173
+ long_lora_indices,
174
+ indices_len,
175
+ )
176
114
- Args:
115
- y: Shape: `[B, H2]`. Output vectors. Will be changed in-place.
116
- x: Shape: `[B, H1]`. Input vectors.
117
- wa_t_all: Shape: `[None, L, R, H1]`. All of the transposed
118
- LoRA A matrices.
119
- wb_t_all: Shape: `[None, L, H2, R]`. All of the transposed
120
- LoRA B matrices.
121
- indicies: Shape: `[B]`. Indices of the LoRA weights.
122
- layer_idx: Layer index of LoRA weights.
123
- scale: Scaling factor.
124
- buffer: Optional. Shape: `[B, R]`. Temporary buffer.
177
+
178
+class PunicaWrapper:
179
"""
126
- _check_punica_support()
127
-
128
- r = wb_t_all.size(-1)
129
- if buffer is None:
130
- # We set the buffer to be float32 by default to avoid
131
- # numerical inaccuracies that would otherwise happen
132
- # due to downcasting.
133
- buffer = torch.zeros((x.size(0), r),
134
- dtype=torch.float32,
135
- device=x.device)
136
- ops.dispatch_bgmv(buffer, x, wa_t_all, indicies, layer_idx, 1.0)
137
- ops.dispatch_bgmv(y, buffer, wb_t_all, indicies, layer_idx, scale)
138
-
139
-
140
-def add_lora_slice(y: torch.Tensor,
141
- x: torch.Tensor,
142
- wa_t_all: torch.Tensor,
143
- wb_t_all: torch.Tensor,
144
- indicies: torch.LongTensor,
145
- layer_idx: int,
146
- scale: float,
147
- y_offset: int,
148
- y_slice_size: int,
149
- *,
150
- buffer: Optional[torch.Tensor] = None):
180
+ PunicaWrapper is designed to manage and provide metadata for the punica
181
+ kernel. The main function is to maintain the state information for
182
+ Multi-LoRA, and to provide the interface for the punica kernel.
183
"""
152
- Same as `add_lora` but you can operate on slices of y.
153
- Pass whole y, define y_offset and y_slice_size.
184
155
- Semantics:
156
- y[i] += (
157
- x[i].unsqueeze(0)
158
- @ wa_t_all[indices[i], layer_idx, :, :].transpose(-1, -2)
159
- @ wb_t_all[indices[i], layer_idx, :, :].transpose(-1, -2)
160
- * scale
161
- ).squeeze(0)
185
+ def __init__(self, max_num_batched_tokens: int, max_batches: int,
186
+ device: str):
187
+ self._token_lora_indices = torch.empty(max_num_batched_tokens,
188
+ dtype=torch.long,
189
+ device=device)
190
+ self._sampler_indices = torch.empty(max_num_batched_tokens,
191
+ dtype=torch.long,
192
+ device=device)
193
+ self._sampler_indices_padded = torch.empty(max_num_batched_tokens,
194
+ dtype=torch.long,
195
+ device=device)
196
+ self._embeddings_indices = torch.empty(2,
197
+ max_num_batched_tokens,
198
+ dtype=torch.long,
199
+ device=device)
200
+ self._long_lora_indices = torch.empty(max_num_batched_tokens,
201
+ dtype=torch.long,
202
+ device=device)
203
163
- Args:
164
- y: Shape: `[B, H2]`. Output vectors. Will be changed in-place.
165
- x: Shape: `[B, H1]`. Input vectors.
166
- wa_t_all: Shape: `[None, L, R, H1]`. All of the transposed
167
- LoRA A matrices.
168
- wb_t_all: Shape: `[None, L, H2, R]`. All of the transposed
169
- LoRA B matrices.
170
- indicies: Shape: `[B]`. Indices of the LoRA weights.
171
- layer_idx: Layer index of LoRA weights.
172
- scale: Scaling factor.
173
- y_offset: Offset to apply to the starting column of y.
174
- y_slice_size: Size of the y column slice.
175
- """
176
- _check_punica_support()
177
-
178
- r = wb_t_all.size(-1)
179
- if buffer is None:
180
- # We set the buffer to be float32 by default to avoid
181
- # numerical inaccuracies that would otherwise happen
182
- # due to downcasting.
183
- buffer = torch.zeros((x.size(0), r),
184
- dtype=torch.float32,
185
- device=x.device)
186
- ops.dispatch_bgmv_low_level(
187
- buffer,
188
- x,
189
- wa_t_all,
190
- indicies,
191
- layer_idx,
192
- 1.0,
193
- x.size(1),
194
- buffer.size(1),
195
- 0,
196
- )
197
- ops.dispatch_bgmv_low_level(
198
- y,
199
- buffer,
200
- wb_t_all,
201
- indicies,
202
- layer_idx,
203
- scale,
204
- buffer.size(1),
205
- y_slice_size,
206
- y_offset,
207
- )
204
+ # 5 is the number of indicies tensors.
205
+ # base_indices, sampler_indices, sampler_indices_padded,
206
+ # embeddings_indices,long_lora_indices
207
+ self.indices_len: List[Optional[int]] = [None] * 5
208
+ # these attributes are the information required for sgmv kernel
209
+ self._seq_start_locs = torch.empty(max_batches,
210
+ dtype=torch.long,
211
+ device=device)
212
+ self._seq_lengths = torch.empty(max_batches,
213
+ dtype=torch.long,
214
+ device=device)
215
+ self._lora_indices_per_batch = torch.empty(max_batches,
216
+ dtype=torch.long,
217
+ device=device)
218
+ self.max_length: int = 0
219
+ self.batch_size: int = -1
220
+ self.is_prefill = False
221
+ self.no_lora = False
222
+
223
+ def update_metadata(
224
+ self,
225
+ mapping: "LoRAMapping",
226
+ lora_index_to_id: List[Optional[int]],
227
+ max_loras: int,
228
+ vocab_size: int,
229
+ extra_vocab_size: int,
230
+ long_lora_context: Optional["LongContextLoRAContext"] = None,
231
+ ):
232
+
233
+ self._update_base_metadata(mapping, lora_index_to_id, max_loras,
234
+ vocab_size, extra_vocab_size,
235
+ long_lora_context)
236
+ if mapping.is_prefill:
237
+ # Update metadata required for prefill-related operators.
238
+ self._update_prefill_metada(self.token_lora_indices)
239
+ self.is_prefill = True
240
+ else:
241
+ self.is_prefill = False
242
+
243
+ def _update_base_metadata(
244
+ self,
245
+ mapping: "LoRAMapping",
246
+ lora_index_to_id: List[Optional[int]],
247
+ max_loras: int,
248
+ vocab_size: int,
249
+ extra_vocab_size: int,
250
+ long_lora_context: Optional["LongContextLoRAContext"] = None,
251
+ ):
252
+ (
253
+ base_indices,
254
+ sampler_indices,
255
+ sampler_indices_padded,
256
+ embeddings_indices,
257
+ long_lora_offsets_tensor,
258
+ indices_len,
259
+ ) = convert_mapping(
260
+ mapping,
261
+ lora_index_to_id,
262
+ max_loras,
263
+ vocab_size,
264
+ extra_vocab_size,
265
+ long_lora_context,
266
+ )
267
+ self._token_lora_indices[:base_indices.shape[0]].copy_(base_indices)
268
+ self._sampler_indices[:sampler_indices.shape[0]].copy_(sampler_indices)
269
+ self._sampler_indices_padded[:sampler_indices_padded.shape[0]].copy_(
270
+ sampler_indices_padded)
271
+ self._embeddings_indices[:embeddings_indices.
272
+ shape[0], :embeddings_indices.shape[1]].copy_(
273
+ embeddings_indices)
274
+ if long_lora_offsets_tensor is not None:
275
+ self._long_lora_indices[:long_lora_offsets_tensor.shape[0]].copy_(
276
+ long_lora_offsets_tensor)
277
+ else:
278
+ self._long_lora_indices.zero_()
279
+
280
+ self.indices_len[:] = indices_len
281
+
282
+ def _update_prefill_metada(self, token_lora_tensor: torch.Tensor) -> None:
283
+
284
+ (b_seq_start_tensor, seq_length_tensor, lora_indices_tensor,
285
+ batch_size, max_length, no_lora) = compute_meta(token_lora_tensor)
286
+
287
+ self._seq_start_locs[:b_seq_start_tensor.shape[0]].copy_(
288
+ b_seq_start_tensor)
289
+ self._seq_lengths[:seq_length_tensor.shape[0]].copy_(seq_length_tensor)
290
+ self._lora_indices_per_batch[:lora_indices_tensor.shape[0]].copy_(
291
+ lora_indices_tensor)
292
+ self.batch_size = batch_size
293
+ self.max_length = max_length
294
+ self.no_lora = no_lora
295
+
296
+ @property
297
+ def prefill_metadata(
298
+ self) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, int, int]:
299
+ """
300
+ This property provides a convenient way to access the necessary
301
+ metadata for prefill-related kernel computations.
302
+ 1. seq_start_locs: Tensor of sequence start positions
303
+ 2. seq_lengths: Tensor of sequence lengths
304
+ 3. lora_indices_per_batch: Tensor of lora indices, and an index of
305
+ -1 means no lora should be applied.
306
+ 4. batch_size: batch size after clustering identical lora indices
307
+ 5. max_length: The maximum sequence length in the batch
308
+ """
309
+ return (self._seq_start_locs[:self.batch_size],
310
+ self._seq_lengths[:self.batch_size],
311
+ self._lora_indices_per_batch[:self.batch_size],
312
+ self.batch_size, self.max_length)
313
+
314
+ @property
315
+ def token_lora_indices(self) -> torch.Tensor:
316
+ """
317
+ This property provides the lora indices corresponding to each token
318
+ in the batch. An index of -1 means no lora should be applied.
319
+ """
320
+ token_lora_len = self.indices_len[0]
321
+ return self._token_lora_indices[:token_lora_len]
322
+
323
+ @property
324
+ def sampler_indices(self) -> torch.Tensor:
325
+ """
326
+ This property is used to access the lora indices specifically for
327
+ LogitsProcessorWithLoRA
328
+ """
329
+ sampler_indices_len = self.indices_len[1]
330
+ return self._sampler_indices[:sampler_indices_len]
331
+
332
+ @property
333
+ def sampler_indices_padded(self) -> torch.Tensor:
334
+ """
335
+ This property provides access to padded sampler indices
336
+ """
337
+ indices_padded_len = self.indices_len[2]
338
+ return self._sampler_indices_padded[:indices_padded_len]
339
+
340
+ @property
341
+ def embeddings_indices(self) -> torch.Tensor:
342
+ """
343
+ This property provides access to the indices used for lora embeddings,
344
+ specifically for VocabParallelEmbeddingWithLoRA
345
+ """
346
+ embeddings_indices_len = self.indices_len[3]
347
+ return self._embeddings_indices[:, :embeddings_indices_len]
348
+
349
+ @property
350
+ def long_lora_indices(self) -> torch.Tensor:
351
+ """
352
+ This property provides access to the indices used for long context
353
+ lora, specifically for LinearScalingRotaryEmbeddingWithLora
354
+ """
355
+ long_lora_len = self.indices_len[4]
356
+ return self._long_lora_indices[:long_lora_len]
357
+
358
+ def shrink_prefill(
359
+ self,
360
+ y: torch.Tensor,
361
+ x: torch.Tensor,
362
+ w_t_all: torch.Tensor,
363
+ scale: float,
364
+ ):
365
+ #No LoRA request, so return directly
366
+ if self.no_lora:
367
+ return
368
+ sgmv_shrink(
369
+ x,
370
+ w_t_all,
371
+ y,
372
+ *self.prefill_metadata,
373
+ scale,
374
+ )
375
+
376
+ def shrink_decode(
377
+ self,
378
+ y: torch.Tensor,
379
+ x: torch.Tensor,
380
+ w_t_all: torch.Tensor,
381
+ scale: float,
382
+ ):
383
+ bgmv_shrink(x, w_t_all, y, self.token_lora_indices, scale)
384
+
385
+ def expand_prefill(
386
+ self,
387
+ y: torch.Tensor,
388
+ x: torch.Tensor,
389
+ w_t_all: torch.Tensor,
390
+ add_input: bool,
391
+ ):
392
+ #No LoRA request, so return directly
393
+ if self.no_lora:
394
+ return
395
+ sgmv_expand(
396
+ x,
397
+ w_t_all,
398
+ y,
399
+ *self.prefill_metadata,
400
+ add_input,
401
+ )
402
+
403
+ def expand_decode(
404
+ self,
405
+ y: torch.Tensor,
406
+ x: torch.Tensor,
407
+ w_t_all: torch.Tensor,
408
+ add_input: bool,
409
+ ):
410
+ bgmv_expand(x, w_t_all, y, self.token_lora_indices, add_input)
411
+
412
+ def expand_slice_prefill(
413
+ self,
414
+ y: torch.Tensor,
415
+ x: torch.Tensor,
416
+ w_t_all: torch.Tensor,
417
+ y_offset: Optional[int],
418
+ y_slice_size: Optional[int],
419
+ add_input: bool,
420
+ ):
421
+ #No LoRA request, so return directly
422
+ if self.no_lora:
423
+ return
424
+ sgmv_expand_slice(
425
+ x,
426
+ w_t_all,
427
+ y,
428
+ *self.prefill_metadata,
429
+ y_offset,
430
+ y_slice_size,
431
+ add_input,
432
+ )
433
+
434
+ def expand_slice_decode(
435
+ self,
436
+ y: torch.Tensor,
437
+ x: torch.Tensor,
438
+ w_t_all: torch.Tensor,
439
+ y_offset: Optional[int],
440
+ y_slice_size: Optional[int],
441
+ add_input: bool,
442
+ ):
443
+ bgmv_expand_slice(x, w_t_all, y, self.token_lora_indices, y_offset,
444
+ y_slice_size, add_input)
445
+
446
+ def add_shrink(
447
+ self,
448
+ y: torch.Tensor,
449
+ x: torch.Tensor,
450
+ w_t_all: torch.Tensor,
451
+ scale: float,
452
+ ):
453
+ """
454
+ Perform the ` y+=x@w_t_all` computation, which is suitable for the
455
+ GEMM of lora'a.
456
+ When `is_prefill is` true, it indicates that it is currently the
457
+ prefill stage, and the `shrink_prefill` function should be called.
458
+ Otherwise, it is the decode stage, and the shrink_decode function
459
+ should be called.
460
+ """
461
+ shrink_fun: Callable = (self.shrink_prefill
462
+ if self.is_prefill else self.shrink_decode)
463
+ shrink_fun(y, x, w_t_all, scale)
464
+
465
+ def add_expand(
466
+ self,
467
+ y: torch.Tensor,
468
+ x: torch.Tensor,
469
+ w_t_all: torch.Tensor,
470
+ add_input: bool = True,
471
+ ):
472
+ """
473
+ Perform the ` y+=x@w_t_all` computation, which is suitable for the
474
+ GEMM of lora'b.
475
+ When `is_prefill` is true, it indicates that it is currently the
476
+ prefill stage, and the `expand_prefill` function should be called.
477
+ Otherwise, it is the decode stage, and the expand_decode function
478
+ should be called.
479
+ """
480
+
481
+ expand_fun: Callable = (self.expand_prefill
482
+ if self.is_prefill else self.expand_decode)
483
+ expand_fun(y, x, w_t_all, add_input)
484
+
485
+ def add_expand_slice(self,
486
+ y: torch.Tensor,
487
+ x: torch.Tensor,
488
+ w_t_all: torch.Tensor,
489
+ y_offset: Optional[int],
490
+ y_slice_size: Optional[int],
491
+ add_input: bool = True):
492
+ """
493
+ Similar to `add_expand`
494
+ """
495
+
496
+ expand_slice_fun: Callable = (self.expand_slice_prefill
497
+ if self.is_prefill else
498
+ self.expand_slice_decode)
499
+ expand_slice_fun(y, x, w_t_all, y_offset, y_slice_size, add_input)
500
+
501
+ def add_lora(self,
502
+ y: torch.Tensor,
503
+ x: torch.Tensor,
504
+ wa_t_all: torch.Tensor,
505
+ wb_t_all: torch.Tensor,
506
+ scale: float,
507
+ y_offset: Optional[int] = None,
508
+ y_slice_size: Optional[int] = None,
509
+ *,
510
+ buffer: Optional[torch.Tensor] = None) -> None:
511
+ """
512
+ Semantics:
513
+ y[i] += (
514
+ x[i].unsqueeze(0)
515
+ @ wa_t_all[indices[i], layer_idx, :, :].transpose(-1, -2)
516
+ @ wb_t_all[indices[i], layer_idx, :, :].transpose(-1, -2)
517
+ * scale
518
+ ).squeeze(0)
519
+ Args:
520
+ y (torch.Tensor): Output tensor. Will be changed in-place.
521
+ x (torch.Tensor): Input tensor
522
+ wa_t_all (torch.Tensor): lora_a's weight
523
+ wb_t_all (torch.Tensor): lora_b's weight
524
+ scale (float): Scaling factor.
525
+ y_offset (Optional[int], optional): Offset to apply to the starting
526
+ column of y.
527
+ y_slice_size (Optional[int], optional): Size of the y column slice..
528
+ buffer (Optional[torch.Tensor], optional): Defaults to None.
529
+ """
530
+ y_org = y
531
+ y = y.view(-1, y.shape[-1])
532
+ x = x.view(-1, x.shape[-1])
533
+ r = wb_t_all.size(-1)
534
+ if buffer is None:
535
+ # We set the buffer to be float32 by default ,refer to:
536
+ # https://github.com/triton-lang/triton/issues/1387
537
+ buffer = torch.zeros((x.size(0), r),
538
+ dtype=torch.float32,
539
+ device=x.device)
540
+
541
+ self.add_shrink(buffer, x, wa_t_all, scale)
542
+ if y_offset is None and y_slice_size is None:
543
+ self.add_expand(y, buffer, wb_t_all, add_input=True)
544
+ else:
545
+ self.add_expand_slice(y,
546
+ buffer,
547
+ wb_t_all,
548
+ y_offset,
549
+ y_slice_size,
550
+ add_input=True)
551
+ y = y.view_as(y_org)
552
+
553
+ def add_lora_packed_nslice(self, y: torch.Tensor, x: torch.Tensor,
554
+ lora_a_stacked: Tuple[torch.Tensor,
555
+ torch.Tensor,
556
+ torch.Tensor],
557
+ lora_b_stacked: Tuple[torch.Tensor,
558
+ torch.Tensor,
559
+ torch.Tensor],
560
+ scale: float,
561
+ output_slices: Tuple[int, ...]) -> None:
562
+ """
563
+ Applies lora to each input. Similar to add_lora, This method is
564
+ used for layers that are composed of multiple sublayers
565
+ (slices) packed together.
566
+ """
567
+ y_org = y
568
+ x = x.view(-1, x.shape[-1])
569
+ y = y.view(-1, y.shape[-1])
570
+ offset_left = 0
571
+ # TODO fuse these kernels
572
+ for slice_idx in range(len(output_slices)):
573
+ self.add_lora(y, x, lora_a_stacked[slice_idx],
574
+ lora_b_stacked[slice_idx], scale, offset_left,
575
+ output_slices[slice_idx])
576
+ offset_left += output_slices[slice_idx]
577
+
578
+ y = y.view_as(y_org)
579
+
580
+ def add_lora_logits(self,
581
+ y: torch.Tensor,
582
+ x: torch.Tensor,
583
+ wa_t_all: torch.Tensor,
584
+ wb_t_all: torch.Tensor,
585
+ scale,
586
+ *,
587
+ buffer: Optional[torch.Tensor] = None) -> None:
588
+ """
589
+ LogitsProcessorWithLoRA always using bgmv
590
+ """
591
+ y_org = y
592
+ y = y.view(-1, y.shape[-1])
593
+ x = x.view(-1, x.shape[-1])
594
+ r = wb_t_all.size(-1)
595
+ if buffer is None:
596
+ # We set the buffer to be float32 by default ,refer to:
597
+ # https://github.com/triton-lang/triton/issues/1387
598
+ buffer = torch.zeros((x.size(0), r),
599
+ dtype=torch.float32,
600
+ device=x.device)
601
+
602
+ bgmv_shrink(x, wa_t_all, buffer, self.sampler_indices, scale)
603
+ bgmv_expand(buffer, wb_t_all, y, self.sampler_indices, add_inputs=True)
604
+ y = y.view_as(y_org)
vllm/triton_utils/__init__.py
+2
-1
@@ -6,5 +6,6 @@ if HAS_TRITON:
6
7
from vllm.triton_utils.custom_cache_manager import (
8
maybe_set_triton_cache_manager)
9
+ from vllm.triton_utils.libentry import libentry
10
10
- __all__ += ["maybe_set_triton_cache_manager"]
11
+ __all__ += ["maybe_set_triton_cache_manager", "libentry"]
vllm/triton_utils/libentry.py
new
+167
@@ -0,0 +1,167 @@
1
+# Copied From https://github.com/FlagOpen/FlagGems
2
+
3
+import inspect
4
+
5
+import triton
6
+
7
+
8
+class LibEntry(triton.KernelInterface):
9
+
10
+ def __init__(
11
+ self,
12
+ fn,
13
+ ):
14
+ self.fn = fn
15
+ self.arg_names = fn.arg_names
16
+ self.divisibility = 16
17
+ self.kernel_cache = dict()
18
+ fn = self.fn
19
+ while not isinstance(fn, triton.runtime.JITFunction):
20
+ fn = fn.fn
21
+ self.jit_function: triton.runtime.JITFunction = fn
22
+ self.specialize_indices = [
23
+ p.num for p in self.jit_function.params
24
+ if not p.is_constexpr and not p.do_not_specialize
25
+ ]
26
+ self.do_not_specialize_indices = [
27
+ p.num for p in self.jit_function.params
28
+ if not p.is_constexpr and p.do_not_specialize
29
+ ]
30
+
31
+ def key(self, spec_args, dns_args, const_args):
32
+ spec_key = [(arg.dtype, arg.data_ptr() %
33
+ self.divisibility == 0) if hasattr(arg, "data_ptr") else
34
+ (type(arg), arg) for arg in spec_args]
35
+ dns_key = [
36
+ arg.dtype if hasattr(
37
+ arg, "data_ptr") else type(arg) if not isinstance(arg, int)
38
+ else "i32" if -(2**31) <= arg and arg <= 2**31 -
39
+ 1 else "u64" if 2**63 <= arg and arg <= 2**64 - 1 else "i64"
40
+ for arg in dns_args
41
+ ]
42
+ # const args passed by position
43
+ return tuple(spec_key + dns_key + const_args)
44
+
45
+ def run(self, *args, **kwargs):
46
+ grid = kwargs["grid"]
47
+ # collect all the arguments
48
+ spec_args = [] # specialize arguments
49
+ dns_args = [] # do not specialize arguments
50
+ const_args = [] # constexpr arguments
51
+ k_args = [] # kernel arguments
52
+ for i, arg in enumerate(args):
53
+ if i in self.specialize_indices:
54
+ k_args.append(arg)
55
+ spec_args.append(arg)
56
+ elif i in self.do_not_specialize_indices:
57
+ k_args.append(arg)
58
+ dns_args.append(arg)
59
+ else:
60
+ const_args.append(arg)
61
+ for p in self.jit_function.params[len(args):]:
62
+ if p.name in kwargs:
63
+ val = kwargs[p.name]
64
+ elif p.default is inspect._empty:
65
+ continue
66
+ else:
67
+ val = p.default
68
+
69
+ if p.is_constexpr:
70
+ const_args.append(val)
71
+ elif p.do_not_specialize:
72
+ dns_args.append(val)
73
+ k_args.append(val)
74
+ else:
75
+ spec_args.append(val)
76
+ k_args.append(val)
77
+
78
+ entry_key = self.key(spec_args, dns_args, const_args)
79
+
80
+ if entry_key not in self.kernel_cache:
81
+ # compile the kernel also completes the related computations
82
+ kernel = self.fn.run(*args, **kwargs)
83
+ fn = self.fn
84
+ # collect constexpr arguments for grid computation
85
+ constexprs = {}
86
+ while not isinstance(fn, triton.runtime.JITFunction):
87
+ if isinstance(fn, triton.runtime.Autotuner):
88
+ config = fn.best_config
89
+ constexprs["num_warps"] = config.num_warps
90
+ constexprs["num_stages"] = config.num_stages
91
+ constexprs["num_ctas"] = config.num_ctas
92
+ constexprs = {**constexprs, **config.kwargs}
93
+ elif isinstance(fn, triton.runtime.Heuristics):
94
+ for v, heur in fn.values.items():
95
+ constexprs[v] = heur({
96
+ **dict(zip(fn.arg_names, args)),
97
+ **kwargs,
98
+ **constexprs,
99
+ })
100
+ else:
101
+ raise RuntimeError("Invalid Runtime Function")
102
+ fn = fn.fn
103
+ # In vLLM, certain kernels like fused_moe_kernel get the
104
+ # best_config(as kwargs) from a configuration json file, rather
105
+ # than using Autotuner & Heuristics. Therefore, all their constexprs
106
+ # (tl.constexpr) are assigned values through the following loop.
107
+ for p in self.jit_function.params:
108
+ if p.is_constexpr and p.name not in constexprs:
109
+ constexprs[p.name] = p.default #default=inspect._empty
110
+ self.kernel_cache[entry_key] = (kernel, constexprs)
111
+ else:
112
+ # load kernel from cache directly
113
+ kernel, constexprs = self.kernel_cache[entry_key]
114
+
115
+ if callable(grid):
116
+ # collect all arguments to the grid fn,ie:
117
+ # 1. args,
118
+ # 2. kwargs,
119
+ # 3. all all other captured arguments in CompiledKernel from
120
+ # Autotunner & Heuristics when kwargs & captured args conflict,
121
+ # captured args have higher priority
122
+ # 4. We must filter out captured args with default value firstly
123
+ constexprs = {
124
+ k: v
125
+ for k, v in constexprs.items() if v is not inspect._empty
126
+ }
127
+ meta = {
128
+ **dict(zip(self.arg_names, args)),
129
+ **kwargs,
130
+ **constexprs,
131
+ }
132
+ grid = grid(meta)
133
+ if isinstance(grid, tuple):
134
+ grid = grid + (1, 1)
135
+ elif isinstance(grid, list):
136
+ grid = grid + [1, 1]
137
+ kernel[grid[0:3]](*k_args)
138
+ # maintaining the same return type as the JITFunction.run
139
+ return kernel
140
+
141
+
142
+def libentry():
143
+ """
144
+ Decorator for triton library entries.
145
+ Motivation:
146
+ The runtime overhead of Triton kernels is the reason for the lower
147
+ performance of small kernels, particularly evident with smaller models.
148
+ Using this decorator can reduce Triton runtime overhead.
149
+ How:
150
+ The `run` function of JITFunction needs to accomplish:
151
+ - Parameter binding using inspect
152
+ - KernelArg type wrapping
153
+ - Cache key calculation
154
+ When dealing with small size, these steps can become bottlenecks in
155
+ Triton runtime. Libentry simplifies these steps to reduce runtime
156
+ overhead, thereby improving the runtime expenses of small kernels.
157
+ NOTE:
158
+ When Triton is upgraded to version 3.0.0, libentry can be removed,
159
+ see: https://github.com/vllm-project/vllm/pull/5036#issuecomment-2243396245
160
+
161
+
162
+ """
163
+
164
+ def decorator(fn):
165
+ return LibEntry(fn)
166
+
167
+ return decorator
vllm/worker/model_runner.py
+6
-6
@@ -578,9 +578,9 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
578
for inter_data in self.inter_data_list
579
])
580
lora_mapping = LoRAMapping(
581
- lora_index_mapping,
582
- lora_prompt_mapping,
583
- )
581
+ **dict(index_mapping=lora_index_mapping,
582
+ prompt_mapping=lora_prompt_mapping,
583
+ is_prefill=not self.decode_only))
584
585
# Prompt adapter data.
586
prompt_adapter_requests: Set[PromptAdapterRequest] = set()
@@ -1152,9 +1152,9 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1152
1153
if self.lora_config:
1154
lora_mapping = LoRAMapping(
1155
- [0] * batch_size,
1156
- [0] * batch_size,
1157
- )
1155
+ **dict(index_mapping=[0] * batch_size,
1156
+ prompt_mapping=[0] * batch_size,
1157
+ is_prefill=False))
1158
self.set_active_loras(set(), lora_mapping)
1159
1160
if self.prompt_adapter_config: