@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2385b60d83
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Kernel] Register punica ops directly (#10522)

Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>

Jee Jee Li committed Nov 22, 2024 at 01:18 UTC 2385b60d8300ce730ae67d9ea945f06de9ec4e21
7 files changed +157 -24
tests/lora/test_punica_variation.py
+17 -6
@@ -6,12 +6,13 @@ maximum ranks.
6 import pytest
7 import torch
8
9 -from vllm.lora.ops.bgmv_expand import bgmv_expand
10 -from vllm.lora.ops.bgmv_expand_slice import bgmv_expand_slice
11 -from vllm.lora.ops.bgmv_shrink import bgmv_shrink
12 -from vllm.lora.ops.sgmv_expand import sgmv_expand
13 -from vllm.lora.ops.sgmv_expand_slice import sgmv_expand_slice
14 -from vllm.lora.ops.sgmv_shrink import sgmv_shrink
9 +# Enable custom op register
10 +import vllm.lora.ops.bgmv_expand
11 +import vllm.lora.ops.bgmv_expand_slice
12 +import vllm.lora.ops.bgmv_shrink
13 +import vllm.lora.ops.sgmv_expand
14 +import vllm.lora.ops.sgmv_expand_slice
15 +import vllm.lora.ops.sgmv_shrink # noqa: F401
16 from vllm.platforms import current_platform
17
18 from .utils import (generate_data, generate_data_for_expand_nslices,
@@ -37,6 +38,16 @@ def assert_close(a, b):
38 torch.testing.assert_close(a, b, rtol=rtol, atol=atol)
39
40
41 +# Unlike test_punica_sizes.py, we directly utilize custom op for
42 +# testing, which verifies the correct registration of these ops.
43 +bgmv_expand = torch.ops.vllm.bgmv_expand
44 +bgmv_expand_slice = torch.ops.vllm.bgmv_expand_slice
45 +bgmv_shrink = torch.ops.vllm.bgmv_shrink
46 +sgmv_expand = torch.ops.vllm.sgmv_expand
47 +sgmv_expand_slice = torch.ops.vllm.sgmv_expand_slice
48 +sgmv_shrink = torch.ops.vllm.sgmv_shrink
49 +
50 +
51 @pytest.mark.parametrize("batches", BATCHES)
52 @pytest.mark.parametrize("num_loras", NUM_LORA)
53 @pytest.mark.parametrize("rank", MAX_RANKS)
vllm/lora/ops/bgmv_expand.py
+20 -3
@@ -9,6 +9,8 @@ import torch
9 import triton
10 import triton.language as tl
11
12 +from vllm.utils import direct_register_custom_op
13 +
14 from .utils import get_lora_op_configs
15
16
@@ -162,9 +164,24 @@ def _bgmv_expand(
164 return
165
166
167 +def bgmv_expand_fake(
168 + inputs: torch.Tensor,
169 + lora_b_weights: torch.Tensor,
170 + output_tensor: torch.Tensor,
171 + lora_indices_tensor: torch.Tensor,
172 + add_inputs: bool = True,
173 +) -> None:
174 + return
175 +
176 +
177 try:
166 - bgmv_expand = torch.library.custom_op("lora::bgmv_expand",
167 - _bgmv_expand,
168 - mutates_args=["output_tensor"])
178 + direct_register_custom_op(
179 + op_name="bgmv_expand",
180 + op_func=_bgmv_expand,
181 + mutates_args=["output_tensor"],
182 + fake_impl=bgmv_expand_fake,
183 + )
184 + bgmv_expand = torch.ops.vllm.bgmv_expand
185 +
186 except AttributeError:
187 bgmv_expand = _bgmv_expand
vllm/lora/ops/bgmv_expand_slice.py
+22 -3
@@ -9,6 +9,8 @@ import torch
9 import triton
10 import triton.language as tl
11
12 +from vllm.utils import direct_register_custom_op
13 +
14 from .utils import get_lora_op_configs
15
16
@@ -179,9 +181,26 @@ def _bgmv_expand_slice(
181 return
182
183
184 +def bgmv_expand_slice_fake(
185 + inputs: torch.Tensor,
186 + lora_b_weights: torch.Tensor,
187 + output_tensor: torch.Tensor,
188 + lora_indices_tensor: torch.Tensor,
189 + slice_offset: int,
190 + slice_size: int,
191 + add_inputs: bool = True,
192 +) -> None:
193 + return
194 +
195 +
196 try:
183 - bgmv_expand_slice = torch.library.custom_op("lora::bgmv_expand_slice",
184 - _bgmv_expand_slice,
185 - mutates_args=["output_tensor"])
197 + direct_register_custom_op(
198 + op_name="bgmv_expand_slice",
199 + op_func=_bgmv_expand_slice,
200 + mutates_args=["output_tensor"],
201 + fake_impl=bgmv_expand_slice_fake,
202 + )
203 + bgmv_expand_slice = torch.ops.vllm.bgmv_expand_slice
204 +
205 except AttributeError:
206 bgmv_expand_slice = _bgmv_expand_slice
vllm/lora/ops/bgmv_shrink.py
+20 -3
@@ -9,6 +9,8 @@ import torch
9 import triton
10 import triton.language as tl
11
12 +from vllm.utils import direct_register_custom_op
13 +
14 from .utils import get_lora_op_configs
15
16
@@ -142,9 +144,24 @@ def _bgmv_shrink(
144 return
145
146
147 +def bgmv_shrink_fake(
148 + inputs: torch.Tensor,
149 + lora_a_weights: torch.Tensor,
150 + output_tensor: torch.Tensor,
151 + lora_indices_tensor: torch.Tensor,
152 + scaling: float = 1.0,
153 +) -> None:
154 + return
155 +
156 +
157 try:
146 - bgmv_shrink = torch.library.custom_op("lora::bgmv_shrink",
147 - _bgmv_shrink,
148 - mutates_args=["output_tensor"])
158 + direct_register_custom_op(
159 + op_name="bgmv_shrink",
160 + op_func=_bgmv_shrink,
161 + mutates_args=["output_tensor"],
162 + fake_impl=bgmv_shrink_fake,
163 + )
164 + bgmv_shrink = torch.ops.vllm.bgmv_shrink
165 +
166 except AttributeError:
167 bgmv_shrink = _bgmv_shrink
vllm/lora/ops/sgmv_expand.py
+26 -3
@@ -9,6 +9,8 @@ import torch
9 import triton
10 import triton.language as tl
11
12 +from vllm.utils import direct_register_custom_op
13 +
14
15 @triton.jit
16 def _sgmv_expand_kernel(
@@ -196,9 +198,30 @@ def _sgmv_expand(
198 return
199
200
201 +def sgmv_expand_fake(
202 + inputs: torch.Tensor,
203 + lora_b_weights: torch.Tensor,
204 + output_tensor: torch.Tensor,
205 + b_seq_start_loc: torch.Tensor,
206 + seq_len_tensor: torch.Tensor,
207 + lora_indices_tensor: torch.Tensor,
208 + batches: int,
209 + max_seq_length: int,
210 + token_nums: int,
211 + add_inputs: bool = False,
212 +) -> None:
213 + return
214 +
215 +
216 try:
200 - sgmv_expand = torch.library.custom_op("lora::sgmv_expand",
201 - _sgmv_expand,
202 - mutates_args=["output_tensor"])
217 +
218 + direct_register_custom_op(
219 + op_name="sgmv_expand",
220 + op_func=_sgmv_expand,
221 + mutates_args=["output_tensor"],
222 + fake_impl=sgmv_expand_fake,
223 + )
224 + sgmv_expand = torch.ops.vllm.sgmv_expand
225 +
226 except AttributeError:
227 sgmv_expand = _sgmv_expand
vllm/lora/ops/sgmv_expand_slice.py
+27 -3
@@ -9,6 +9,8 @@ import torch
9 import triton
10 import triton.language as tl
11
12 +from vllm.utils import direct_register_custom_op
13 +
14
15 @triton.jit
16 def _sgmv_expand_slice_kernel(
@@ -209,9 +211,31 @@ def _sgmv_expand_slice(
211 return
212
213
214 +def sgmv_expand_slice_fake(
215 + inputs: torch.Tensor,
216 + lora_b_weights: torch.Tensor,
217 + output_tensor: torch.Tensor,
218 + b_seq_start_loc: torch.Tensor,
219 + seq_len_tensor: torch.Tensor,
220 + lora_indices_tensor: torch.Tensor,
221 + batches: int,
222 + max_seq_length: int,
223 + token_nums: int,
224 + slice_offset: int,
225 + slice_size: int,
226 + add_inputs: bool = False,
227 +) -> None:
228 + return
229 +
230 +
231 try:
213 - sgmv_expand_slice = torch.library.custom_op("lora::sgmv_expand_slice",
214 - _sgmv_expand_slice,
215 - mutates_args=["output_tensor"])
232 + direct_register_custom_op(
233 + op_name="sgmv_expand_slice",
234 + op_func=_sgmv_expand_slice,
235 + mutates_args=["output_tensor"],
236 + fake_impl=sgmv_expand_slice_fake,
237 + )
238 + sgmv_expand_slice = torch.ops.vllm.sgmv_expand_slice
239 +
240 except AttributeError:
241 sgmv_expand_slice = _sgmv_expand_slice
vllm/lora/ops/sgmv_shrink.py
+25 -3
@@ -9,6 +9,8 @@ import torch
9 import triton
10 import triton.language as tl
11
12 +from vllm.utils import direct_register_custom_op
13 +
14
15 @triton.jit
16 def _sgmv_shrink_kernel(
@@ -190,9 +192,29 @@ def _sgmv_shrink(
192 return
193
194
195 +def sgmv_shrink_fake(
196 + inputs: torch.Tensor,
197 + lora_a_weights: torch.Tensor,
198 + output_tensor: torch.Tensor,
199 + b_seq_start_loc: torch.Tensor,
200 + seq_len_tensor: torch.Tensor,
201 + lora_indices_tensor: torch.Tensor,
202 + batches: int,
203 + max_seq_length: int,
204 + token_nums: int,
205 + scaling: float,
206 +) -> None:
207 + return
208 +
209 +
210 try:
194 - sgmv_shrink = torch.library.custom_op("lora::sgmv_shrink",
195 - _sgmv_shrink,
196 - mutates_args=["output_tensor"])
211 + direct_register_custom_op(
212 + op_name="sgmv_shrink",
213 + op_func=_sgmv_shrink,
214 + mutates_args=["output_tensor"],
215 + fake_impl=sgmv_shrink_fake,
216 + )
217 + sgmv_shrink = torch.ops.vllm.sgmv_shrink
218 +
219 except AttributeError:
220 sgmv_shrink = _sgmv_shrink