A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[Kernel] Register punica ops directly (#10522)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Nov 22, 2024 at 01:18 UTC
2385b60d8300ce730ae67d9ea945f06de9ec4e21
7 files changed
+157
-24
tests/lora/test_punica_variation.py
+17
-6
@@ -6,12 +6,13 @@ maximum ranks.
6
import pytest
7
import torch
8
9
-from vllm.lora.ops.bgmv_expand import bgmv_expand
10
-from vllm.lora.ops.bgmv_expand_slice import bgmv_expand_slice
11
-from vllm.lora.ops.bgmv_shrink import bgmv_shrink
12
-from vllm.lora.ops.sgmv_expand import sgmv_expand
13
-from vllm.lora.ops.sgmv_expand_slice import sgmv_expand_slice
14
-from vllm.lora.ops.sgmv_shrink import sgmv_shrink
9
+# Enable custom op register
10
+import vllm.lora.ops.bgmv_expand
11
+import vllm.lora.ops.bgmv_expand_slice
12
+import vllm.lora.ops.bgmv_shrink
13
+import vllm.lora.ops.sgmv_expand
14
+import vllm.lora.ops.sgmv_expand_slice
15
+import vllm.lora.ops.sgmv_shrink # noqa: F401
16
from vllm.platforms import current_platform
17
18
from .utils import (generate_data, generate_data_for_expand_nslices,
@@ -37,6 +38,16 @@ def assert_close(a, b):
38
torch.testing.assert_close(a, b, rtol=rtol, atol=atol)
39
40
41
+# Unlike test_punica_sizes.py, we directly utilize custom op for
42
+# testing, which verifies the correct registration of these ops.
43
+bgmv_expand = torch.ops.vllm.bgmv_expand
44
+bgmv_expand_slice = torch.ops.vllm.bgmv_expand_slice
45
+bgmv_shrink = torch.ops.vllm.bgmv_shrink
46
+sgmv_expand = torch.ops.vllm.sgmv_expand
47
+sgmv_expand_slice = torch.ops.vllm.sgmv_expand_slice
48
+sgmv_shrink = torch.ops.vllm.sgmv_shrink
49
+
50
+
51
@pytest.mark.parametrize("batches", BATCHES)
52
@pytest.mark.parametrize("num_loras", NUM_LORA)
53
@pytest.mark.parametrize("rank", MAX_RANKS)
vllm/lora/ops/bgmv_expand.py
+20
-3
@@ -9,6 +9,8 @@ import torch
9
import triton
10
import triton.language as tl
11
12
+from vllm.utils import direct_register_custom_op
13
+
14
from .utils import get_lora_op_configs
15
16
@@ -162,9 +164,24 @@ def _bgmv_expand(
164
return
165
166
167
+def bgmv_expand_fake(
168
+ inputs: torch.Tensor,
169
+ lora_b_weights: torch.Tensor,
170
+ output_tensor: torch.Tensor,
171
+ lora_indices_tensor: torch.Tensor,
172
+ add_inputs: bool = True,
173
+) -> None:
174
+ return
175
+
176
+
177
try:
166
- bgmv_expand = torch.library.custom_op("lora::bgmv_expand",
167
- _bgmv_expand,
168
- mutates_args=["output_tensor"])
178
+ direct_register_custom_op(
179
+ op_name="bgmv_expand",
180
+ op_func=_bgmv_expand,
181
+ mutates_args=["output_tensor"],
182
+ fake_impl=bgmv_expand_fake,
183
+ )
184
+ bgmv_expand = torch.ops.vllm.bgmv_expand
185
+
186
except AttributeError:
187
bgmv_expand = _bgmv_expand
vllm/lora/ops/bgmv_expand_slice.py
+22
-3
@@ -9,6 +9,8 @@ import torch
9
import triton
10
import triton.language as tl
11
12
+from vllm.utils import direct_register_custom_op
13
+
14
from .utils import get_lora_op_configs
15
16
@@ -179,9 +181,26 @@ def _bgmv_expand_slice(
181
return
182
183
184
+def bgmv_expand_slice_fake(
185
+ inputs: torch.Tensor,
186
+ lora_b_weights: torch.Tensor,
187
+ output_tensor: torch.Tensor,
188
+ lora_indices_tensor: torch.Tensor,
189
+ slice_offset: int,
190
+ slice_size: int,
191
+ add_inputs: bool = True,
192
+) -> None:
193
+ return
194
+
195
+
196
try:
183
- bgmv_expand_slice = torch.library.custom_op("lora::bgmv_expand_slice",
184
- _bgmv_expand_slice,
185
- mutates_args=["output_tensor"])
197
+ direct_register_custom_op(
198
+ op_name="bgmv_expand_slice",
199
+ op_func=_bgmv_expand_slice,
200
+ mutates_args=["output_tensor"],
201
+ fake_impl=bgmv_expand_slice_fake,
202
+ )
203
+ bgmv_expand_slice = torch.ops.vllm.bgmv_expand_slice
204
+
205
except AttributeError:
206
bgmv_expand_slice = _bgmv_expand_slice
vllm/lora/ops/bgmv_shrink.py
+20
-3
@@ -9,6 +9,8 @@ import torch
9
import triton
10
import triton.language as tl
11
12
+from vllm.utils import direct_register_custom_op
13
+
14
from .utils import get_lora_op_configs
15
16
@@ -142,9 +144,24 @@ def _bgmv_shrink(
144
return
145
146
147
+def bgmv_shrink_fake(
148
+ inputs: torch.Tensor,
149
+ lora_a_weights: torch.Tensor,
150
+ output_tensor: torch.Tensor,
151
+ lora_indices_tensor: torch.Tensor,
152
+ scaling: float = 1.0,
153
+) -> None:
154
+ return
155
+
156
+
157
try:
146
- bgmv_shrink = torch.library.custom_op("lora::bgmv_shrink",
147
- _bgmv_shrink,
148
- mutates_args=["output_tensor"])
158
+ direct_register_custom_op(
159
+ op_name="bgmv_shrink",
160
+ op_func=_bgmv_shrink,
161
+ mutates_args=["output_tensor"],
162
+ fake_impl=bgmv_shrink_fake,
163
+ )
164
+ bgmv_shrink = torch.ops.vllm.bgmv_shrink
165
+
166
except AttributeError:
167
bgmv_shrink = _bgmv_shrink
vllm/lora/ops/sgmv_expand.py
+26
-3
@@ -9,6 +9,8 @@ import torch
9
import triton
10
import triton.language as tl
11
12
+from vllm.utils import direct_register_custom_op
13
+
14
15
@triton.jit
16
def _sgmv_expand_kernel(
@@ -196,9 +198,30 @@ def _sgmv_expand(
198
return
199
200
201
+def sgmv_expand_fake(
202
+ inputs: torch.Tensor,
203
+ lora_b_weights: torch.Tensor,
204
+ output_tensor: torch.Tensor,
205
+ b_seq_start_loc: torch.Tensor,
206
+ seq_len_tensor: torch.Tensor,
207
+ lora_indices_tensor: torch.Tensor,
208
+ batches: int,
209
+ max_seq_length: int,
210
+ token_nums: int,
211
+ add_inputs: bool = False,
212
+) -> None:
213
+ return
214
+
215
+
216
try:
200
- sgmv_expand = torch.library.custom_op("lora::sgmv_expand",
201
- _sgmv_expand,
202
- mutates_args=["output_tensor"])
217
+
218
+ direct_register_custom_op(
219
+ op_name="sgmv_expand",
220
+ op_func=_sgmv_expand,
221
+ mutates_args=["output_tensor"],
222
+ fake_impl=sgmv_expand_fake,
223
+ )
224
+ sgmv_expand = torch.ops.vllm.sgmv_expand
225
+
226
except AttributeError:
227
sgmv_expand = _sgmv_expand
vllm/lora/ops/sgmv_expand_slice.py
+27
-3
@@ -9,6 +9,8 @@ import torch
9
import triton
10
import triton.language as tl
11
12
+from vllm.utils import direct_register_custom_op
13
+
14
15
@triton.jit
16
def _sgmv_expand_slice_kernel(
@@ -209,9 +211,31 @@ def _sgmv_expand_slice(
211
return
212
213
214
+def sgmv_expand_slice_fake(
215
+ inputs: torch.Tensor,
216
+ lora_b_weights: torch.Tensor,
217
+ output_tensor: torch.Tensor,
218
+ b_seq_start_loc: torch.Tensor,
219
+ seq_len_tensor: torch.Tensor,
220
+ lora_indices_tensor: torch.Tensor,
221
+ batches: int,
222
+ max_seq_length: int,
223
+ token_nums: int,
224
+ slice_offset: int,
225
+ slice_size: int,
226
+ add_inputs: bool = False,
227
+) -> None:
228
+ return
229
+
230
+
231
try:
213
- sgmv_expand_slice = torch.library.custom_op("lora::sgmv_expand_slice",
214
- _sgmv_expand_slice,
215
- mutates_args=["output_tensor"])
232
+ direct_register_custom_op(
233
+ op_name="sgmv_expand_slice",
234
+ op_func=_sgmv_expand_slice,
235
+ mutates_args=["output_tensor"],
236
+ fake_impl=sgmv_expand_slice_fake,
237
+ )
238
+ sgmv_expand_slice = torch.ops.vllm.sgmv_expand_slice
239
+
240
except AttributeError:
241
sgmv_expand_slice = _sgmv_expand_slice
vllm/lora/ops/sgmv_shrink.py
+25
-3
@@ -9,6 +9,8 @@ import torch
9
import triton
10
import triton.language as tl
11
12
+from vllm.utils import direct_register_custom_op
13
+
14
15
@triton.jit
16
def _sgmv_shrink_kernel(
@@ -190,9 +192,29 @@ def _sgmv_shrink(
192
return
193
194
195
+def sgmv_shrink_fake(
196
+ inputs: torch.Tensor,
197
+ lora_a_weights: torch.Tensor,
198
+ output_tensor: torch.Tensor,
199
+ b_seq_start_loc: torch.Tensor,
200
+ seq_len_tensor: torch.Tensor,
201
+ lora_indices_tensor: torch.Tensor,
202
+ batches: int,
203
+ max_seq_length: int,
204
+ token_nums: int,
205
+ scaling: float,
206
+) -> None:
207
+ return
208
+
209
+
210
try:
194
- sgmv_shrink = torch.library.custom_op("lora::sgmv_shrink",
195
- _sgmv_shrink,
196
- mutates_args=["output_tensor"])
211
+ direct_register_custom_op(
212
+ op_name="sgmv_shrink",
213
+ op_func=_sgmv_shrink,
214
+ mutates_args=["output_tensor"],
215
+ fake_impl=sgmv_shrink_fake,
216
+ )
217
+ sgmv_shrink = torch.ops.vllm.sgmv_shrink
218
+
219
except AttributeError:
220
sgmv_shrink = _sgmv_shrink