A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Kernel] register punica functions as torch ops (#7591)
bnellnm committed
Aug 16, 2024 at 16:59 UTC
9f698563564c57adb6b4a1ab99d0bb722d70dd5b
6 files changed
+46
-42
vllm/lora/ops/bgmv_expand.py
+8
-12
@@ -5,8 +5,6 @@ Punica: Multi-Tenant LoRA Serving.
5
https://arxiv.org/abs/2310.18547
6
"""
7
8
-from typing import Dict, Optional
9
-
8
import torch
9
import triton
10
import triton.language as tl
@@ -86,14 +84,13 @@ def _bgmv_expand_kernel(
84
85
86
@torch.inference_mode()
89
-def bgmv_expand(
87
+def _bgmv_expand(
88
inputs: torch.Tensor,
89
lora_b_weights: torch.Tensor,
90
output_tensor: torch.Tensor,
91
lora_indices_tensor: torch.Tensor,
92
add_inputs: bool = True,
95
- override_config: Optional[Dict[str, int]] = None,
96
-):
93
+) -> None:
94
"""
95
Args:
96
inputs (torch.Tensor): input tensor
@@ -105,10 +102,7 @@ def bgmv_expand(
102
batches (int): batch size
103
add_inputs (bool, optional): Defaults to False. adds the final lora
104
results to the output.
108
- override_config (Optional[Dict[str, int]], optional): Defaults to None.
109
- Triton grid config
105
"""
111
-
106
assert inputs.dtype in [torch.float16, torch.bfloat16, torch.float32]
107
assert lora_b_weights.dtype in [
108
torch.float16,
@@ -138,10 +132,7 @@ def bgmv_expand(
132
]:
133
CAST_TYPE = True
134
batches = lora_indices_tensor.size(0)
141
- if override_config:
142
- config = override_config
143
- else:
144
- config = get_lora_op_configs("expand", batches, N)
135
+ config = get_lora_op_configs("expand", batches, N)
136
grid = lambda META: (
137
META["SPLIT_N"],
138
batches,
@@ -167,3 +158,8 @@ def bgmv_expand(
158
**config,
159
)
160
return
161
+
162
+
163
+bgmv_expand = torch.library.custom_op("lora::bgmv_expand",
164
+ _bgmv_expand,
165
+ mutates_args=["output_tensor"])
vllm/lora/ops/bgmv_expand_slice.py
+8
-12
@@ -5,8 +5,6 @@ Punica: Multi-Tenant LoRA Serving.
5
https://arxiv.org/abs/2310.18547
6
"""
7
8
-from typing import Dict, Optional
9
-
8
import torch
9
import triton
10
import triton.language as tl
@@ -89,7 +87,7 @@ def _bgmv_expand_slice_kernel(
87
88
89
@torch.inference_mode()
92
-def bgmv_expand_slice(
90
+def _bgmv_expand_slice(
91
inputs: torch.Tensor,
92
lora_b_weights: torch.Tensor,
93
output_tensor: torch.Tensor,
@@ -97,8 +95,7 @@ def bgmv_expand_slice(
95
slice_offset: int,
96
slice_size: int,
97
add_inputs: bool = True,
100
- override_config: Optional[Dict[str, int]] = None,
101
-):
98
+) -> None:
99
"""
100
Args:
101
inputs (torch.Tensor): input tensor
@@ -111,10 +108,7 @@ def bgmv_expand_slice(
108
slice_size (int): current output_tensor's size
109
batches (int): batch size
110
add_inputs (bool, optional): Defaults to False.
114
- override_config (Optional[Dict[str, int]], optional): Defaults to None.
115
- Triton grid config
111
"""
117
-
112
assert inputs.dtype in [torch.float16, torch.bfloat16, torch.float32]
113
assert lora_b_weights.dtype in [
114
torch.float16,
@@ -149,10 +143,7 @@ def bgmv_expand_slice(
143
144
batches = lora_indices_tensor.size(0)
145
152
- if override_config:
153
- config = override_config
154
- else:
155
- config = get_lora_op_configs("expand", batches, N)
146
+ config = get_lora_op_configs("expand", batches, N)
147
148
grid = lambda META: (
149
META["SPLIT_N"],
@@ -180,3 +171,8 @@ def bgmv_expand_slice(
171
**config,
172
)
173
return
174
+
175
+
176
+bgmv_expand_slice = torch.library.custom_op("lora::bgmv_expand_slice",
177
+ _bgmv_expand_slice,
178
+ mutates_args=["output_tensor"])
vllm/lora/ops/bgmv_shrink.py
+9
-12
@@ -5,8 +5,6 @@ Punica: Multi-Tenant LoRA Serving.
5
https://arxiv.org/abs/2310.18547
6
"""
7
8
-from typing import Dict, Optional
9
-
8
import torch
9
import triton
10
import triton.language as tl
@@ -78,14 +76,13 @@ def _bgmv_shrink_kernel(
76
77
78
@torch.inference_mode()
81
-def bgmv_shrink(
79
+def _bgmv_shrink(
80
inputs: torch.Tensor,
81
lora_a_weights: torch.Tensor,
82
output_tensor: torch.Tensor,
83
lora_indices_tensor: torch.Tensor,
84
scaling: float = 1.0,
87
- override_config: Optional[Dict[str, int]] = None,
88
-):
85
+) -> None:
86
"""
87
Args:
88
inputs (torch.Tensor): input tensor
@@ -96,8 +93,6 @@ def bgmv_shrink(
93
applied.
94
batches (int): batch size
95
scaling (float): Scaling factor.
99
- override_config (Optional[Dict[str, int]], optional): Defaults to None.
100
- Triton grid config
96
"""
97
assert inputs.dtype == lora_a_weights.dtype
98
assert inputs.dtype in [torch.float16, torch.bfloat16]
@@ -119,11 +114,8 @@ def bgmv_shrink(
114
batches = lora_indices_tensor.size(0)
115
N, K = lora_a_weights.shape[-2:] # K=hidden_size,N=rank
116
BLOCK_N = triton.next_power_of_2(N)
122
- if override_config:
123
- config = override_config
124
- else:
125
- # First try to load optimal config from the file
126
- config = get_lora_op_configs("bgmv_shrink", batches, K)
117
+ # First try to load optimal config from the file
118
+ config = get_lora_op_configs("bgmv_shrink", batches, K)
119
120
grid = lambda META: (
121
META["SPLIT_K"],
@@ -148,3 +140,8 @@ def bgmv_shrink(
140
**config,
141
)
142
return
143
+
144
+
145
+bgmv_shrink = torch.library.custom_op("lora::bgmv_shrink",
146
+ _bgmv_shrink,
147
+ mutates_args=["output_tensor"])
vllm/lora/ops/sgmv_expand.py
+7
-2
@@ -97,7 +97,7 @@ def _sgmv_expand_kernel(
97
98
99
@torch.inference_mode()
100
-def sgmv_expand(
100
+def _sgmv_expand(
101
inputs: torch.Tensor,
102
lora_b_weights: torch.Tensor,
103
output_tensor: torch.Tensor,
@@ -107,7 +107,7 @@ def sgmv_expand(
107
batches: int,
108
max_seq_length: int,
109
add_inputs: bool = False,
110
-):
110
+) -> None:
111
"""
112
Args:
113
inputs (torch.Tensor): input tensor
@@ -190,3 +190,8 @@ def sgmv_expand(
190
CAST_TYPE,
191
)
192
return
193
+
194
+
195
+sgmv_expand = torch.library.custom_op("lora::sgmv_expand",
196
+ _sgmv_expand,
197
+ mutates_args=["output_tensor"])
vllm/lora/ops/sgmv_expand_slice.py
+7
-2
@@ -103,7 +103,7 @@ def _sgmv_expand_slice_kernel(
103
104
105
@torch.inference_mode()
106
-def sgmv_expand_slice(
106
+def _sgmv_expand_slice(
107
inputs: torch.Tensor,
108
lora_b_weights: torch.Tensor,
109
output_tensor: torch.Tensor,
@@ -115,7 +115,7 @@ def sgmv_expand_slice(
115
slice_offset: int,
116
slice_size: int,
117
add_inputs: bool = False,
118
-):
118
+) -> None:
119
"""_summary_
120
121
Args:
@@ -203,3 +203,8 @@ def sgmv_expand_slice(
203
CAST_TYPE,
204
)
205
return
206
+
207
+
208
+sgmv_expand_slice = torch.library.custom_op("lora::sgmv_expand_slice",
209
+ _sgmv_expand_slice,
210
+ mutates_args=["output_tensor"])
vllm/lora/ops/sgmv_shrink.py
+7
-2
@@ -101,7 +101,7 @@ def _sgmv_shrink_kernel(
101
102
103
@torch.inference_mode()
104
-def sgmv_shrink(
104
+def _sgmv_shrink(
105
inputs: torch.Tensor,
106
lora_a_weights: torch.Tensor,
107
output_tensor: torch.Tensor,
@@ -111,7 +111,7 @@ def sgmv_shrink(
111
batches: int,
112
max_seq_length: int,
113
scaling: float,
114
-):
114
+) -> None:
115
"""
116
117
Args:
@@ -187,3 +187,8 @@ def sgmv_shrink(
187
SPLIT_K,
188
)
189
return
190
+
191
+
192
+sgmv_shrink = torch.library.custom_op("lora::sgmv_shrink",
193
+ _sgmv_shrink,
194
+ mutates_args=["output_tensor"])