@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9f69856356
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Kernel] register punica functions as torch ops (#7591)

bnellnm committed Aug 16, 2024 at 16:59 UTC 9f698563564c57adb6b4a1ab99d0bb722d70dd5b
6 files changed +46 -42
vllm/lora/ops/bgmv_expand.py
+8 -12
@@ -5,8 +5,6 @@ Punica: Multi-Tenant LoRA Serving.
5 https://arxiv.org/abs/2310.18547
6 """
7
8 -from typing import Dict, Optional
9 -
8 import torch
9 import triton
10 import triton.language as tl
@@ -86,14 +84,13 @@ def _bgmv_expand_kernel(
84
85
86 @torch.inference_mode()
89 -def bgmv_expand(
87 +def _bgmv_expand(
88 inputs: torch.Tensor,
89 lora_b_weights: torch.Tensor,
90 output_tensor: torch.Tensor,
91 lora_indices_tensor: torch.Tensor,
92 add_inputs: bool = True,
95 - override_config: Optional[Dict[str, int]] = None,
96 -):
93 +) -> None:
94 """
95 Args:
96 inputs (torch.Tensor): input tensor
@@ -105,10 +102,7 @@ def bgmv_expand(
102 batches (int): batch size
103 add_inputs (bool, optional): Defaults to False. adds the final lora
104 results to the output.
108 - override_config (Optional[Dict[str, int]], optional): Defaults to None.
109 - Triton grid config
105 """
111 -
106 assert inputs.dtype in [torch.float16, torch.bfloat16, torch.float32]
107 assert lora_b_weights.dtype in [
108 torch.float16,
@@ -138,10 +132,7 @@ def bgmv_expand(
132 ]:
133 CAST_TYPE = True
134 batches = lora_indices_tensor.size(0)
141 - if override_config:
142 - config = override_config
143 - else:
144 - config = get_lora_op_configs("expand", batches, N)
135 + config = get_lora_op_configs("expand", batches, N)
136 grid = lambda META: (
137 META["SPLIT_N"],
138 batches,
@@ -167,3 +158,8 @@ def bgmv_expand(
158 **config,
159 )
160 return
161 +
162 +
163 +bgmv_expand = torch.library.custom_op("lora::bgmv_expand",
164 + _bgmv_expand,
165 + mutates_args=["output_tensor"])
vllm/lora/ops/bgmv_expand_slice.py
+8 -12
@@ -5,8 +5,6 @@ Punica: Multi-Tenant LoRA Serving.
5 https://arxiv.org/abs/2310.18547
6 """
7
8 -from typing import Dict, Optional
9 -
8 import torch
9 import triton
10 import triton.language as tl
@@ -89,7 +87,7 @@ def _bgmv_expand_slice_kernel(
87
88
89 @torch.inference_mode()
92 -def bgmv_expand_slice(
90 +def _bgmv_expand_slice(
91 inputs: torch.Tensor,
92 lora_b_weights: torch.Tensor,
93 output_tensor: torch.Tensor,
@@ -97,8 +95,7 @@ def bgmv_expand_slice(
95 slice_offset: int,
96 slice_size: int,
97 add_inputs: bool = True,
100 - override_config: Optional[Dict[str, int]] = None,
101 -):
98 +) -> None:
99 """
100 Args:
101 inputs (torch.Tensor): input tensor
@@ -111,10 +108,7 @@ def bgmv_expand_slice(
108 slice_size (int): current output_tensor's size
109 batches (int): batch size
110 add_inputs (bool, optional): Defaults to False.
114 - override_config (Optional[Dict[str, int]], optional): Defaults to None.
115 - Triton grid config
111 """
117 -
112 assert inputs.dtype in [torch.float16, torch.bfloat16, torch.float32]
113 assert lora_b_weights.dtype in [
114 torch.float16,
@@ -149,10 +143,7 @@ def bgmv_expand_slice(
143
144 batches = lora_indices_tensor.size(0)
145
152 - if override_config:
153 - config = override_config
154 - else:
155 - config = get_lora_op_configs("expand", batches, N)
146 + config = get_lora_op_configs("expand", batches, N)
147
148 grid = lambda META: (
149 META["SPLIT_N"],
@@ -180,3 +171,8 @@ def bgmv_expand_slice(
171 **config,
172 )
173 return
174 +
175 +
176 +bgmv_expand_slice = torch.library.custom_op("lora::bgmv_expand_slice",
177 + _bgmv_expand_slice,
178 + mutates_args=["output_tensor"])
vllm/lora/ops/bgmv_shrink.py
+9 -12
@@ -5,8 +5,6 @@ Punica: Multi-Tenant LoRA Serving.
5 https://arxiv.org/abs/2310.18547
6 """
7
8 -from typing import Dict, Optional
9 -
8 import torch
9 import triton
10 import triton.language as tl
@@ -78,14 +76,13 @@ def _bgmv_shrink_kernel(
76
77
78 @torch.inference_mode()
81 -def bgmv_shrink(
79 +def _bgmv_shrink(
80 inputs: torch.Tensor,
81 lora_a_weights: torch.Tensor,
82 output_tensor: torch.Tensor,
83 lora_indices_tensor: torch.Tensor,
84 scaling: float = 1.0,
87 - override_config: Optional[Dict[str, int]] = None,
88 -):
85 +) -> None:
86 """
87 Args:
88 inputs (torch.Tensor): input tensor
@@ -96,8 +93,6 @@ def bgmv_shrink(
93 applied.
94 batches (int): batch size
95 scaling (float): Scaling factor.
99 - override_config (Optional[Dict[str, int]], optional): Defaults to None.
100 - Triton grid config
96 """
97 assert inputs.dtype == lora_a_weights.dtype
98 assert inputs.dtype in [torch.float16, torch.bfloat16]
@@ -119,11 +114,8 @@ def bgmv_shrink(
114 batches = lora_indices_tensor.size(0)
115 N, K = lora_a_weights.shape[-2:] # K=hidden_size,N=rank
116 BLOCK_N = triton.next_power_of_2(N)
122 - if override_config:
123 - config = override_config
124 - else:
125 - # First try to load optimal config from the file
126 - config = get_lora_op_configs("bgmv_shrink", batches, K)
117 + # First try to load optimal config from the file
118 + config = get_lora_op_configs("bgmv_shrink", batches, K)
119
120 grid = lambda META: (
121 META["SPLIT_K"],
@@ -148,3 +140,8 @@ def bgmv_shrink(
140 **config,
141 )
142 return
143 +
144 +
145 +bgmv_shrink = torch.library.custom_op("lora::bgmv_shrink",
146 + _bgmv_shrink,
147 + mutates_args=["output_tensor"])
vllm/lora/ops/sgmv_expand.py
+7 -2
@@ -97,7 +97,7 @@ def _sgmv_expand_kernel(
97
98
99 @torch.inference_mode()
100 -def sgmv_expand(
100 +def _sgmv_expand(
101 inputs: torch.Tensor,
102 lora_b_weights: torch.Tensor,
103 output_tensor: torch.Tensor,
@@ -107,7 +107,7 @@ def sgmv_expand(
107 batches: int,
108 max_seq_length: int,
109 add_inputs: bool = False,
110 -):
110 +) -> None:
111 """
112 Args:
113 inputs (torch.Tensor): input tensor
@@ -190,3 +190,8 @@ def sgmv_expand(
190 CAST_TYPE,
191 )
192 return
193 +
194 +
195 +sgmv_expand = torch.library.custom_op("lora::sgmv_expand",
196 + _sgmv_expand,
197 + mutates_args=["output_tensor"])
vllm/lora/ops/sgmv_expand_slice.py
+7 -2
@@ -103,7 +103,7 @@ def _sgmv_expand_slice_kernel(
103
104
105 @torch.inference_mode()
106 -def sgmv_expand_slice(
106 +def _sgmv_expand_slice(
107 inputs: torch.Tensor,
108 lora_b_weights: torch.Tensor,
109 output_tensor: torch.Tensor,
@@ -115,7 +115,7 @@ def sgmv_expand_slice(
115 slice_offset: int,
116 slice_size: int,
117 add_inputs: bool = False,
118 -):
118 +) -> None:
119 """_summary_
120
121 Args:
@@ -203,3 +203,8 @@ def sgmv_expand_slice(
203 CAST_TYPE,
204 )
205 return
206 +
207 +
208 +sgmv_expand_slice = torch.library.custom_op("lora::sgmv_expand_slice",
209 + _sgmv_expand_slice,
210 + mutates_args=["output_tensor"])
vllm/lora/ops/sgmv_shrink.py
+7 -2
@@ -101,7 +101,7 @@ def _sgmv_shrink_kernel(
101
102
103 @torch.inference_mode()
104 -def sgmv_shrink(
104 +def _sgmv_shrink(
105 inputs: torch.Tensor,
106 lora_a_weights: torch.Tensor,
107 output_tensor: torch.Tensor,
@@ -111,7 +111,7 @@ def sgmv_shrink(
111 batches: int,
112 max_seq_length: int,
113 scaling: float,
114 -):
114 +) -> None:
115 """
116
117 Args:
@@ -187,3 +187,8 @@ def sgmv_shrink(
187 SPLIT_K,
188 )
189 return
190 +
191 +
192 +sgmv_shrink = torch.library.custom_op("lora::sgmv_shrink",
193 + _sgmv_shrink,
194 + mutates_args=["output_tensor"])