A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[LoRA][Kernel] Remove the unused libentry module (#10214)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Nov 11, 2024 at 17:43 UTC
36e4acd02a955f71ebb7b220cbfae4a4379bc57b
7 files changed
+49
-276
tests/lora/test_punica_sizes.py
+24
-49
@@ -4,8 +4,6 @@ hidden_sizes included in the LoRA models currently supported by vLLM. It tests
4
whether the corresponding Triton kernel can run normally when tensor parallelism
5
is set to [1, 2, 4, 8, 16, 32, 64].
6
"""
7
-from unittest.mock import patch
8
-
7
import pytest
8
import torch
9
@@ -16,7 +14,6 @@ from vllm.lora.ops.sgmv_expand import sgmv_expand
14
from vllm.lora.ops.sgmv_expand_slice import sgmv_expand_slice
15
from vllm.lora.ops.sgmv_shrink import sgmv_shrink
16
from vllm.platforms import current_platform
19
-from vllm.triton_utils.libentry import LibEntry
17
18
from .utils import (generate_data, generate_data_for_expand_nslices,
19
ref_torch_groupgemm)
@@ -235,9 +232,6 @@ def test_punica_bgmv(
232
seed: int,
233
device: str,
234
):
238
- from vllm.lora.ops.bgmv_expand import _bgmv_expand_kernel
239
- from vllm.lora.ops.bgmv_shrink import _bgmv_shrink_kernel
240
-
235
torch.set_default_device(device)
236
current_platform.seed_everything(seed)
237
@@ -262,33 +256,21 @@ def test_punica_bgmv(
256
device,
257
)
258
if op_type == "shrink":
265
- # The current _bgmv_shrink_kernel does not require the libentry
266
- # decoration. The purpose of adding this patch is to test the
267
- # correctness of libentry.
268
- with patch(
269
- "vllm.lora.ops.bgmv_shrink._bgmv_shrink_kernel",
270
- LibEntry(_bgmv_shrink_kernel),
271
- ):
272
- bgmv_shrink(
273
- inputs_tensor,
274
- lora_weights,
275
- our_out_tensor,
276
- indices,
277
- scaling,
278
- )
259
+ bgmv_shrink(
260
+ inputs_tensor,
261
+ lora_weights,
262
+ our_out_tensor,
263
+ indices,
264
+ scaling,
265
+ )
266
else:
280
- # ditto
281
- with patch(
282
- "vllm.lora.ops.bgmv_expand._bgmv_expand_kernel",
283
- LibEntry(_bgmv_expand_kernel),
284
- ):
285
- bgmv_expand(
286
- inputs_tensor,
287
- lora_weights,
288
- our_out_tensor,
289
- indices,
290
- add_inputs=True,
291
- )
267
+ bgmv_expand(
268
+ inputs_tensor,
269
+ lora_weights,
270
+ our_out_tensor,
271
+ indices,
272
+ add_inputs=True,
273
+ )
274
ref_torch_groupgemm(
275
ref_out_tensor,
276
inputs_tensor,
@@ -324,7 +306,6 @@ def test_punica_expand_nslices(
306
seed: int,
307
device: str,
308
):
327
- from vllm.lora.ops.bgmv_expand_slice import _bgmv_expand_slice_kernel
309
310
torch.set_default_device(device)
311
current_platform.seed_everything(seed)
@@ -374,22 +355,16 @@ def test_punica_expand_nslices(
355
add_inputs=True,
356
)
357
else:
377
- # The current _bgmv_expand_slice_kernel does not require the
378
- # libentry decoration. The purpose of adding this patch is to test
379
- # the correctness of libentry.
380
- with patch(
381
- "vllm.lora.ops.bgmv_expand_slice._bgmv_expand_slice_kernel",
382
- LibEntry(_bgmv_expand_slice_kernel),
383
- ):
384
- bgmv_expand_slice(
385
- inputs_tensor,
386
- lora_weights,
387
- our_outputs,
388
- indices,
389
- slice_offset,
390
- slice_size=hidden_size,
391
- add_inputs=True,
392
- )
358
+
359
+ bgmv_expand_slice(
360
+ inputs_tensor,
361
+ lora_weights,
362
+ our_outputs,
363
+ indices,
364
+ slice_offset,
365
+ slice_size=hidden_size,
366
+ add_inputs=True,
367
+ )
368
ref_torch_groupgemm(
369
ref_outputs[:, slice_offset:slice_offset + hidden_size],
370
inputs_tensor,
tests/lora/test_punica_variation.py
+24
-49
@@ -3,8 +3,6 @@ This script is mainly used to test whether trtion kernels can run normally
3
under different conditions, including various batches, numbers of LoRA , and
4
maximum ranks.
5
"""
6
-from unittest.mock import patch
7
-
6
import pytest
7
import torch
8
@@ -15,7 +13,6 @@ from vllm.lora.ops.sgmv_expand import sgmv_expand
13
from vllm.lora.ops.sgmv_expand_slice import sgmv_expand_slice
14
from vllm.lora.ops.sgmv_shrink import sgmv_shrink
15
from vllm.platforms import current_platform
18
-from vllm.triton_utils.libentry import LibEntry
16
17
from .utils import (generate_data, generate_data_for_expand_nslices,
18
ref_torch_groupgemm)
@@ -150,8 +147,6 @@ def test_punica_bgmv(
147
seed: int,
148
device: str,
149
):
153
- from vllm.lora.ops.bgmv_expand import _bgmv_expand_kernel
154
- from vllm.lora.ops.bgmv_shrink import _bgmv_shrink_kernel
150
151
torch.set_default_device(device)
152
current_platform.seed_everything(seed)
@@ -177,33 +172,22 @@ def test_punica_bgmv(
172
device,
173
)
174
if op_type == "shrink":
180
- # The current _bgmv_shrink_kernel does not require the libentry
181
- # decoration. The purpose of adding this patch is to test the
182
- # correctness of libentry.
183
- with patch(
184
- "vllm.lora.ops.bgmv_shrink._bgmv_shrink_kernel",
185
- LibEntry(_bgmv_shrink_kernel),
186
- ):
187
- bgmv_shrink(
188
- inputs_tensor,
189
- lora_weights,
190
- our_out_tensor,
191
- indices,
192
- scaling,
193
- )
175
+ bgmv_shrink(
176
+ inputs_tensor,
177
+ lora_weights,
178
+ our_out_tensor,
179
+ indices,
180
+ scaling,
181
+ )
182
else:
195
- # ditto
196
- with patch(
197
- "vllm.lora.ops.bgmv_expand._bgmv_expand_kernel",
198
- LibEntry(_bgmv_expand_kernel),
199
- ):
200
- bgmv_expand(
201
- inputs_tensor,
202
- lora_weights,
203
- our_out_tensor,
204
- indices,
205
- add_inputs=True,
206
- )
183
+
184
+ bgmv_expand(
185
+ inputs_tensor,
186
+ lora_weights,
187
+ our_out_tensor,
188
+ indices,
189
+ add_inputs=True,
190
+ )
191
ref_torch_groupgemm(
192
ref_out_tensor,
193
inputs_tensor,
@@ -239,8 +223,6 @@ def test_punica_expand_nslices(
223
seed: int,
224
device: str,
225
):
242
- from vllm.lora.ops.bgmv_expand_slice import _bgmv_expand_slice_kernel
243
-
226
torch.set_default_device(device)
227
current_platform.seed_everything(seed)
228
@@ -289,22 +271,15 @@ def test_punica_expand_nslices(
271
add_inputs=True,
272
)
273
else:
292
- # The current _bgmv_expand_slice_kernel does not require the
293
- # libentry decoration. The purpose of adding this patch is to test
294
- # the correctness of libentry.
295
- with patch(
296
- "vllm.lora.ops.bgmv_expand_slice._bgmv_expand_slice_kernel",
297
- LibEntry(_bgmv_expand_slice_kernel),
298
- ):
299
- bgmv_expand_slice(
300
- inputs_tensor,
301
- lora_weights,
302
- our_outputs,
303
- indices,
304
- slice_offset,
305
- slice_size=hidden_size,
306
- add_inputs=True,
307
- )
274
+ bgmv_expand_slice(
275
+ inputs_tensor,
276
+ lora_weights,
277
+ our_outputs,
278
+ indices,
279
+ slice_offset,
280
+ slice_size=hidden_size,
281
+ add_inputs=True,
282
+ )
283
ref_torch_groupgemm(
284
ref_outputs[:, slice_offset:slice_offset + hidden_size],
285
inputs_tensor,
vllm/lora/ops/sgmv_expand.py
-3
@@ -9,10 +9,7 @@ import torch
9
import triton
10
import triton.language as tl
11
12
-from vllm.triton_utils import libentry
12
14
-
15
-@libentry()
13
@triton.jit
14
def _sgmv_expand_kernel(
15
input_ptr,
vllm/lora/ops/sgmv_expand_slice.py
-3
@@ -9,10 +9,7 @@ import torch
9
import triton
10
import triton.language as tl
11
12
-from vllm.triton_utils import libentry
12
14
-
15
-@libentry()
13
@triton.jit
14
def _sgmv_expand_slice_kernel(
15
input_ptr,
vllm/lora/ops/sgmv_shrink.py
-3
@@ -9,10 +9,7 @@ import torch
9
import triton
10
import triton.language as tl
11
12
-from vllm.triton_utils import libentry
12
14
-
15
-@libentry()
13
@triton.jit
14
def _sgmv_shrink_kernel(
15
input_ptr,
vllm/triton_utils/__init__.py
+1
-2
@@ -6,6 +6,5 @@ if HAS_TRITON:
6
7
from vllm.triton_utils.custom_cache_manager import (
8
maybe_set_triton_cache_manager)
9
- from vllm.triton_utils.libentry import libentry
9
11
- __all__ += ["maybe_set_triton_cache_manager", "libentry"]
10
+ __all__ += ["maybe_set_triton_cache_manager"]
vllm/triton_utils/libentry.py
deleted
-167
@@ -1,167 +0,0 @@
1
-# Copied From https://github.com/FlagOpen/FlagGems
2
-
3
-import inspect
4
-
5
-import triton
6
-
7
-
8
-class LibEntry(triton.KernelInterface):
9
-
10
- def __init__(
11
- self,
12
- fn,
13
- ):
14
- self.fn = fn
15
- self.arg_names = fn.arg_names
16
- self.divisibility = 16
17
- self.kernel_cache = dict()
18
- fn = self.fn
19
- while not isinstance(fn, triton.runtime.JITFunction):
20
- fn = fn.fn
21
- self.jit_function: triton.runtime.JITFunction = fn
22
- self.specialize_indices = [
23
- p.num for p in self.jit_function.params
24
- if not p.is_constexpr and not p.do_not_specialize
25
- ]
26
- self.do_not_specialize_indices = [
27
- p.num for p in self.jit_function.params
28
- if not p.is_constexpr and p.do_not_specialize
29
- ]
30
-
31
- def key(self, spec_args, dns_args, const_args):
32
- spec_key = [(arg.dtype, arg.data_ptr() %
33
- self.divisibility == 0) if hasattr(arg, "data_ptr") else
34
- (type(arg), arg) for arg in spec_args]
35
- dns_key = [
36
- arg.dtype if hasattr(
37
- arg, "data_ptr") else type(arg) if not isinstance(arg, int)
38
- else "i32" if arg >= -(2**31) and arg <= 2**31 -
39
- 1 else "u64" if arg >= 2**63 and arg <= 2**64 - 1 else "i64"
40
- for arg in dns_args
41
- ]
42
- # const args passed by position
43
- return tuple(spec_key + dns_key + const_args)
44
-
45
- def run(self, *args, **kwargs):
46
- grid = kwargs["grid"]
47
- # collect all the arguments
48
- spec_args = [] # specialize arguments
49
- dns_args = [] # do not specialize arguments
50
- const_args = [] # constexpr arguments
51
- k_args = [] # kernel arguments
52
- for i, arg in enumerate(args):
53
- if i in self.specialize_indices:
54
- k_args.append(arg)
55
- spec_args.append(arg)
56
- elif i in self.do_not_specialize_indices:
57
- k_args.append(arg)
58
- dns_args.append(arg)
59
- else:
60
- const_args.append(arg)
61
- for p in self.jit_function.params[len(args):]:
62
- if p.name in kwargs:
63
- val = kwargs[p.name]
64
- elif p.default is inspect._empty:
65
- continue
66
- else:
67
- val = p.default
68
-
69
- if p.is_constexpr:
70
- const_args.append(val)
71
- elif p.do_not_specialize:
72
- dns_args.append(val)
73
- k_args.append(val)
74
- else:
75
- spec_args.append(val)
76
- k_args.append(val)
77
-
78
- entry_key = self.key(spec_args, dns_args, const_args)
79
-
80
- if entry_key not in self.kernel_cache:
81
- # compile the kernel also completes the related computations
82
- kernel = self.fn.run(*args, **kwargs)
83
- fn = self.fn
84
- # collect constexpr arguments for grid computation
85
- constexprs = {}
86
- while not isinstance(fn, triton.runtime.JITFunction):
87
- if isinstance(fn, triton.runtime.Autotuner):
88
- config = fn.best_config
89
- constexprs["num_warps"] = config.num_warps
90
- constexprs["num_stages"] = config.num_stages
91
- constexprs["num_ctas"] = config.num_ctas
92
- constexprs = {**constexprs, **config.kwargs}
93
- elif isinstance(fn, triton.runtime.Heuristics):
94
- for v, heur in fn.values.items():
95
- constexprs[v] = heur({
96
- **dict(zip(fn.arg_names, args)),
97
- **kwargs,
98
- **constexprs,
99
- })
100
- else:
101
- raise RuntimeError("Invalid Runtime Function")
102
- fn = fn.fn
103
- # In vLLM, certain kernels like fused_moe_kernel get the
104
- # best_config(as kwargs) from a configuration json file, rather
105
- # than using Autotuner & Heuristics. Therefore, all their constexprs
106
- # (tl.constexpr) are assigned values through the following loop.
107
- for p in self.jit_function.params:
108
- if p.is_constexpr and p.name not in constexprs:
109
- constexprs[p.name] = p.default #default=inspect._empty
110
- self.kernel_cache[entry_key] = (kernel, constexprs)
111
- else:
112
- # load kernel from cache directly
113
- kernel, constexprs = self.kernel_cache[entry_key]
114
-
115
- if callable(grid):
116
- # collect all arguments to the grid fn,ie:
117
- # 1. args,
118
- # 2. kwargs,
119
- # 3. all all other captured arguments in CompiledKernel from
120
- # Autotunner & Heuristics when kwargs & captured args conflict,
121
- # captured args have higher priority
122
- # 4. We must filter out captured args with default value firstly
123
- constexprs = {
124
- k: v
125
- for k, v in constexprs.items() if v is not inspect._empty
126
- }
127
- meta = {
128
- **dict(zip(self.arg_names, args)),
129
- **kwargs,
130
- **constexprs,
131
- }
132
- grid = grid(meta)
133
- if isinstance(grid, tuple):
134
- grid = grid + (1, 1)
135
- elif isinstance(grid, list):
136
- grid = grid + [1, 1]
137
- kernel[grid[0:3]](*k_args)
138
- # maintaining the same return type as the JITFunction.run
139
- return kernel
140
-
141
-
142
-def libentry():
143
- """
144
- Decorator for triton library entries.
145
- Motivation:
146
- The runtime overhead of Triton kernels is the reason for the lower
147
- performance of small kernels, particularly evident with smaller models.
148
- Using this decorator can reduce Triton runtime overhead.
149
- How:
150
- The `run` function of JITFunction needs to accomplish:
151
- - Parameter binding using inspect
152
- - KernelArg type wrapping
153
- - Cache key calculation
154
- When dealing with small size, these steps can become bottlenecks in
155
- Triton runtime. Libentry simplifies these steps to reduce runtime
156
- overhead, thereby improving the runtime expenses of small kernels.
157
- NOTE:
158
- When Triton is upgraded to version 3.0.0, libentry can be removed,
159
- see: https://github.com/vllm-project/vllm/pull/5036#issuecomment-2243396245
160
-
161
-
162
- """
163
-
164
- def decorator(fn):
165
- return LibEntry(fn)
166
-
167
- return decorator