@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 36e4acd02a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[LoRA][Kernel] Remove the unused libentry module (#10214)

Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>

Jee Jee Li committed Nov 11, 2024 at 17:43 UTC 36e4acd02a955f71ebb7b220cbfae4a4379bc57b
7 files changed +49 -276
tests/lora/test_punica_sizes.py
+24 -49
@@ -4,8 +4,6 @@ hidden_sizes included in the LoRA models currently supported by vLLM. It tests
4 whether the corresponding Triton kernel can run normally when tensor parallelism
5 is set to [1, 2, 4, 8, 16, 32, 64].
6 """
7 -from unittest.mock import patch
8 -
7 import pytest
8 import torch
9
@@ -16,7 +14,6 @@ from vllm.lora.ops.sgmv_expand import sgmv_expand
14 from vllm.lora.ops.sgmv_expand_slice import sgmv_expand_slice
15 from vllm.lora.ops.sgmv_shrink import sgmv_shrink
16 from vllm.platforms import current_platform
19 -from vllm.triton_utils.libentry import LibEntry
17
18 from .utils import (generate_data, generate_data_for_expand_nslices,
19 ref_torch_groupgemm)
@@ -235,9 +232,6 @@ def test_punica_bgmv(
232 seed: int,
233 device: str,
234 ):
238 - from vllm.lora.ops.bgmv_expand import _bgmv_expand_kernel
239 - from vllm.lora.ops.bgmv_shrink import _bgmv_shrink_kernel
240 -
235 torch.set_default_device(device)
236 current_platform.seed_everything(seed)
237
@@ -262,33 +256,21 @@ def test_punica_bgmv(
256 device,
257 )
258 if op_type == "shrink":
265 - # The current _bgmv_shrink_kernel does not require the libentry
266 - # decoration. The purpose of adding this patch is to test the
267 - # correctness of libentry.
268 - with patch(
269 - "vllm.lora.ops.bgmv_shrink._bgmv_shrink_kernel",
270 - LibEntry(_bgmv_shrink_kernel),
271 - ):
272 - bgmv_shrink(
273 - inputs_tensor,
274 - lora_weights,
275 - our_out_tensor,
276 - indices,
277 - scaling,
278 - )
259 + bgmv_shrink(
260 + inputs_tensor,
261 + lora_weights,
262 + our_out_tensor,
263 + indices,
264 + scaling,
265 + )
266 else:
280 - # ditto
281 - with patch(
282 - "vllm.lora.ops.bgmv_expand._bgmv_expand_kernel",
283 - LibEntry(_bgmv_expand_kernel),
284 - ):
285 - bgmv_expand(
286 - inputs_tensor,
287 - lora_weights,
288 - our_out_tensor,
289 - indices,
290 - add_inputs=True,
291 - )
267 + bgmv_expand(
268 + inputs_tensor,
269 + lora_weights,
270 + our_out_tensor,
271 + indices,
272 + add_inputs=True,
273 + )
274 ref_torch_groupgemm(
275 ref_out_tensor,
276 inputs_tensor,
@@ -324,7 +306,6 @@ def test_punica_expand_nslices(
306 seed: int,
307 device: str,
308 ):
327 - from vllm.lora.ops.bgmv_expand_slice import _bgmv_expand_slice_kernel
309
310 torch.set_default_device(device)
311 current_platform.seed_everything(seed)
@@ -374,22 +355,16 @@ def test_punica_expand_nslices(
355 add_inputs=True,
356 )
357 else:
377 - # The current _bgmv_expand_slice_kernel does not require the
378 - # libentry decoration. The purpose of adding this patch is to test
379 - # the correctness of libentry.
380 - with patch(
381 - "vllm.lora.ops.bgmv_expand_slice._bgmv_expand_slice_kernel",
382 - LibEntry(_bgmv_expand_slice_kernel),
383 - ):
384 - bgmv_expand_slice(
385 - inputs_tensor,
386 - lora_weights,
387 - our_outputs,
388 - indices,
389 - slice_offset,
390 - slice_size=hidden_size,
391 - add_inputs=True,
392 - )
358 +
359 + bgmv_expand_slice(
360 + inputs_tensor,
361 + lora_weights,
362 + our_outputs,
363 + indices,
364 + slice_offset,
365 + slice_size=hidden_size,
366 + add_inputs=True,
367 + )
368 ref_torch_groupgemm(
369 ref_outputs[:, slice_offset:slice_offset + hidden_size],
370 inputs_tensor,
tests/lora/test_punica_variation.py
+24 -49
@@ -3,8 +3,6 @@ This script is mainly used to test whether trtion kernels can run normally
3 under different conditions, including various batches, numbers of LoRA , and
4 maximum ranks.
5 """
6 -from unittest.mock import patch
7 -
6 import pytest
7 import torch
8
@@ -15,7 +13,6 @@ from vllm.lora.ops.sgmv_expand import sgmv_expand
13 from vllm.lora.ops.sgmv_expand_slice import sgmv_expand_slice
14 from vllm.lora.ops.sgmv_shrink import sgmv_shrink
15 from vllm.platforms import current_platform
18 -from vllm.triton_utils.libentry import LibEntry
16
17 from .utils import (generate_data, generate_data_for_expand_nslices,
18 ref_torch_groupgemm)
@@ -150,8 +147,6 @@ def test_punica_bgmv(
147 seed: int,
148 device: str,
149 ):
153 - from vllm.lora.ops.bgmv_expand import _bgmv_expand_kernel
154 - from vllm.lora.ops.bgmv_shrink import _bgmv_shrink_kernel
150
151 torch.set_default_device(device)
152 current_platform.seed_everything(seed)
@@ -177,33 +172,22 @@ def test_punica_bgmv(
172 device,
173 )
174 if op_type == "shrink":
180 - # The current _bgmv_shrink_kernel does not require the libentry
181 - # decoration. The purpose of adding this patch is to test the
182 - # correctness of libentry.
183 - with patch(
184 - "vllm.lora.ops.bgmv_shrink._bgmv_shrink_kernel",
185 - LibEntry(_bgmv_shrink_kernel),
186 - ):
187 - bgmv_shrink(
188 - inputs_tensor,
189 - lora_weights,
190 - our_out_tensor,
191 - indices,
192 - scaling,
193 - )
175 + bgmv_shrink(
176 + inputs_tensor,
177 + lora_weights,
178 + our_out_tensor,
179 + indices,
180 + scaling,
181 + )
182 else:
195 - # ditto
196 - with patch(
197 - "vllm.lora.ops.bgmv_expand._bgmv_expand_kernel",
198 - LibEntry(_bgmv_expand_kernel),
199 - ):
200 - bgmv_expand(
201 - inputs_tensor,
202 - lora_weights,
203 - our_out_tensor,
204 - indices,
205 - add_inputs=True,
206 - )
183 +
184 + bgmv_expand(
185 + inputs_tensor,
186 + lora_weights,
187 + our_out_tensor,
188 + indices,
189 + add_inputs=True,
190 + )
191 ref_torch_groupgemm(
192 ref_out_tensor,
193 inputs_tensor,
@@ -239,8 +223,6 @@ def test_punica_expand_nslices(
223 seed: int,
224 device: str,
225 ):
242 - from vllm.lora.ops.bgmv_expand_slice import _bgmv_expand_slice_kernel
243 -
226 torch.set_default_device(device)
227 current_platform.seed_everything(seed)
228
@@ -289,22 +271,15 @@ def test_punica_expand_nslices(
271 add_inputs=True,
272 )
273 else:
292 - # The current _bgmv_expand_slice_kernel does not require the
293 - # libentry decoration. The purpose of adding this patch is to test
294 - # the correctness of libentry.
295 - with patch(
296 - "vllm.lora.ops.bgmv_expand_slice._bgmv_expand_slice_kernel",
297 - LibEntry(_bgmv_expand_slice_kernel),
298 - ):
299 - bgmv_expand_slice(
300 - inputs_tensor,
301 - lora_weights,
302 - our_outputs,
303 - indices,
304 - slice_offset,
305 - slice_size=hidden_size,
306 - add_inputs=True,
307 - )
274 + bgmv_expand_slice(
275 + inputs_tensor,
276 + lora_weights,
277 + our_outputs,
278 + indices,
279 + slice_offset,
280 + slice_size=hidden_size,
281 + add_inputs=True,
282 + )
283 ref_torch_groupgemm(
284 ref_outputs[:, slice_offset:slice_offset + hidden_size],
285 inputs_tensor,
vllm/lora/ops/sgmv_expand.py
-3
@@ -9,10 +9,7 @@ import torch
9 import triton
10 import triton.language as tl
11
12 -from vllm.triton_utils import libentry
12
14 -
15 -@libentry()
13 @triton.jit
14 def _sgmv_expand_kernel(
15 input_ptr,
vllm/lora/ops/sgmv_expand_slice.py
-3
@@ -9,10 +9,7 @@ import torch
9 import triton
10 import triton.language as tl
11
12 -from vllm.triton_utils import libentry
12
14 -
15 -@libentry()
13 @triton.jit
14 def _sgmv_expand_slice_kernel(
15 input_ptr,
vllm/lora/ops/sgmv_shrink.py
-3
@@ -9,10 +9,7 @@ import torch
9 import triton
10 import triton.language as tl
11
12 -from vllm.triton_utils import libentry
12
14 -
15 -@libentry()
13 @triton.jit
14 def _sgmv_shrink_kernel(
15 input_ptr,
vllm/triton_utils/__init__.py
+1 -2
@@ -6,6 +6,5 @@ if HAS_TRITON:
6
7 from vllm.triton_utils.custom_cache_manager import (
8 maybe_set_triton_cache_manager)
9 - from vllm.triton_utils.libentry import libentry
9
11 - __all__ += ["maybe_set_triton_cache_manager", "libentry"]
10 + __all__ += ["maybe_set_triton_cache_manager"]
vllm/triton_utils/libentry.py deleted
-167
@@ -1,167 +0,0 @@
1 -# Copied From https://github.com/FlagOpen/FlagGems
2 -
3 -import inspect
4 -
5 -import triton
6 -
7 -
8 -class LibEntry(triton.KernelInterface):
9 -
10 - def __init__(
11 - self,
12 - fn,
13 - ):
14 - self.fn = fn
15 - self.arg_names = fn.arg_names
16 - self.divisibility = 16
17 - self.kernel_cache = dict()
18 - fn = self.fn
19 - while not isinstance(fn, triton.runtime.JITFunction):
20 - fn = fn.fn
21 - self.jit_function: triton.runtime.JITFunction = fn
22 - self.specialize_indices = [
23 - p.num for p in self.jit_function.params
24 - if not p.is_constexpr and not p.do_not_specialize
25 - ]
26 - self.do_not_specialize_indices = [
27 - p.num for p in self.jit_function.params
28 - if not p.is_constexpr and p.do_not_specialize
29 - ]
30 -
31 - def key(self, spec_args, dns_args, const_args):
32 - spec_key = [(arg.dtype, arg.data_ptr() %
33 - self.divisibility == 0) if hasattr(arg, "data_ptr") else
34 - (type(arg), arg) for arg in spec_args]
35 - dns_key = [
36 - arg.dtype if hasattr(
37 - arg, "data_ptr") else type(arg) if not isinstance(arg, int)
38 - else "i32" if arg >= -(2**31) and arg <= 2**31 -
39 - 1 else "u64" if arg >= 2**63 and arg <= 2**64 - 1 else "i64"
40 - for arg in dns_args
41 - ]
42 - # const args passed by position
43 - return tuple(spec_key + dns_key + const_args)
44 -
45 - def run(self, *args, **kwargs):
46 - grid = kwargs["grid"]
47 - # collect all the arguments
48 - spec_args = [] # specialize arguments
49 - dns_args = [] # do not specialize arguments
50 - const_args = [] # constexpr arguments
51 - k_args = [] # kernel arguments
52 - for i, arg in enumerate(args):
53 - if i in self.specialize_indices:
54 - k_args.append(arg)
55 - spec_args.append(arg)
56 - elif i in self.do_not_specialize_indices:
57 - k_args.append(arg)
58 - dns_args.append(arg)
59 - else:
60 - const_args.append(arg)
61 - for p in self.jit_function.params[len(args):]:
62 - if p.name in kwargs:
63 - val = kwargs[p.name]
64 - elif p.default is inspect._empty:
65 - continue
66 - else:
67 - val = p.default
68 -
69 - if p.is_constexpr:
70 - const_args.append(val)
71 - elif p.do_not_specialize:
72 - dns_args.append(val)
73 - k_args.append(val)
74 - else:
75 - spec_args.append(val)
76 - k_args.append(val)
77 -
78 - entry_key = self.key(spec_args, dns_args, const_args)
79 -
80 - if entry_key not in self.kernel_cache:
81 - # compile the kernel also completes the related computations
82 - kernel = self.fn.run(*args, **kwargs)
83 - fn = self.fn
84 - # collect constexpr arguments for grid computation
85 - constexprs = {}
86 - while not isinstance(fn, triton.runtime.JITFunction):
87 - if isinstance(fn, triton.runtime.Autotuner):
88 - config = fn.best_config
89 - constexprs["num_warps"] = config.num_warps
90 - constexprs["num_stages"] = config.num_stages
91 - constexprs["num_ctas"] = config.num_ctas
92 - constexprs = {**constexprs, **config.kwargs}
93 - elif isinstance(fn, triton.runtime.Heuristics):
94 - for v, heur in fn.values.items():
95 - constexprs[v] = heur({
96 - **dict(zip(fn.arg_names, args)),
97 - **kwargs,
98 - **constexprs,
99 - })
100 - else:
101 - raise RuntimeError("Invalid Runtime Function")
102 - fn = fn.fn
103 - # In vLLM, certain kernels like fused_moe_kernel get the
104 - # best_config(as kwargs) from a configuration json file, rather
105 - # than using Autotuner & Heuristics. Therefore, all their constexprs
106 - # (tl.constexpr) are assigned values through the following loop.
107 - for p in self.jit_function.params:
108 - if p.is_constexpr and p.name not in constexprs:
109 - constexprs[p.name] = p.default #default=inspect._empty
110 - self.kernel_cache[entry_key] = (kernel, constexprs)
111 - else:
112 - # load kernel from cache directly
113 - kernel, constexprs = self.kernel_cache[entry_key]
114 -
115 - if callable(grid):
116 - # collect all arguments to the grid fn,ie:
117 - # 1. args,
118 - # 2. kwargs,
119 - # 3. all all other captured arguments in CompiledKernel from
120 - # Autotunner & Heuristics when kwargs & captured args conflict,
121 - # captured args have higher priority
122 - # 4. We must filter out captured args with default value firstly
123 - constexprs = {
124 - k: v
125 - for k, v in constexprs.items() if v is not inspect._empty
126 - }
127 - meta = {
128 - **dict(zip(self.arg_names, args)),
129 - **kwargs,
130 - **constexprs,
131 - }
132 - grid = grid(meta)
133 - if isinstance(grid, tuple):
134 - grid = grid + (1, 1)
135 - elif isinstance(grid, list):
136 - grid = grid + [1, 1]
137 - kernel[grid[0:3]](*k_args)
138 - # maintaining the same return type as the JITFunction.run
139 - return kernel
140 -
141 -
142 -def libentry():
143 - """
144 - Decorator for triton library entries.
145 - Motivation:
146 - The runtime overhead of Triton kernels is the reason for the lower
147 - performance of small kernels, particularly evident with smaller models.
148 - Using this decorator can reduce Triton runtime overhead.
149 - How:
150 - The `run` function of JITFunction needs to accomplish:
151 - - Parameter binding using inspect
152 - - KernelArg type wrapping
153 - - Cache key calculation
154 - When dealing with small size, these steps can become bottlenecks in
155 - Triton runtime. Libentry simplifies these steps to reduce runtime
156 - overhead, thereby improving the runtime expenses of small kernels.
157 - NOTE:
158 - When Triton is upgraded to version 3.0.0, libentry can be removed,
159 - see: https://github.com/vllm-project/vllm/pull/5036#issuecomment-2243396245
160 -
161 -
162 - """
163 -
164 - def decorator(fn):
165 - return LibEntry(fn)
166 -
167 - return decorator