@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 78a141d768
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[Misc] LoRA - Refactor Punica ops tests (#12970)

Signed-off-by: Varun Sundar Rabindranath <varun@neuralmagic.com> Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com>

Varun Sundar Rabindranath committed Feb 11, 2025 at 12:56 UTC 78a141d768a18edc8c598a57d992e6aa56a33259
4 files changed +686 -725
tests/lora/test_punica_ops.py new
+652
@@ -0,0 +1,652 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +from threading import Lock
3 +from typing import List
4 +
5 +import pytest
6 +import torch
7 +
8 +import vllm.lora.ops.triton_ops # noqa: F401
9 +from vllm.lora.ops.torch_ops import (bgmv_expand, bgmv_expand_slice,
10 + bgmv_shrink, sgmv_expand,
11 + sgmv_expand_slice, sgmv_shrink)
12 +from vllm.lora.ops.triton_ops.utils import _LORA_A_PTR_DICT, _LORA_B_PTR_DICT
13 +from vllm.platforms import current_platform
14 +
15 +from .utils import (PunicaTensors, assert_close, generate_data,
16 + generate_data_for_expand_nslices,
17 + generate_data_for_nslices)
18 +
19 +
20 +# Utility shrink and expand operations used as reference implementations.
21 +def sgmv_shrink_for_nslices(
22 + nslices: int, inputs_tensor: torch.Tensor,
23 + lora_weights_lst: List[torch.Tensor], out_tensor: torch.Tensor,
24 + b_seq_start_loc: torch.Tensor, seq_len_tensor: torch.Tensor,
25 + prompt_lora_mapping: torch.Tensor, batches: int, max_seq_length: int,
26 + num_tokens: int, scaling: float):
27 + """
28 + Wrapper around sgmv_shrink that handles any nslices.
29 + """
30 + for index in range(nslices):
31 + sgmv_shrink(
32 + inputs_tensor,
33 + lora_weights_lst[index],
34 + out_tensor[index],
35 + b_seq_start_loc,
36 + seq_len_tensor,
37 + prompt_lora_mapping,
38 + batches,
39 + max_seq_length,
40 + num_tokens,
41 + scaling,
42 + )
43 +
44 +
45 +def sgmv_expand_for_nslices(nslices: int, hidden_size: int,
46 + inputs_tensor: torch.Tensor,
47 + lora_weights_lst: List[torch.Tensor],
48 + out_tensor: torch.Tensor,
49 + b_seq_start_loc: torch.Tensor,
50 + seq_len_tensor: torch.Tensor,
51 + prompt_lora_mapping: torch.Tensor, batches: int,
52 + max_seq_length: int, num_tokens: int,
53 + add_inputs: bool) -> None:
54 + """
55 + Wrapper around sgmv_expand that handles any nslices.
56 + """
57 + if nslices == 1:
58 + # Verify the torch's sgmv_expand op
59 + sgmv_expand(
60 + inputs_tensor[0],
61 + lora_weights_lst[0],
62 + out_tensor,
63 + b_seq_start_loc,
64 + seq_len_tensor,
65 + prompt_lora_mapping,
66 + batches,
67 + max_seq_length,
68 + num_tokens,
69 + add_inputs=add_inputs,
70 + )
71 + else:
72 + slice_offset = 0
73 + for index in range(nslices):
74 + lora_weights = lora_weights_lst[index]
75 + sgmv_expand_slice(
76 + inputs_tensor[index],
77 + lora_weights,
78 + out_tensor,
79 + b_seq_start_loc,
80 + seq_len_tensor,
81 + prompt_lora_mapping,
82 + batches,
83 + max_seq_length,
84 + num_tokens,
85 + slice_offset,
86 + hidden_size,
87 + add_inputs=add_inputs,
88 + )
89 + slice_offset += hidden_size
90 +
91 +
92 +_dict_lock = Lock()
93 +
94 +
95 +def check_sgmv_shrink(batches: int, num_loras: int, rank: int,
96 + hidden_size: int, nslices: int, dtype: torch.dtype,
97 + device: str, seq_length: int, scaling: float):
98 + """
99 + Compare outputs of vllm.sgmv_shrink kernel against a reference
100 + implementation.
101 + """
102 + data: PunicaTensors = generate_data_for_nslices(
103 + batches,
104 + hidden_size,
105 + num_loras,
106 + rank,
107 + seq_length,
108 + nslices,
109 + dtype,
110 + "shrink",
111 + device,
112 + )
113 + max_seq_length, token_nums = data.meta()
114 +
115 + # Preventing cache error pointer.
116 + with _dict_lock:
117 + _LORA_A_PTR_DICT.clear()
118 + torch.ops.vllm.sgmv_shrink(
119 + data.inputs_tensor,
120 + data.lora_weights,
121 + data.our_out_tensor,
122 + data.b_seq_start_loc,
123 + data.seq_len_tensor,
124 + data.prompt_lora_mapping,
125 + batches,
126 + max_seq_length,
127 + token_nums,
128 + scaling,
129 + )
130 +
131 + sgmv_shrink_for_nslices(
132 + nslices,
133 + data.inputs_tensor,
134 + data.lora_weights,
135 + data.ref_out_tensor,
136 + data.b_seq_start_loc,
137 + data.seq_len_tensor,
138 + data.prompt_lora_mapping,
139 + batches,
140 + max_seq_length,
141 + token_nums,
142 + scaling,
143 + )
144 + assert_close(data.our_out_tensor, data.ref_out_tensor)
145 +
146 +
147 +def check_sgmv_expand(batches: int, num_loras: int, rank: int,
148 + hidden_size: int, nslices: int, dtype: torch.dtype,
149 + device: str, seq_length: int, add_inputs: bool):
150 + """
151 + Compare outputs of vllm.sgmv_expand kernel against a reference
152 + implementation.
153 + """
154 + data: PunicaTensors = generate_data_for_nslices(
155 + batches,
156 + hidden_size,
157 + num_loras,
158 + rank,
159 + seq_length,
160 + nslices,
161 + dtype,
162 + "expand",
163 + device,
164 + )
165 +
166 + max_seq_length, token_nums = data.meta()
167 +
168 + with _dict_lock:
169 + _LORA_B_PTR_DICT.clear()
170 + torch.ops.vllm.sgmv_expand(
171 + data.inputs_tensor,
172 + data.lora_weights,
173 + data.our_out_tensor,
174 + data.b_seq_start_loc,
175 + data.seq_len_tensor,
176 + data.prompt_lora_mapping,
177 + batches,
178 + max_seq_length,
179 + token_nums,
180 + offset_start=0,
181 + add_inputs=add_inputs,
182 + )
183 +
184 + sgmv_expand_for_nslices(nslices,
185 + hidden_size,
186 + data.inputs_tensor,
187 + data.lora_weights,
188 + data.ref_out_tensor,
189 + data.b_seq_start_loc,
190 + data.seq_len_tensor,
191 + data.prompt_lora_mapping,
192 + batches,
193 + max_seq_length,
194 + token_nums,
195 + add_inputs=add_inputs)
196 +
197 + assert_close(data.our_out_tensor, data.ref_out_tensor)
198 +
199 +
200 +def check_bgmv_shrink(batches: int, num_loras: int, rank: int,
201 + hidden_size: int, dtype: torch.dtype, device: str,
202 + scaling: float):
203 + """
204 + Compare vllm.bgmv_shrink against a reference implementation.
205 + """
206 + seq_length = 1
207 + data: PunicaTensors = generate_data(
208 + batches,
209 + hidden_size,
210 + num_loras,
211 + rank,
212 + seq_length,
213 + dtype,
214 + "shrink",
215 + device,
216 + )
217 +
218 + torch.ops.vllm.bgmv_shrink(
219 + data.inputs_tensor,
220 + data.lora_weights,
221 + data.our_out_tensor,
222 + data.token_lora_mapping,
223 + scaling,
224 + )
225 +
226 + bgmv_shrink(
227 + data.inputs_tensor,
228 + data.lora_weights,
229 + data.ref_out_tensor,
230 + data.token_lora_mapping,
231 + scaling,
232 + )
233 +
234 + data.ref_out_tensor = data.ref_out_tensor.to(torch.float32)
235 + assert_close(data.our_out_tensor, data.ref_out_tensor)
236 +
237 +
238 +def check_bgmv_expand(batches: int, num_loras: int, rank: int,
239 + hidden_size: int, dtype: torch.dtype, device: str,
240 + add_inputs: bool):
241 + """
242 + Compare vllm.bgmv_expand against a reference implementation.
243 + """
244 + seq_length = 1
245 + data: PunicaTensors = generate_data(
246 + batches,
247 + hidden_size,
248 + num_loras,
249 + rank,
250 + seq_length,
251 + dtype,
252 + "expand",
253 + device,
254 + )
255 +
256 + torch.ops.vllm.bgmv_expand(
257 + data.inputs_tensor,
258 + data.lora_weights,
259 + data.our_out_tensor,
260 + data.token_lora_mapping,
261 + add_inputs=add_inputs,
262 + )
263 + bgmv_expand(
264 + data.inputs_tensor,
265 + data.lora_weights,
266 + data.ref_out_tensor,
267 + data.token_lora_mapping,
268 + add_inputs=add_inputs,
269 + )
270 + assert_close(data.our_out_tensor, data.ref_out_tensor)
271 +
272 +
273 +def check_bgmv_expand_slice(batches: int, num_loras: int, rank: int,
274 + hidden_size: int, nslices: int, dtype: torch.dtype,
275 + device: str, add_inputs: bool):
276 + """
277 + Compare vllm.bgmv_expand_slice against a reference implementation.
278 + """
279 + seq_length = 1
280 + data: PunicaTensors = generate_data_for_expand_nslices(
281 + batches,
282 + hidden_size,
283 + num_loras,
284 + rank,
285 + seq_length,
286 + dtype,
287 + nslices,
288 + device,
289 + )
290 +
291 + slice_offset = 0
292 + for index in range(nslices):
293 + torch.ops.vllm.bgmv_expand_slice(
294 + data.inputs_tensor,
295 + data.lora_weights[index],
296 + data.our_out_tensor,
297 + data.token_lora_mapping,
298 + slice_offset,
299 + slice_size=hidden_size,
300 + add_inputs=add_inputs,
301 + )
302 + bgmv_expand_slice(
303 + data.inputs_tensor,
304 + data.lora_weights[index],
305 + data.ref_out_tensor,
306 + data.token_lora_mapping,
307 + slice_offset,
308 + slice_size=hidden_size,
309 + add_inputs=add_inputs,
310 + )
311 +
312 + slice_offset += hidden_size
313 + assert_close(data.our_out_tensor, data.ref_out_tensor)
314 +
315 +
316 +# Tests
317 +# We test the punica kernels along 2 verticals mainly.
318 +# 1. Variations in hidden_dim size
319 +# 2. Variations in all other parameters like (batch_size, max_rank, num_loras
320 +# etc.)
321 +
322 +# We have collected the hidden_sizes included in the LoRA models
323 +# currently supported by vLLM. It tests whether the corresponding Triton
324 +# kernel can run normally when tensor parallelism is set to
325 +# [1, 2, 4, 8, 16, 32, 64].
326 +HIDDEN_SIZES = [
327 + 128,
328 + 256,
329 + 512,
330 + 896,
331 + 1024,
332 + 1152,
333 + 1216,
334 + 1280,
335 + 1536,
336 + 1664,
337 + 2048,
338 + 2240,
339 + 2304,
340 + 2368,
341 + 2432,
342 + 2560,
343 + 2752,
344 + 3072,
345 + 3328,
346 + 3456,
347 + 3584,
348 + 3712,
349 + 4096,
350 + 4480,
351 + 4608,
352 + 4736,
353 + 4864,
354 + 5120,
355 + 5504,
356 + 5632,
357 + 5888,
358 + 6144,
359 + 6400,
360 + 6848,
361 + 6912,
362 + 7168,
363 + 7424,
364 + 8192,
365 + 8960,
366 + 9216,
367 + 9472,
368 + 10240,
369 + 11008,
370 + 11264,
371 + 13824,
372 + 14336,
373 + 14784,
374 + 14848,
375 + 15360,
376 + 18944,
377 + 22016,
378 + 22528,
379 + 24576,
380 + 27392,
381 + 27648,
382 + 29568,
383 + 29696,
384 + 32000,
385 + 32256,
386 + 32512,
387 + 32768,
388 + 33024,
389 + 36864,
390 + 43264,
391 + 49152,
392 + 49408,
393 + 60544,
394 + 60672,
395 + 64000,
396 + 64256,
397 + 102400,
398 + 102656,
399 + 128000,
400 + 128256,
401 +]
402 +#The size of TP
403 +divisibility = [1, 2, 8, 16, 64]
404 +
405 +all_hidden_size = []
406 +for div in divisibility:
407 + for hidden_size in HIDDEN_SIZES:
408 + all_hidden_size.append(hidden_size // div)
409 +
410 +HIDDEN_SIZES = list(set(all_hidden_size))
411 +
412 +# Test params that focuses on hidden_size variation.
413 +hs_test_params = {
414 + "hidden_sizes": HIDDEN_SIZES,
415 + "batches": [4],
416 + "num_loras": [4],
417 + "max_ranks": [32],
418 +}
419 +
420 +# General tests params that tests for variations in all dimensions
421 +# except hidden_size.
422 +test_params = {
423 + "hidden_sizes": [2049],
424 + "batches": [1, 4, 16, 32],
425 + "num_loras": [1, 8, 32, 128],
426 + "max_ranks": [1, 4, 8, 16, 32, 64, 128, 256],
427 +}
428 +
429 +DTYPES = [torch.float16, torch.bfloat16]
430 +DEVICES = [f"cuda:{0}"]
431 +SEED = [0]
432 +
433 +
434 +@pytest.mark.parametrize("batches", test_params['batches'])
435 +@pytest.mark.parametrize("num_loras", test_params['num_loras'])
436 +@pytest.mark.parametrize("rank", test_params['max_ranks'])
437 +@pytest.mark.parametrize("hidden_size", test_params['hidden_sizes'])
438 +@pytest.mark.parametrize("nslices", [1, 2, 3])
439 +@pytest.mark.parametrize("dtype", DTYPES)
440 +@pytest.mark.parametrize("device", DEVICES)
441 +@pytest.mark.parametrize("seed", SEED)
442 +@pytest.mark.parametrize("op_type", ["shrink", "expand"])
443 +def test_punica_sgmv(
444 + batches: int,
445 + num_loras: int,
446 + rank: int,
447 + hidden_size: int,
448 + nslices: int,
449 + dtype: torch.dtype,
450 + device: str,
451 + seed: int,
452 + op_type: str,
453 +):
454 + torch.set_default_device(device)
455 + current_platform.seed_everything(seed)
456 +
457 + if op_type == "shrink":
458 + check_sgmv_shrink(batches=batches,
459 + num_loras=num_loras,
460 + rank=rank,
461 + hidden_size=hidden_size,
462 + nslices=nslices,
463 + dtype=dtype,
464 + device=device,
465 + seq_length=128,
466 + scaling=0.5)
467 + else:
468 + check_sgmv_expand(batches=batches,
469 + num_loras=num_loras,
470 + rank=rank,
471 + hidden_size=hidden_size,
472 + nslices=nslices,
473 + dtype=dtype,
474 + device=device,
475 + seq_length=128,
476 + add_inputs=True)
477 +
478 +
479 +@pytest.mark.parametrize("batches", hs_test_params['batches'])
480 +@pytest.mark.parametrize("num_loras", hs_test_params['num_loras'])
481 +@pytest.mark.parametrize("rank", hs_test_params['max_ranks'])
482 +@pytest.mark.parametrize("hidden_size", hs_test_params['hidden_sizes'])
483 +@pytest.mark.parametrize("nslices", [1, 2, 3])
484 +@pytest.mark.parametrize("dtype", DTYPES)
485 +@pytest.mark.parametrize("device", DEVICES)
486 +@pytest.mark.parametrize("seed", SEED)
487 +@pytest.mark.parametrize("op_type", ["shrink", "expand"])
488 +def test_punica_sgmv_hidden_size(
489 + batches: int,
490 + num_loras: int,
491 + rank: int,
492 + hidden_size: int,
493 + nslices: int,
494 + dtype: torch.dtype,
495 + device: str,
496 + seed: int,
497 + op_type: str,
498 +):
499 + torch.set_default_device(device)
500 + current_platform.seed_everything(seed)
501 +
502 + if op_type == "shrink":
503 + check_sgmv_shrink(batches=batches,
504 + num_loras=num_loras,
505 + rank=rank,
506 + hidden_size=hidden_size,
507 + nslices=nslices,
508 + dtype=dtype,
509 + device=device,
510 + seq_length=128,
511 + scaling=0.5)
512 + else:
513 + check_sgmv_expand(batches=batches,
514 + num_loras=num_loras,
515 + rank=rank,
516 + hidden_size=hidden_size,
517 + nslices=nslices,
518 + dtype=dtype,
519 + device=device,
520 + seq_length=128,
521 + add_inputs=True)
522 +
523 +
524 +@pytest.mark.parametrize("batches", test_params['batches'])
525 +@pytest.mark.parametrize("num_loras", test_params['num_loras'])
526 +@pytest.mark.parametrize("rank", test_params['max_ranks'])
527 +@pytest.mark.parametrize("hidden_size", test_params['hidden_sizes'])
528 +@pytest.mark.parametrize("dtype", DTYPES)
529 +@pytest.mark.parametrize("device", DEVICES)
530 +@pytest.mark.parametrize("seed", SEED)
531 +@pytest.mark.parametrize("op_type", ["shrink", "expand"])
532 +def test_punica_bgmv(
533 + batches: int,
534 + num_loras: int,
535 + rank: int,
536 + hidden_size: int,
537 + dtype: torch.dtype,
538 + device: str,
539 + seed: int,
540 + op_type: str,
541 +):
542 + torch.set_default_device(device)
543 + current_platform.seed_everything(seed)
544 +
545 + if op_type == "shrink":
546 + check_bgmv_shrink(batches=batches,
547 + num_loras=num_loras,
548 + rank=rank,
549 + hidden_size=hidden_size,
550 + dtype=dtype,
551 + device=device,
552 + scaling=0.5)
553 + else:
554 + check_bgmv_expand(batches=batches,
555 + num_loras=num_loras,
556 + rank=rank,
557 + hidden_size=hidden_size,
558 + dtype=dtype,
559 + device=device,
560 + add_inputs=True)
561 +
562 +
563 +@pytest.mark.parametrize("batches", hs_test_params['batches'])
564 +@pytest.mark.parametrize("num_loras", hs_test_params['num_loras'])
565 +@pytest.mark.parametrize("rank", hs_test_params['max_ranks'])
566 +@pytest.mark.parametrize("hidden_size", hs_test_params['hidden_sizes'])
567 +@pytest.mark.parametrize("dtype", DTYPES)
568 +@pytest.mark.parametrize("device", DEVICES)
569 +@pytest.mark.parametrize("seed", SEED)
570 +@pytest.mark.parametrize("op_type", ["shrink", "expand"])
571 +def test_punica_bgmv_hidden_size(
572 + batches: int,
573 + num_loras: int,
574 + rank: int,
575 + hidden_size: int,
576 + dtype: torch.dtype,
577 + device: str,
578 + seed: int,
579 + op_type: str,
580 +):
581 + torch.set_default_device(device)
582 + current_platform.seed_everything(seed)
583 +
584 + if op_type == "shrink":
585 + check_bgmv_shrink(batches=batches,
586 + num_loras=num_loras,
587 + rank=rank,
588 + hidden_size=hidden_size,
589 + dtype=dtype,
590 + device=device,
591 + scaling=0.5)
592 + else:
593 + check_bgmv_expand(batches=batches,
594 + num_loras=num_loras,
595 + rank=rank,
596 + hidden_size=hidden_size,
597 + dtype=dtype,
598 + device=device,
599 + add_inputs=True)
600 +
601 +
602 +@pytest.mark.parametrize("batches", test_params['batches'])
603 +@pytest.mark.parametrize("num_loras", test_params['num_loras'])
604 +@pytest.mark.parametrize("rank", test_params['max_ranks'])
605 +@pytest.mark.parametrize("hidden_size", test_params['hidden_sizes'])
606 +@pytest.mark.parametrize("nslices", [2, 3])
607 +@pytest.mark.parametrize("dtype", DTYPES)
608 +@pytest.mark.parametrize("device", DEVICES)
609 +@pytest.mark.parametrize("seed", SEED)
610 +def test_punica_bgmv_expand_nslices(batches: int, num_loras: int, rank: int,
611 + hidden_size: int, nslices: int,
612 + dtype: torch.dtype, device: str,
613 + seed: int):
614 +
615 + torch.set_default_device(device)
616 + current_platform.seed_everything(seed)
617 +
618 + check_bgmv_expand_slice(batches=batches,
619 + num_loras=num_loras,
620 + rank=rank,
621 + hidden_size=hidden_size,
622 + nslices=nslices,
623 + dtype=dtype,
624 + device=device,
625 + add_inputs=True)
626 +
627 +
628 +@pytest.mark.parametrize("batches", hs_test_params['batches'])
629 +@pytest.mark.parametrize("num_loras", hs_test_params['num_loras'])
630 +@pytest.mark.parametrize("rank", hs_test_params['max_ranks'])
631 +@pytest.mark.parametrize("hidden_size", hs_test_params['hidden_sizes'])
632 +@pytest.mark.parametrize("nslices", [2, 3])
633 +@pytest.mark.parametrize("dtype", DTYPES)
634 +@pytest.mark.parametrize("device", DEVICES)
635 +@pytest.mark.parametrize("seed", SEED)
636 +def test_punica_bgmv_expand_nslices_hidden_size(batches: int, num_loras: int,
637 + rank: int, hidden_size: int,
638 + nslices: int,
639 + dtype: torch.dtype,
640 + device: str, seed: int):
641 +
642 + torch.set_default_device(device)
643 + current_platform.seed_everything(seed)
644 +
645 + check_bgmv_expand_slice(batches=batches,
646 + num_loras=num_loras,
647 + rank=rank,
648 + hidden_size=hidden_size,
649 + nslices=nslices,
650 + dtype=dtype,
651 + device=device,
652 + add_inputs=True)
tests/lora/test_punica_ops_sizes.py deleted
-401
@@ -1,401 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -"""
3 -This script is mainly used to tests various hidden_sizes. We have collected the
4 -hidden_sizes included in the LoRA models currently supported by vLLM. It tests
5 -whether the corresponding Triton kernel can run normally when tensor parallelism
6 -is set to [1, 2, 4, 8, 16, 32, 64].
7 -"""
8 -from threading import Lock
9 -
10 -import pytest
11 -import torch
12 -
13 -import vllm.lora.ops.triton_ops # noqa: F401
14 -from vllm.lora.ops.torch_ops import (bgmv_expand, bgmv_expand_slice,
15 - bgmv_shrink, sgmv_expand,
16 - sgmv_expand_slice, sgmv_shrink)
17 -from vllm.lora.ops.triton_ops.utils import _LORA_A_PTR_DICT, _LORA_B_PTR_DICT
18 -from vllm.platforms import current_platform
19 -
20 -from .utils import (assert_close, generate_data,
21 - generate_data_for_expand_nslices,
22 - generate_data_for_nslices)
23 -
24 -HIDDEN_SIZES = [
25 - 128,
26 - 256,
27 - 512,
28 - 896,
29 - 1024,
30 - 1152,
31 - 1216,
32 - 1280,
33 - 1536,
34 - 1664,
35 - 2048,
36 - 2240,
37 - 2304,
38 - 2368,
39 - 2432,
40 - 2560,
41 - 2752,
42 - 3072,
43 - 3328,
44 - 3456,
45 - 3584,
46 - 3712,
47 - 4096,
48 - 4480,
49 - 4608,
50 - 4736,
51 - 4864,
52 - 5120,
53 - 5504,
54 - 5632,
55 - 5888,
56 - 6144,
57 - 6400,
58 - 6848,
59 - 6912,
60 - 7168,
61 - 7424,
62 - 8192,
63 - 8960,
64 - 9216,
65 - 9472,
66 - 10240,
67 - 11008,
68 - 11264,
69 - 13824,
70 - 14336,
71 - 14784,
72 - 14848,
73 - 15360,
74 - 18944,
75 - 22016,
76 - 22528,
77 - 24576,
78 - 27392,
79 - 27648,
80 - 29568,
81 - 29696,
82 - 32000,
83 - 32256,
84 - 32512,
85 - 32768,
86 - 33024,
87 - 36864,
88 - 43264,
89 - 49152,
90 - 49408,
91 - 60544,
92 - 60672,
93 - 64000,
94 - 64256,
95 - 102400,
96 - 102656,
97 - 128000,
98 - 128256,
99 -]
100 -#The size of TP
101 -divisibility = [1, 2, 8, 16, 64]
102 -
103 -all_hidden_size = []
104 -for div in divisibility:
105 - for hidden_size in HIDDEN_SIZES:
106 - all_hidden_size.append(hidden_size // div)
107 -
108 -HIDDEN_SIZES = list(set(all_hidden_size))
109 -
110 -BATCHES = [4]
111 -NUM_LORA = [4]
112 -DTYPES = [torch.float16, torch.bfloat16]
113 -MAX_RANKS = [32]
114 -SCALES = [0.5]
115 -SEED = [0]
116 -DEVICES = [f"cuda:{0}"]
117 -
118 -_dict_lock = Lock()
119 -
120 -
121 -@pytest.mark.parametrize("batches", BATCHES)
122 -@pytest.mark.parametrize("num_loras", NUM_LORA)
123 -@pytest.mark.parametrize("rank", MAX_RANKS)
124 -@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
125 -@pytest.mark.parametrize("scaling", SCALES)
126 -@pytest.mark.parametrize("nslices", [1, 2, 3])
127 -@pytest.mark.parametrize("dtype", DTYPES)
128 -@pytest.mark.parametrize("op_type", ["shrink", "expand"])
129 -@pytest.mark.parametrize("seed", SEED)
130 -@pytest.mark.parametrize("device", DEVICES)
131 -def test_punica_sgmv(
132 - batches: int,
133 - num_loras: int,
134 - rank: int,
135 - hidden_size: int,
136 - scaling: float,
137 - nslices: int,
138 - dtype: torch.dtype,
139 - op_type: str,
140 - seed: int,
141 - device: str,
142 -):
143 - torch.set_default_device(device)
144 - current_platform.seed_everything(seed)
145 -
146 - seq_length = 128
147 - (
148 - inputs_tensor,
149 - lora_weights_lst,
150 - our_out_tensor,
151 - ref_out_tensor,
152 - b_seq_start_loc,
153 - lora_indices_tensor,
154 - seq_len_tensor,
155 - indices,
156 - ) = generate_data_for_nslices(
157 - batches,
158 - hidden_size,
159 - num_loras,
160 - rank,
161 - seq_length,
162 - nslices,
163 - dtype,
164 - op_type,
165 - device,
166 - )
167 - max_seq_length = seq_len_tensor.max()
168 - token_nums = seq_len_tensor.sum().item()
169 - if isinstance(max_seq_length, tuple):
170 - max_seq_length = max_seq_length[0].item()
171 - else:
172 - max_seq_length = max_seq_length.item()
173 - if op_type == "shrink":
174 - # Preventing cache error pointer.
175 - with _dict_lock:
176 - _LORA_A_PTR_DICT.clear()
177 - torch.ops.vllm.sgmv_shrink(
178 - inputs_tensor,
179 - lora_weights_lst,
180 - our_out_tensor,
181 - b_seq_start_loc,
182 - seq_len_tensor,
183 - lora_indices_tensor,
184 - batches,
185 - max_seq_length,
186 - token_nums,
187 - scaling,
188 - )
189 - for index in range(nslices):
190 - sgmv_shrink(
191 - inputs_tensor,
192 - lora_weights_lst[index],
193 - ref_out_tensor[index],
194 - b_seq_start_loc,
195 - seq_len_tensor,
196 - lora_indices_tensor,
197 - batches,
198 - max_seq_length,
199 - token_nums,
200 - scaling,
201 - )
202 -
203 - else:
204 - with _dict_lock:
205 - _LORA_B_PTR_DICT.clear()
206 - torch.ops.vllm.sgmv_expand(
207 - inputs_tensor,
208 - lora_weights_lst,
209 - our_out_tensor,
210 - b_seq_start_loc,
211 - seq_len_tensor,
212 - lora_indices_tensor,
213 - batches,
214 - max_seq_length,
215 - token_nums,
216 - offset_start=0,
217 - add_inputs=True,
218 - )
219 - if nslices == 1:
220 - # Verify the torch's sgmv_expand op
221 - sgmv_expand(
222 - inputs_tensor[0],
223 - lora_weights_lst[0],
224 - ref_out_tensor,
225 - b_seq_start_loc,
226 - seq_len_tensor,
227 - lora_indices_tensor,
228 - batches,
229 - max_seq_length,
230 - token_nums,
231 - add_inputs=True,
232 - )
233 - else:
234 - slice_offset = 0
235 - for index in range(nslices):
236 - lora_weights = lora_weights_lst[index]
237 - sgmv_expand_slice(
238 - inputs_tensor[index],
239 - lora_weights,
240 - ref_out_tensor,
241 - b_seq_start_loc,
242 - seq_len_tensor,
243 - lora_indices_tensor,
244 - batches,
245 - max_seq_length,
246 - token_nums,
247 - slice_offset,
248 - hidden_size,
249 - add_inputs=True,
250 - )
251 - slice_offset += hidden_size
252 -
253 - assert_close(our_out_tensor, ref_out_tensor)
254 -
255 -
256 -@pytest.mark.parametrize("batches", BATCHES)
257 -@pytest.mark.parametrize("num_loras", NUM_LORA)
258 -@pytest.mark.parametrize("rank", MAX_RANKS)
259 -@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
260 -@pytest.mark.parametrize("scaling", SCALES)
261 -@pytest.mark.parametrize("dtype", DTYPES)
262 -@pytest.mark.parametrize("op_type", ["shrink", "expand"])
263 -@pytest.mark.parametrize("seed", SEED)
264 -@pytest.mark.parametrize("device", DEVICES)
265 -def test_punica_bgmv(
266 - batches: int,
267 - num_loras: int,
268 - rank: int,
269 - hidden_size: int,
270 - scaling: float,
271 - dtype: torch.dtype,
272 - op_type: str,
273 - seed: int,
274 - device: str,
275 -):
276 - torch.set_default_device(device)
277 - current_platform.seed_everything(seed)
278 -
279 - seq_length = 1
280 - (
281 - inputs_tensor,
282 - lora_weights,
283 - our_out_tensor,
284 - ref_out_tensor,
285 - b_seq_start_loc,
286 - lora_indices_tensor,
287 - seq_len_tensor,
288 - indices,
289 - ) = generate_data(
290 - batches,
291 - hidden_size,
292 - num_loras,
293 - rank,
294 - seq_length,
295 - dtype,
296 - op_type,
297 - device,
298 - )
299 - if op_type == "shrink":
300 - torch.ops.vllm.bgmv_shrink(
301 - inputs_tensor,
302 - lora_weights,
303 - our_out_tensor,
304 - indices,
305 - scaling,
306 - )
307 -
308 - bgmv_shrink(
309 - inputs_tensor,
310 - lora_weights,
311 - ref_out_tensor,
312 - indices,
313 - scaling,
314 - )
315 -
316 - else:
317 - torch.ops.vllm.bgmv_expand(
318 - inputs_tensor,
319 - lora_weights,
320 - our_out_tensor,
321 - indices,
322 - add_inputs=True,
323 - )
324 - bgmv_expand(
325 - inputs_tensor,
326 - lora_weights,
327 - ref_out_tensor,
328 - indices,
329 - add_inputs=True,
330 - )
331 -
332 - if op_type == "shrink":
333 - ref_out_tensor = ref_out_tensor.to(torch.float32)
334 - assert_close(our_out_tensor, ref_out_tensor)
335 -
336 -
337 -@pytest.mark.parametrize("batches", BATCHES)
338 -@pytest.mark.parametrize("num_loras", NUM_LORA)
339 -@pytest.mark.parametrize("rank", MAX_RANKS)
340 -@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
341 -@pytest.mark.parametrize("nslices", [2, 3])
342 -@pytest.mark.parametrize("dtype", DTYPES)
343 -@pytest.mark.parametrize("seed", SEED)
344 -@pytest.mark.parametrize("device", DEVICES)
345 -def test_punica_bgmv_expand_nslices(
346 - batches: int,
347 - num_loras: int,
348 - rank: int,
349 - hidden_size: int,
350 - nslices: int,
351 - dtype: torch.dtype,
352 - seed: int,
353 - device: str,
354 -):
355 - torch.set_default_device(device)
356 - current_platform.seed_everything(seed)
357 -
358 - seq_length = 1
359 - (
360 - inputs_tensor,
361 - lora_weights_lst,
362 - our_outputs,
363 - ref_outputs,
364 - b_seq_start_loc,
365 - lora_indices_tensor,
366 - seq_len_tensor,
367 - indices,
368 - ) = generate_data_for_expand_nslices(
369 - batches,
370 - hidden_size,
371 - num_loras,
372 - rank,
373 - seq_length,
374 - dtype,
375 - nslices,
376 - device,
377 - )
378 - slice_offset = 0
379 - for index in range(nslices):
380 - lora_weights = lora_weights_lst[index]
381 - torch.ops.vllm.bgmv_expand_slice(
382 - inputs_tensor,
383 - lora_weights,
384 - our_outputs,
385 - indices,
386 - slice_offset,
387 - slice_size=hidden_size,
388 - add_inputs=True,
389 - )
390 - bgmv_expand_slice(
391 - inputs_tensor,
392 - lora_weights,
393 - ref_outputs,
394 - indices,
395 - slice_offset,
396 - slice_size=hidden_size,
397 - add_inputs=True,
398 - )
399 -
400 - slice_offset += hidden_size
401 - assert_close(our_outputs, ref_outputs)
tests/lora/test_punica_ops_variation.py deleted
-317
@@ -1,317 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -"""
3 -This script is mainly used to test whether trtion kernels can run normally
4 -under different conditions, including various batches, numbers of LoRA , and
5 -maximum ranks.
6 -"""
7 -from threading import Lock
8 -
9 -import pytest
10 -import torch
11 -
12 -# Enable custom op register
13 -import vllm.lora.ops.triton_ops # noqa: F401
14 -from vllm.lora.ops.torch_ops import (bgmv_expand, bgmv_expand_slice,
15 - bgmv_shrink, sgmv_expand,
16 - sgmv_expand_slice, sgmv_shrink)
17 -from vllm.lora.ops.triton_ops.utils import _LORA_A_PTR_DICT, _LORA_B_PTR_DICT
18 -from vllm.platforms import current_platform
19 -
20 -from .utils import (assert_close, generate_data,
21 - generate_data_for_expand_nslices,
22 - generate_data_for_nslices)
23 -
24 -HIDDEN_SIZES = [2049]
25 -
26 -BATCHES = [1, 4, 16, 32]
27 -NUM_LORA = [1, 8, 32, 128]
28 -DTYPES = [torch.float16, torch.bfloat16]
29 -MAX_RANKS = [1, 4, 8, 16, 32, 64, 128, 256]
30 -SCALES = [0.5]
31 -SEED = [0]
32 -DEVICES = [f"cuda:{0}"]
33 -
34 -_dict_lock = Lock()
35 -
36 -
37 -@pytest.mark.parametrize("batches", BATCHES)
38 -@pytest.mark.parametrize("num_loras", NUM_LORA)
39 -@pytest.mark.parametrize("rank", MAX_RANKS)
40 -@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
41 -@pytest.mark.parametrize("scaling", SCALES)
42 -@pytest.mark.parametrize("nslices", [1, 2, 3])
43 -@pytest.mark.parametrize("dtype", DTYPES)
44 -@pytest.mark.parametrize("op_type", ["shrink", "expand"])
45 -@pytest.mark.parametrize("seed", SEED)
46 -@pytest.mark.parametrize("device", DEVICES)
47 -def test_punica_sgmv(
48 - batches: int,
49 - num_loras: int,
50 - rank: int,
51 - hidden_size: int,
52 - scaling: float,
53 - nslices: int,
54 - dtype: torch.dtype,
55 - op_type: str,
56 - seed: int,
57 - device: str,
58 -):
59 - torch.set_default_device(device)
60 - current_platform.seed_everything(seed)
61 -
62 - seq_length = 128
63 - (
64 - inputs_tensor,
65 - lora_weights_lst,
66 - our_out_tensor,
67 - ref_out_tensor,
68 - b_seq_start_loc,
69 - lora_indices_tensor,
70 - seq_len_tensor,
71 - indices,
72 - ) = generate_data_for_nslices(
73 - batches,
74 - hidden_size,
75 - num_loras,
76 - rank,
77 - seq_length,
78 - nslices,
79 - dtype,
80 - op_type,
81 - device,
82 - )
83 - max_seq_length = seq_len_tensor.max()
84 - token_nums = seq_len_tensor.sum().item()
85 - if isinstance(max_seq_length, tuple):
86 - max_seq_length = max_seq_length[0].item()
87 - else:
88 - max_seq_length = max_seq_length.item()
89 - if op_type == "shrink":
90 - # Preventing cache error pointer.
91 - with _dict_lock:
92 - _LORA_A_PTR_DICT.clear()
93 - torch.ops.vllm.sgmv_shrink(
94 - inputs_tensor,
95 - lora_weights_lst,
96 - our_out_tensor,
97 - b_seq_start_loc,
98 - seq_len_tensor,
99 - lora_indices_tensor,
100 - batches,
101 - max_seq_length,
102 - token_nums,
103 - scaling,
104 - )
105 - for index in range(nslices):
106 - sgmv_shrink(
107 - inputs_tensor,
108 - lora_weights_lst[index],
109 - ref_out_tensor[index],
110 - b_seq_start_loc,
111 - seq_len_tensor,
112 - lora_indices_tensor,
113 - batches,
114 - max_seq_length,
115 - token_nums,
116 - scaling,
117 - )
118 -
119 - else:
120 - with _dict_lock:
121 - _LORA_B_PTR_DICT.clear()
122 - torch.ops.vllm.sgmv_expand(
123 - inputs_tensor,
124 - lora_weights_lst,
125 - our_out_tensor,
126 - b_seq_start_loc,
127 - seq_len_tensor,
128 - lora_indices_tensor,
129 - batches,
130 - max_seq_length,
131 - token_nums,
132 - offset_start=0,
133 - add_inputs=True,
134 - )
135 - slice_offset = 0
136 - if nslices == 1:
137 - # Verify the torch's sgmv_expand op
138 - sgmv_expand(
139 - inputs_tensor[0],
140 - lora_weights_lst[0],
141 - ref_out_tensor,
142 - b_seq_start_loc,
143 - seq_len_tensor,
144 - lora_indices_tensor,
145 - batches,
146 - max_seq_length,
147 - token_nums,
148 - add_inputs=True,
149 - )
150 - else:
151 - for index in range(nslices):
152 - lora_weights = lora_weights_lst[index]
153 - sgmv_expand_slice(
154 - inputs_tensor[index],
155 - lora_weights,
156 - ref_out_tensor,
157 - b_seq_start_loc,
158 - seq_len_tensor,
159 - lora_indices_tensor,
160 - batches,
161 - max_seq_length,
162 - token_nums,
163 - slice_offset,
164 - hidden_size,
165 - add_inputs=True,
166 - )
167 - slice_offset += hidden_size
168 -
169 - assert_close(our_out_tensor, ref_out_tensor)
170 -
171 -
172 -@pytest.mark.parametrize("batches", BATCHES)
173 -@pytest.mark.parametrize("num_loras", NUM_LORA)
174 -@pytest.mark.parametrize("rank", MAX_RANKS)
175 -@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
176 -@pytest.mark.parametrize("scaling", SCALES)
177 -@pytest.mark.parametrize("dtype", DTYPES)
178 -@pytest.mark.parametrize("op_type", ["shrink", "expand"])
179 -@pytest.mark.parametrize("seed", SEED)
180 -@pytest.mark.parametrize("device", DEVICES)
181 -def test_punica_bgmv(
182 - batches: int,
183 - num_loras: int,
184 - rank: int,
185 - hidden_size: int,
186 - scaling: float,
187 - dtype: torch.dtype,
188 - op_type: str,
189 - seed: int,
190 - device: str,
191 -):
192 - torch.set_default_device(device)
193 - current_platform.seed_everything(seed)
194 -
195 - seq_length = 1
196 - (
197 - inputs_tensor,
198 - lora_weights,
199 - our_out_tensor,
200 - ref_out_tensor,
201 - b_seq_start_loc,
202 - lora_indices_tensor,
203 - seq_len_tensor,
204 - indices,
205 - ) = generate_data(
206 - batches,
207 - hidden_size,
208 - num_loras,
209 - rank,
210 - seq_length,
211 - dtype,
212 - op_type,
213 - device,
214 - )
215 - if op_type == "shrink":
216 - torch.ops.vllm.bgmv_shrink(
217 - inputs_tensor,
218 - lora_weights,
219 - our_out_tensor,
220 - indices,
221 - scaling,
222 - )
223 -
224 - bgmv_shrink(
225 - inputs_tensor,
226 - lora_weights,
227 - ref_out_tensor,
228 - indices,
229 - scaling,
230 - )
231 -
232 - else:
233 - torch.ops.vllm.bgmv_expand(
234 - inputs_tensor,
235 - lora_weights,
236 - our_out_tensor,
237 - indices,
238 - add_inputs=True,
239 - )
240 - bgmv_expand(
241 - inputs_tensor,
242 - lora_weights,
243 - ref_out_tensor,
244 - indices,
245 - add_inputs=True,
246 - )
247 -
248 - if op_type == "shrink":
249 - ref_out_tensor = ref_out_tensor.to(torch.float32)
250 - assert_close(our_out_tensor, ref_out_tensor)
251 -
252 -
253 -@pytest.mark.parametrize("batches", BATCHES)
254 -@pytest.mark.parametrize("num_loras", NUM_LORA)
255 -@pytest.mark.parametrize("rank", MAX_RANKS)
256 -@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
257 -@pytest.mark.parametrize("nslices", [2, 3])
258 -@pytest.mark.parametrize("dtype", DTYPES)
259 -@pytest.mark.parametrize("seed", SEED)
260 -@pytest.mark.parametrize("device", DEVICES)
261 -def test_punica_bgmv_expand_nslices(
262 - batches: int,
263 - num_loras: int,
264 - rank: int,
265 - hidden_size: int,
266 - nslices: int,
267 - dtype: torch.dtype,
268 - seed: int,
269 - device: str,
270 -):
271 - torch.set_default_device(device)
272 - current_platform.seed_everything(seed)
273 -
274 - seq_length = 1
275 - (
276 - inputs_tensor,
277 - lora_weights_lst,
278 - our_outputs,
279 - ref_outputs,
280 - b_seq_start_loc,
281 - lora_indices_tensor,
282 - seq_len_tensor,
283 - indices,
284 - ) = generate_data_for_expand_nslices(
285 - batches,
286 - hidden_size,
287 - num_loras,
288 - rank,
289 - seq_length,
290 - dtype,
291 - nslices,
292 - device,
293 - )
294 - slice_offset = 0
295 - for index in range(nslices):
296 - lora_weights = lora_weights_lst[index]
297 - torch.ops.vllm.bgmv_expand_slice(
298 - inputs_tensor,
299 - lora_weights,
300 - our_outputs,
301 - indices,
302 - slice_offset,
303 - slice_size=hidden_size,
304 - add_inputs=True,
305 - )
306 - bgmv_expand_slice(
307 - inputs_tensor,
308 - lora_weights,
309 - ref_outputs,
310 - indices,
311 - slice_offset,
312 - slice_size=hidden_size,
313 - add_inputs=True,
314 - )
315 -
316 - slice_offset += hidden_size
317 - assert_close(our_outputs, ref_outputs)
tests/lora/utils.py
+34 -7
@@ -1,6 +1,7 @@
1 # SPDX-License-Identifier: Apache-2.0
2
3 -from typing import Dict, List, Optional
3 +from dataclasses import dataclass
4 +from typing import Dict, List, Optional, Tuple, Union
5
6 import torch
7
@@ -106,6 +107,31 @@ def assert_close(a, b):
107 torch.testing.assert_close(a, b, rtol=rtol, atol=atol)
108
109
110 +@dataclass
111 +class PunicaTensors:
112 + inputs_tensor: torch.Tensor
113 + lora_weights: Union[torch.Tensor, List[torch.Tensor]]
114 + our_out_tensor: torch.Tensor
115 + ref_out_tensor: torch.Tensor
116 + b_seq_start_loc: torch.Tensor
117 + prompt_lora_mapping: torch.Tensor
118 + seq_len_tensor: torch.Tensor
119 + token_lora_mapping: torch.Tensor
120 +
121 + def meta(self) -> Tuple[int, int]:
122 + """
123 + Infer max_seq_length and token_nums from the tensors
124 + and return them.
125 + """
126 + max_seq_length = self.seq_len_tensor.max()
127 + token_nums = self.seq_len_tensor.sum().item()
128 + if isinstance(max_seq_length, tuple):
129 + max_seq_length = max_seq_length[0].item()
130 + else:
131 + max_seq_length = max_seq_length.item()
132 + return max_seq_length, token_nums
133 +
134 +
135 def generate_data(
136 batches,
137 hidden_size,
@@ -115,7 +141,7 @@ def generate_data(
141 dtype,
142 op_type,
143 device,
118 -):
144 +) -> PunicaTensors:
145 seq_len_tensor = torch.randint(seq_length, seq_length + 1,
146 (batches, )).to(device)
147 b_seq_start_loc = torch.cumsum(
@@ -164,7 +190,8 @@ def generate_data(
190 indices[current_offset:current_offset +
191 seq_len_tensor[b_id]].copy_(lora_index)
192 current_offset += seq_len_tensor[b_id].item()
167 - return (
193 +
194 + return PunicaTensors(
195 inputs_tensor,
196 lora_weights,
197 our_out_tensor,
@@ -185,7 +212,7 @@ def generate_data_for_expand_nslices(
212 dtype,
213 nslices,
214 device,
188 -):
215 +) -> PunicaTensors:
216 seq_len_tensor = torch.randint(seq_length, seq_length + 1,
217 (batches, )).to(device)
218 b_seq_start_loc = torch.cumsum(
@@ -222,7 +249,7 @@ def generate_data_for_expand_nslices(
249 current_offset += seq_len_tensor[b_id].item()
250
251 lora_indices_tensor = lora_indices_tensor.to(device)
225 - return (
252 + return PunicaTensors(
253 inputs_tensor,
254 lora_weights_lst,
255 our_out_tensor,
@@ -244,7 +271,7 @@ def generate_data_for_nslices(
271 dtype,
272 op_type,
273 device,
247 -):
274 +) -> PunicaTensors:
275 seq_len_tensor = torch.randint(seq_length, seq_length + 1,
276 (batches, )).to(device)
277 b_seq_start_loc = torch.cumsum(
@@ -302,7 +329,7 @@ def generate_data_for_nslices(
329 current_offset += seq_len_tensor[b_id].item()
330
331 lora_indices_tensor = lora_indices_tensor.to(device)
305 - return (
332 + return PunicaTensors(
333 inputs_tensor,
334 lora_weights_lst,
335 our_out_tensor,