A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[Misc] LoRA - Refactor Punica ops tests (#12970)
Signed-off-by: Varun Sundar Rabindranath <varun@neuralmagic.com> Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com>
Varun Sundar Rabindranath committed
Feb 11, 2025 at 12:56 UTC
78a141d768a18edc8c598a57d992e6aa56a33259
4 files changed
+686
-725
tests/lora/test_punica_ops.py
new
+652
@@ -0,0 +1,652 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+from threading import Lock
3
+from typing import List
4
+
5
+import pytest
6
+import torch
7
+
8
+import vllm.lora.ops.triton_ops # noqa: F401
9
+from vllm.lora.ops.torch_ops import (bgmv_expand, bgmv_expand_slice,
10
+ bgmv_shrink, sgmv_expand,
11
+ sgmv_expand_slice, sgmv_shrink)
12
+from vllm.lora.ops.triton_ops.utils import _LORA_A_PTR_DICT, _LORA_B_PTR_DICT
13
+from vllm.platforms import current_platform
14
+
15
+from .utils import (PunicaTensors, assert_close, generate_data,
16
+ generate_data_for_expand_nslices,
17
+ generate_data_for_nslices)
18
+
19
+
20
+# Utility shrink and expand operations used as reference implementations.
21
+def sgmv_shrink_for_nslices(
22
+ nslices: int, inputs_tensor: torch.Tensor,
23
+ lora_weights_lst: List[torch.Tensor], out_tensor: torch.Tensor,
24
+ b_seq_start_loc: torch.Tensor, seq_len_tensor: torch.Tensor,
25
+ prompt_lora_mapping: torch.Tensor, batches: int, max_seq_length: int,
26
+ num_tokens: int, scaling: float):
27
+ """
28
+ Wrapper around sgmv_shrink that handles any nslices.
29
+ """
30
+ for index in range(nslices):
31
+ sgmv_shrink(
32
+ inputs_tensor,
33
+ lora_weights_lst[index],
34
+ out_tensor[index],
35
+ b_seq_start_loc,
36
+ seq_len_tensor,
37
+ prompt_lora_mapping,
38
+ batches,
39
+ max_seq_length,
40
+ num_tokens,
41
+ scaling,
42
+ )
43
+
44
+
45
+def sgmv_expand_for_nslices(nslices: int, hidden_size: int,
46
+ inputs_tensor: torch.Tensor,
47
+ lora_weights_lst: List[torch.Tensor],
48
+ out_tensor: torch.Tensor,
49
+ b_seq_start_loc: torch.Tensor,
50
+ seq_len_tensor: torch.Tensor,
51
+ prompt_lora_mapping: torch.Tensor, batches: int,
52
+ max_seq_length: int, num_tokens: int,
53
+ add_inputs: bool) -> None:
54
+ """
55
+ Wrapper around sgmv_expand that handles any nslices.
56
+ """
57
+ if nslices == 1:
58
+ # Verify the torch's sgmv_expand op
59
+ sgmv_expand(
60
+ inputs_tensor[0],
61
+ lora_weights_lst[0],
62
+ out_tensor,
63
+ b_seq_start_loc,
64
+ seq_len_tensor,
65
+ prompt_lora_mapping,
66
+ batches,
67
+ max_seq_length,
68
+ num_tokens,
69
+ add_inputs=add_inputs,
70
+ )
71
+ else:
72
+ slice_offset = 0
73
+ for index in range(nslices):
74
+ lora_weights = lora_weights_lst[index]
75
+ sgmv_expand_slice(
76
+ inputs_tensor[index],
77
+ lora_weights,
78
+ out_tensor,
79
+ b_seq_start_loc,
80
+ seq_len_tensor,
81
+ prompt_lora_mapping,
82
+ batches,
83
+ max_seq_length,
84
+ num_tokens,
85
+ slice_offset,
86
+ hidden_size,
87
+ add_inputs=add_inputs,
88
+ )
89
+ slice_offset += hidden_size
90
+
91
+
92
+_dict_lock = Lock()
93
+
94
+
95
+def check_sgmv_shrink(batches: int, num_loras: int, rank: int,
96
+ hidden_size: int, nslices: int, dtype: torch.dtype,
97
+ device: str, seq_length: int, scaling: float):
98
+ """
99
+ Compare outputs of vllm.sgmv_shrink kernel against a reference
100
+ implementation.
101
+ """
102
+ data: PunicaTensors = generate_data_for_nslices(
103
+ batches,
104
+ hidden_size,
105
+ num_loras,
106
+ rank,
107
+ seq_length,
108
+ nslices,
109
+ dtype,
110
+ "shrink",
111
+ device,
112
+ )
113
+ max_seq_length, token_nums = data.meta()
114
+
115
+ # Preventing cache error pointer.
116
+ with _dict_lock:
117
+ _LORA_A_PTR_DICT.clear()
118
+ torch.ops.vllm.sgmv_shrink(
119
+ data.inputs_tensor,
120
+ data.lora_weights,
121
+ data.our_out_tensor,
122
+ data.b_seq_start_loc,
123
+ data.seq_len_tensor,
124
+ data.prompt_lora_mapping,
125
+ batches,
126
+ max_seq_length,
127
+ token_nums,
128
+ scaling,
129
+ )
130
+
131
+ sgmv_shrink_for_nslices(
132
+ nslices,
133
+ data.inputs_tensor,
134
+ data.lora_weights,
135
+ data.ref_out_tensor,
136
+ data.b_seq_start_loc,
137
+ data.seq_len_tensor,
138
+ data.prompt_lora_mapping,
139
+ batches,
140
+ max_seq_length,
141
+ token_nums,
142
+ scaling,
143
+ )
144
+ assert_close(data.our_out_tensor, data.ref_out_tensor)
145
+
146
+
147
+def check_sgmv_expand(batches: int, num_loras: int, rank: int,
148
+ hidden_size: int, nslices: int, dtype: torch.dtype,
149
+ device: str, seq_length: int, add_inputs: bool):
150
+ """
151
+ Compare outputs of vllm.sgmv_expand kernel against a reference
152
+ implementation.
153
+ """
154
+ data: PunicaTensors = generate_data_for_nslices(
155
+ batches,
156
+ hidden_size,
157
+ num_loras,
158
+ rank,
159
+ seq_length,
160
+ nslices,
161
+ dtype,
162
+ "expand",
163
+ device,
164
+ )
165
+
166
+ max_seq_length, token_nums = data.meta()
167
+
168
+ with _dict_lock:
169
+ _LORA_B_PTR_DICT.clear()
170
+ torch.ops.vllm.sgmv_expand(
171
+ data.inputs_tensor,
172
+ data.lora_weights,
173
+ data.our_out_tensor,
174
+ data.b_seq_start_loc,
175
+ data.seq_len_tensor,
176
+ data.prompt_lora_mapping,
177
+ batches,
178
+ max_seq_length,
179
+ token_nums,
180
+ offset_start=0,
181
+ add_inputs=add_inputs,
182
+ )
183
+
184
+ sgmv_expand_for_nslices(nslices,
185
+ hidden_size,
186
+ data.inputs_tensor,
187
+ data.lora_weights,
188
+ data.ref_out_tensor,
189
+ data.b_seq_start_loc,
190
+ data.seq_len_tensor,
191
+ data.prompt_lora_mapping,
192
+ batches,
193
+ max_seq_length,
194
+ token_nums,
195
+ add_inputs=add_inputs)
196
+
197
+ assert_close(data.our_out_tensor, data.ref_out_tensor)
198
+
199
+
200
+def check_bgmv_shrink(batches: int, num_loras: int, rank: int,
201
+ hidden_size: int, dtype: torch.dtype, device: str,
202
+ scaling: float):
203
+ """
204
+ Compare vllm.bgmv_shrink against a reference implementation.
205
+ """
206
+ seq_length = 1
207
+ data: PunicaTensors = generate_data(
208
+ batches,
209
+ hidden_size,
210
+ num_loras,
211
+ rank,
212
+ seq_length,
213
+ dtype,
214
+ "shrink",
215
+ device,
216
+ )
217
+
218
+ torch.ops.vllm.bgmv_shrink(
219
+ data.inputs_tensor,
220
+ data.lora_weights,
221
+ data.our_out_tensor,
222
+ data.token_lora_mapping,
223
+ scaling,
224
+ )
225
+
226
+ bgmv_shrink(
227
+ data.inputs_tensor,
228
+ data.lora_weights,
229
+ data.ref_out_tensor,
230
+ data.token_lora_mapping,
231
+ scaling,
232
+ )
233
+
234
+ data.ref_out_tensor = data.ref_out_tensor.to(torch.float32)
235
+ assert_close(data.our_out_tensor, data.ref_out_tensor)
236
+
237
+
238
+def check_bgmv_expand(batches: int, num_loras: int, rank: int,
239
+ hidden_size: int, dtype: torch.dtype, device: str,
240
+ add_inputs: bool):
241
+ """
242
+ Compare vllm.bgmv_expand against a reference implementation.
243
+ """
244
+ seq_length = 1
245
+ data: PunicaTensors = generate_data(
246
+ batches,
247
+ hidden_size,
248
+ num_loras,
249
+ rank,
250
+ seq_length,
251
+ dtype,
252
+ "expand",
253
+ device,
254
+ )
255
+
256
+ torch.ops.vllm.bgmv_expand(
257
+ data.inputs_tensor,
258
+ data.lora_weights,
259
+ data.our_out_tensor,
260
+ data.token_lora_mapping,
261
+ add_inputs=add_inputs,
262
+ )
263
+ bgmv_expand(
264
+ data.inputs_tensor,
265
+ data.lora_weights,
266
+ data.ref_out_tensor,
267
+ data.token_lora_mapping,
268
+ add_inputs=add_inputs,
269
+ )
270
+ assert_close(data.our_out_tensor, data.ref_out_tensor)
271
+
272
+
273
+def check_bgmv_expand_slice(batches: int, num_loras: int, rank: int,
274
+ hidden_size: int, nslices: int, dtype: torch.dtype,
275
+ device: str, add_inputs: bool):
276
+ """
277
+ Compare vllm.bgmv_expand_slice against a reference implementation.
278
+ """
279
+ seq_length = 1
280
+ data: PunicaTensors = generate_data_for_expand_nslices(
281
+ batches,
282
+ hidden_size,
283
+ num_loras,
284
+ rank,
285
+ seq_length,
286
+ dtype,
287
+ nslices,
288
+ device,
289
+ )
290
+
291
+ slice_offset = 0
292
+ for index in range(nslices):
293
+ torch.ops.vllm.bgmv_expand_slice(
294
+ data.inputs_tensor,
295
+ data.lora_weights[index],
296
+ data.our_out_tensor,
297
+ data.token_lora_mapping,
298
+ slice_offset,
299
+ slice_size=hidden_size,
300
+ add_inputs=add_inputs,
301
+ )
302
+ bgmv_expand_slice(
303
+ data.inputs_tensor,
304
+ data.lora_weights[index],
305
+ data.ref_out_tensor,
306
+ data.token_lora_mapping,
307
+ slice_offset,
308
+ slice_size=hidden_size,
309
+ add_inputs=add_inputs,
310
+ )
311
+
312
+ slice_offset += hidden_size
313
+ assert_close(data.our_out_tensor, data.ref_out_tensor)
314
+
315
+
316
+# Tests
317
+# We test the punica kernels along 2 verticals mainly.
318
+# 1. Variations in hidden_dim size
319
+# 2. Variations in all other parameters like (batch_size, max_rank, num_loras
320
+# etc.)
321
+
322
+# We have collected the hidden_sizes included in the LoRA models
323
+# currently supported by vLLM. It tests whether the corresponding Triton
324
+# kernel can run normally when tensor parallelism is set to
325
+# [1, 2, 4, 8, 16, 32, 64].
326
+HIDDEN_SIZES = [
327
+ 128,
328
+ 256,
329
+ 512,
330
+ 896,
331
+ 1024,
332
+ 1152,
333
+ 1216,
334
+ 1280,
335
+ 1536,
336
+ 1664,
337
+ 2048,
338
+ 2240,
339
+ 2304,
340
+ 2368,
341
+ 2432,
342
+ 2560,
343
+ 2752,
344
+ 3072,
345
+ 3328,
346
+ 3456,
347
+ 3584,
348
+ 3712,
349
+ 4096,
350
+ 4480,
351
+ 4608,
352
+ 4736,
353
+ 4864,
354
+ 5120,
355
+ 5504,
356
+ 5632,
357
+ 5888,
358
+ 6144,
359
+ 6400,
360
+ 6848,
361
+ 6912,
362
+ 7168,
363
+ 7424,
364
+ 8192,
365
+ 8960,
366
+ 9216,
367
+ 9472,
368
+ 10240,
369
+ 11008,
370
+ 11264,
371
+ 13824,
372
+ 14336,
373
+ 14784,
374
+ 14848,
375
+ 15360,
376
+ 18944,
377
+ 22016,
378
+ 22528,
379
+ 24576,
380
+ 27392,
381
+ 27648,
382
+ 29568,
383
+ 29696,
384
+ 32000,
385
+ 32256,
386
+ 32512,
387
+ 32768,
388
+ 33024,
389
+ 36864,
390
+ 43264,
391
+ 49152,
392
+ 49408,
393
+ 60544,
394
+ 60672,
395
+ 64000,
396
+ 64256,
397
+ 102400,
398
+ 102656,
399
+ 128000,
400
+ 128256,
401
+]
402
+#The size of TP
403
+divisibility = [1, 2, 8, 16, 64]
404
+
405
+all_hidden_size = []
406
+for div in divisibility:
407
+ for hidden_size in HIDDEN_SIZES:
408
+ all_hidden_size.append(hidden_size // div)
409
+
410
+HIDDEN_SIZES = list(set(all_hidden_size))
411
+
412
+# Test params that focuses on hidden_size variation.
413
+hs_test_params = {
414
+ "hidden_sizes": HIDDEN_SIZES,
415
+ "batches": [4],
416
+ "num_loras": [4],
417
+ "max_ranks": [32],
418
+}
419
+
420
+# General tests params that tests for variations in all dimensions
421
+# except hidden_size.
422
+test_params = {
423
+ "hidden_sizes": [2049],
424
+ "batches": [1, 4, 16, 32],
425
+ "num_loras": [1, 8, 32, 128],
426
+ "max_ranks": [1, 4, 8, 16, 32, 64, 128, 256],
427
+}
428
+
429
+DTYPES = [torch.float16, torch.bfloat16]
430
+DEVICES = [f"cuda:{0}"]
431
+SEED = [0]
432
+
433
+
434
+@pytest.mark.parametrize("batches", test_params['batches'])
435
+@pytest.mark.parametrize("num_loras", test_params['num_loras'])
436
+@pytest.mark.parametrize("rank", test_params['max_ranks'])
437
+@pytest.mark.parametrize("hidden_size", test_params['hidden_sizes'])
438
+@pytest.mark.parametrize("nslices", [1, 2, 3])
439
+@pytest.mark.parametrize("dtype", DTYPES)
440
+@pytest.mark.parametrize("device", DEVICES)
441
+@pytest.mark.parametrize("seed", SEED)
442
+@pytest.mark.parametrize("op_type", ["shrink", "expand"])
443
+def test_punica_sgmv(
444
+ batches: int,
445
+ num_loras: int,
446
+ rank: int,
447
+ hidden_size: int,
448
+ nslices: int,
449
+ dtype: torch.dtype,
450
+ device: str,
451
+ seed: int,
452
+ op_type: str,
453
+):
454
+ torch.set_default_device(device)
455
+ current_platform.seed_everything(seed)
456
+
457
+ if op_type == "shrink":
458
+ check_sgmv_shrink(batches=batches,
459
+ num_loras=num_loras,
460
+ rank=rank,
461
+ hidden_size=hidden_size,
462
+ nslices=nslices,
463
+ dtype=dtype,
464
+ device=device,
465
+ seq_length=128,
466
+ scaling=0.5)
467
+ else:
468
+ check_sgmv_expand(batches=batches,
469
+ num_loras=num_loras,
470
+ rank=rank,
471
+ hidden_size=hidden_size,
472
+ nslices=nslices,
473
+ dtype=dtype,
474
+ device=device,
475
+ seq_length=128,
476
+ add_inputs=True)
477
+
478
+
479
+@pytest.mark.parametrize("batches", hs_test_params['batches'])
480
+@pytest.mark.parametrize("num_loras", hs_test_params['num_loras'])
481
+@pytest.mark.parametrize("rank", hs_test_params['max_ranks'])
482
+@pytest.mark.parametrize("hidden_size", hs_test_params['hidden_sizes'])
483
+@pytest.mark.parametrize("nslices", [1, 2, 3])
484
+@pytest.mark.parametrize("dtype", DTYPES)
485
+@pytest.mark.parametrize("device", DEVICES)
486
+@pytest.mark.parametrize("seed", SEED)
487
+@pytest.mark.parametrize("op_type", ["shrink", "expand"])
488
+def test_punica_sgmv_hidden_size(
489
+ batches: int,
490
+ num_loras: int,
491
+ rank: int,
492
+ hidden_size: int,
493
+ nslices: int,
494
+ dtype: torch.dtype,
495
+ device: str,
496
+ seed: int,
497
+ op_type: str,
498
+):
499
+ torch.set_default_device(device)
500
+ current_platform.seed_everything(seed)
501
+
502
+ if op_type == "shrink":
503
+ check_sgmv_shrink(batches=batches,
504
+ num_loras=num_loras,
505
+ rank=rank,
506
+ hidden_size=hidden_size,
507
+ nslices=nslices,
508
+ dtype=dtype,
509
+ device=device,
510
+ seq_length=128,
511
+ scaling=0.5)
512
+ else:
513
+ check_sgmv_expand(batches=batches,
514
+ num_loras=num_loras,
515
+ rank=rank,
516
+ hidden_size=hidden_size,
517
+ nslices=nslices,
518
+ dtype=dtype,
519
+ device=device,
520
+ seq_length=128,
521
+ add_inputs=True)
522
+
523
+
524
+@pytest.mark.parametrize("batches", test_params['batches'])
525
+@pytest.mark.parametrize("num_loras", test_params['num_loras'])
526
+@pytest.mark.parametrize("rank", test_params['max_ranks'])
527
+@pytest.mark.parametrize("hidden_size", test_params['hidden_sizes'])
528
+@pytest.mark.parametrize("dtype", DTYPES)
529
+@pytest.mark.parametrize("device", DEVICES)
530
+@pytest.mark.parametrize("seed", SEED)
531
+@pytest.mark.parametrize("op_type", ["shrink", "expand"])
532
+def test_punica_bgmv(
533
+ batches: int,
534
+ num_loras: int,
535
+ rank: int,
536
+ hidden_size: int,
537
+ dtype: torch.dtype,
538
+ device: str,
539
+ seed: int,
540
+ op_type: str,
541
+):
542
+ torch.set_default_device(device)
543
+ current_platform.seed_everything(seed)
544
+
545
+ if op_type == "shrink":
546
+ check_bgmv_shrink(batches=batches,
547
+ num_loras=num_loras,
548
+ rank=rank,
549
+ hidden_size=hidden_size,
550
+ dtype=dtype,
551
+ device=device,
552
+ scaling=0.5)
553
+ else:
554
+ check_bgmv_expand(batches=batches,
555
+ num_loras=num_loras,
556
+ rank=rank,
557
+ hidden_size=hidden_size,
558
+ dtype=dtype,
559
+ device=device,
560
+ add_inputs=True)
561
+
562
+
563
+@pytest.mark.parametrize("batches", hs_test_params['batches'])
564
+@pytest.mark.parametrize("num_loras", hs_test_params['num_loras'])
565
+@pytest.mark.parametrize("rank", hs_test_params['max_ranks'])
566
+@pytest.mark.parametrize("hidden_size", hs_test_params['hidden_sizes'])
567
+@pytest.mark.parametrize("dtype", DTYPES)
568
+@pytest.mark.parametrize("device", DEVICES)
569
+@pytest.mark.parametrize("seed", SEED)
570
+@pytest.mark.parametrize("op_type", ["shrink", "expand"])
571
+def test_punica_bgmv_hidden_size(
572
+ batches: int,
573
+ num_loras: int,
574
+ rank: int,
575
+ hidden_size: int,
576
+ dtype: torch.dtype,
577
+ device: str,
578
+ seed: int,
579
+ op_type: str,
580
+):
581
+ torch.set_default_device(device)
582
+ current_platform.seed_everything(seed)
583
+
584
+ if op_type == "shrink":
585
+ check_bgmv_shrink(batches=batches,
586
+ num_loras=num_loras,
587
+ rank=rank,
588
+ hidden_size=hidden_size,
589
+ dtype=dtype,
590
+ device=device,
591
+ scaling=0.5)
592
+ else:
593
+ check_bgmv_expand(batches=batches,
594
+ num_loras=num_loras,
595
+ rank=rank,
596
+ hidden_size=hidden_size,
597
+ dtype=dtype,
598
+ device=device,
599
+ add_inputs=True)
600
+
601
+
602
+@pytest.mark.parametrize("batches", test_params['batches'])
603
+@pytest.mark.parametrize("num_loras", test_params['num_loras'])
604
+@pytest.mark.parametrize("rank", test_params['max_ranks'])
605
+@pytest.mark.parametrize("hidden_size", test_params['hidden_sizes'])
606
+@pytest.mark.parametrize("nslices", [2, 3])
607
+@pytest.mark.parametrize("dtype", DTYPES)
608
+@pytest.mark.parametrize("device", DEVICES)
609
+@pytest.mark.parametrize("seed", SEED)
610
+def test_punica_bgmv_expand_nslices(batches: int, num_loras: int, rank: int,
611
+ hidden_size: int, nslices: int,
612
+ dtype: torch.dtype, device: str,
613
+ seed: int):
614
+
615
+ torch.set_default_device(device)
616
+ current_platform.seed_everything(seed)
617
+
618
+ check_bgmv_expand_slice(batches=batches,
619
+ num_loras=num_loras,
620
+ rank=rank,
621
+ hidden_size=hidden_size,
622
+ nslices=nslices,
623
+ dtype=dtype,
624
+ device=device,
625
+ add_inputs=True)
626
+
627
+
628
+@pytest.mark.parametrize("batches", hs_test_params['batches'])
629
+@pytest.mark.parametrize("num_loras", hs_test_params['num_loras'])
630
+@pytest.mark.parametrize("rank", hs_test_params['max_ranks'])
631
+@pytest.mark.parametrize("hidden_size", hs_test_params['hidden_sizes'])
632
+@pytest.mark.parametrize("nslices", [2, 3])
633
+@pytest.mark.parametrize("dtype", DTYPES)
634
+@pytest.mark.parametrize("device", DEVICES)
635
+@pytest.mark.parametrize("seed", SEED)
636
+def test_punica_bgmv_expand_nslices_hidden_size(batches: int, num_loras: int,
637
+ rank: int, hidden_size: int,
638
+ nslices: int,
639
+ dtype: torch.dtype,
640
+ device: str, seed: int):
641
+
642
+ torch.set_default_device(device)
643
+ current_platform.seed_everything(seed)
644
+
645
+ check_bgmv_expand_slice(batches=batches,
646
+ num_loras=num_loras,
647
+ rank=rank,
648
+ hidden_size=hidden_size,
649
+ nslices=nslices,
650
+ dtype=dtype,
651
+ device=device,
652
+ add_inputs=True)
tests/lora/test_punica_ops_sizes.py
deleted
-401
@@ -1,401 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-"""
3
-This script is mainly used to tests various hidden_sizes. We have collected the
4
-hidden_sizes included in the LoRA models currently supported by vLLM. It tests
5
-whether the corresponding Triton kernel can run normally when tensor parallelism
6
-is set to [1, 2, 4, 8, 16, 32, 64].
7
-"""
8
-from threading import Lock
9
-
10
-import pytest
11
-import torch
12
-
13
-import vllm.lora.ops.triton_ops # noqa: F401
14
-from vllm.lora.ops.torch_ops import (bgmv_expand, bgmv_expand_slice,
15
- bgmv_shrink, sgmv_expand,
16
- sgmv_expand_slice, sgmv_shrink)
17
-from vllm.lora.ops.triton_ops.utils import _LORA_A_PTR_DICT, _LORA_B_PTR_DICT
18
-from vllm.platforms import current_platform
19
-
20
-from .utils import (assert_close, generate_data,
21
- generate_data_for_expand_nslices,
22
- generate_data_for_nslices)
23
-
24
-HIDDEN_SIZES = [
25
- 128,
26
- 256,
27
- 512,
28
- 896,
29
- 1024,
30
- 1152,
31
- 1216,
32
- 1280,
33
- 1536,
34
- 1664,
35
- 2048,
36
- 2240,
37
- 2304,
38
- 2368,
39
- 2432,
40
- 2560,
41
- 2752,
42
- 3072,
43
- 3328,
44
- 3456,
45
- 3584,
46
- 3712,
47
- 4096,
48
- 4480,
49
- 4608,
50
- 4736,
51
- 4864,
52
- 5120,
53
- 5504,
54
- 5632,
55
- 5888,
56
- 6144,
57
- 6400,
58
- 6848,
59
- 6912,
60
- 7168,
61
- 7424,
62
- 8192,
63
- 8960,
64
- 9216,
65
- 9472,
66
- 10240,
67
- 11008,
68
- 11264,
69
- 13824,
70
- 14336,
71
- 14784,
72
- 14848,
73
- 15360,
74
- 18944,
75
- 22016,
76
- 22528,
77
- 24576,
78
- 27392,
79
- 27648,
80
- 29568,
81
- 29696,
82
- 32000,
83
- 32256,
84
- 32512,
85
- 32768,
86
- 33024,
87
- 36864,
88
- 43264,
89
- 49152,
90
- 49408,
91
- 60544,
92
- 60672,
93
- 64000,
94
- 64256,
95
- 102400,
96
- 102656,
97
- 128000,
98
- 128256,
99
-]
100
-#The size of TP
101
-divisibility = [1, 2, 8, 16, 64]
102
-
103
-all_hidden_size = []
104
-for div in divisibility:
105
- for hidden_size in HIDDEN_SIZES:
106
- all_hidden_size.append(hidden_size // div)
107
-
108
-HIDDEN_SIZES = list(set(all_hidden_size))
109
-
110
-BATCHES = [4]
111
-NUM_LORA = [4]
112
-DTYPES = [torch.float16, torch.bfloat16]
113
-MAX_RANKS = [32]
114
-SCALES = [0.5]
115
-SEED = [0]
116
-DEVICES = [f"cuda:{0}"]
117
-
118
-_dict_lock = Lock()
119
-
120
-
121
-@pytest.mark.parametrize("batches", BATCHES)
122
-@pytest.mark.parametrize("num_loras", NUM_LORA)
123
-@pytest.mark.parametrize("rank", MAX_RANKS)
124
-@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
125
-@pytest.mark.parametrize("scaling", SCALES)
126
-@pytest.mark.parametrize("nslices", [1, 2, 3])
127
-@pytest.mark.parametrize("dtype", DTYPES)
128
-@pytest.mark.parametrize("op_type", ["shrink", "expand"])
129
-@pytest.mark.parametrize("seed", SEED)
130
-@pytest.mark.parametrize("device", DEVICES)
131
-def test_punica_sgmv(
132
- batches: int,
133
- num_loras: int,
134
- rank: int,
135
- hidden_size: int,
136
- scaling: float,
137
- nslices: int,
138
- dtype: torch.dtype,
139
- op_type: str,
140
- seed: int,
141
- device: str,
142
-):
143
- torch.set_default_device(device)
144
- current_platform.seed_everything(seed)
145
-
146
- seq_length = 128
147
- (
148
- inputs_tensor,
149
- lora_weights_lst,
150
- our_out_tensor,
151
- ref_out_tensor,
152
- b_seq_start_loc,
153
- lora_indices_tensor,
154
- seq_len_tensor,
155
- indices,
156
- ) = generate_data_for_nslices(
157
- batches,
158
- hidden_size,
159
- num_loras,
160
- rank,
161
- seq_length,
162
- nslices,
163
- dtype,
164
- op_type,
165
- device,
166
- )
167
- max_seq_length = seq_len_tensor.max()
168
- token_nums = seq_len_tensor.sum().item()
169
- if isinstance(max_seq_length, tuple):
170
- max_seq_length = max_seq_length[0].item()
171
- else:
172
- max_seq_length = max_seq_length.item()
173
- if op_type == "shrink":
174
- # Preventing cache error pointer.
175
- with _dict_lock:
176
- _LORA_A_PTR_DICT.clear()
177
- torch.ops.vllm.sgmv_shrink(
178
- inputs_tensor,
179
- lora_weights_lst,
180
- our_out_tensor,
181
- b_seq_start_loc,
182
- seq_len_tensor,
183
- lora_indices_tensor,
184
- batches,
185
- max_seq_length,
186
- token_nums,
187
- scaling,
188
- )
189
- for index in range(nslices):
190
- sgmv_shrink(
191
- inputs_tensor,
192
- lora_weights_lst[index],
193
- ref_out_tensor[index],
194
- b_seq_start_loc,
195
- seq_len_tensor,
196
- lora_indices_tensor,
197
- batches,
198
- max_seq_length,
199
- token_nums,
200
- scaling,
201
- )
202
-
203
- else:
204
- with _dict_lock:
205
- _LORA_B_PTR_DICT.clear()
206
- torch.ops.vllm.sgmv_expand(
207
- inputs_tensor,
208
- lora_weights_lst,
209
- our_out_tensor,
210
- b_seq_start_loc,
211
- seq_len_tensor,
212
- lora_indices_tensor,
213
- batches,
214
- max_seq_length,
215
- token_nums,
216
- offset_start=0,
217
- add_inputs=True,
218
- )
219
- if nslices == 1:
220
- # Verify the torch's sgmv_expand op
221
- sgmv_expand(
222
- inputs_tensor[0],
223
- lora_weights_lst[0],
224
- ref_out_tensor,
225
- b_seq_start_loc,
226
- seq_len_tensor,
227
- lora_indices_tensor,
228
- batches,
229
- max_seq_length,
230
- token_nums,
231
- add_inputs=True,
232
- )
233
- else:
234
- slice_offset = 0
235
- for index in range(nslices):
236
- lora_weights = lora_weights_lst[index]
237
- sgmv_expand_slice(
238
- inputs_tensor[index],
239
- lora_weights,
240
- ref_out_tensor,
241
- b_seq_start_loc,
242
- seq_len_tensor,
243
- lora_indices_tensor,
244
- batches,
245
- max_seq_length,
246
- token_nums,
247
- slice_offset,
248
- hidden_size,
249
- add_inputs=True,
250
- )
251
- slice_offset += hidden_size
252
-
253
- assert_close(our_out_tensor, ref_out_tensor)
254
-
255
-
256
-@pytest.mark.parametrize("batches", BATCHES)
257
-@pytest.mark.parametrize("num_loras", NUM_LORA)
258
-@pytest.mark.parametrize("rank", MAX_RANKS)
259
-@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
260
-@pytest.mark.parametrize("scaling", SCALES)
261
-@pytest.mark.parametrize("dtype", DTYPES)
262
-@pytest.mark.parametrize("op_type", ["shrink", "expand"])
263
-@pytest.mark.parametrize("seed", SEED)
264
-@pytest.mark.parametrize("device", DEVICES)
265
-def test_punica_bgmv(
266
- batches: int,
267
- num_loras: int,
268
- rank: int,
269
- hidden_size: int,
270
- scaling: float,
271
- dtype: torch.dtype,
272
- op_type: str,
273
- seed: int,
274
- device: str,
275
-):
276
- torch.set_default_device(device)
277
- current_platform.seed_everything(seed)
278
-
279
- seq_length = 1
280
- (
281
- inputs_tensor,
282
- lora_weights,
283
- our_out_tensor,
284
- ref_out_tensor,
285
- b_seq_start_loc,
286
- lora_indices_tensor,
287
- seq_len_tensor,
288
- indices,
289
- ) = generate_data(
290
- batches,
291
- hidden_size,
292
- num_loras,
293
- rank,
294
- seq_length,
295
- dtype,
296
- op_type,
297
- device,
298
- )
299
- if op_type == "shrink":
300
- torch.ops.vllm.bgmv_shrink(
301
- inputs_tensor,
302
- lora_weights,
303
- our_out_tensor,
304
- indices,
305
- scaling,
306
- )
307
-
308
- bgmv_shrink(
309
- inputs_tensor,
310
- lora_weights,
311
- ref_out_tensor,
312
- indices,
313
- scaling,
314
- )
315
-
316
- else:
317
- torch.ops.vllm.bgmv_expand(
318
- inputs_tensor,
319
- lora_weights,
320
- our_out_tensor,
321
- indices,
322
- add_inputs=True,
323
- )
324
- bgmv_expand(
325
- inputs_tensor,
326
- lora_weights,
327
- ref_out_tensor,
328
- indices,
329
- add_inputs=True,
330
- )
331
-
332
- if op_type == "shrink":
333
- ref_out_tensor = ref_out_tensor.to(torch.float32)
334
- assert_close(our_out_tensor, ref_out_tensor)
335
-
336
-
337
-@pytest.mark.parametrize("batches", BATCHES)
338
-@pytest.mark.parametrize("num_loras", NUM_LORA)
339
-@pytest.mark.parametrize("rank", MAX_RANKS)
340
-@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
341
-@pytest.mark.parametrize("nslices", [2, 3])
342
-@pytest.mark.parametrize("dtype", DTYPES)
343
-@pytest.mark.parametrize("seed", SEED)
344
-@pytest.mark.parametrize("device", DEVICES)
345
-def test_punica_bgmv_expand_nslices(
346
- batches: int,
347
- num_loras: int,
348
- rank: int,
349
- hidden_size: int,
350
- nslices: int,
351
- dtype: torch.dtype,
352
- seed: int,
353
- device: str,
354
-):
355
- torch.set_default_device(device)
356
- current_platform.seed_everything(seed)
357
-
358
- seq_length = 1
359
- (
360
- inputs_tensor,
361
- lora_weights_lst,
362
- our_outputs,
363
- ref_outputs,
364
- b_seq_start_loc,
365
- lora_indices_tensor,
366
- seq_len_tensor,
367
- indices,
368
- ) = generate_data_for_expand_nslices(
369
- batches,
370
- hidden_size,
371
- num_loras,
372
- rank,
373
- seq_length,
374
- dtype,
375
- nslices,
376
- device,
377
- )
378
- slice_offset = 0
379
- for index in range(nslices):
380
- lora_weights = lora_weights_lst[index]
381
- torch.ops.vllm.bgmv_expand_slice(
382
- inputs_tensor,
383
- lora_weights,
384
- our_outputs,
385
- indices,
386
- slice_offset,
387
- slice_size=hidden_size,
388
- add_inputs=True,
389
- )
390
- bgmv_expand_slice(
391
- inputs_tensor,
392
- lora_weights,
393
- ref_outputs,
394
- indices,
395
- slice_offset,
396
- slice_size=hidden_size,
397
- add_inputs=True,
398
- )
399
-
400
- slice_offset += hidden_size
401
- assert_close(our_outputs, ref_outputs)
tests/lora/test_punica_ops_variation.py
deleted
-317
@@ -1,317 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-"""
3
-This script is mainly used to test whether trtion kernels can run normally
4
-under different conditions, including various batches, numbers of LoRA , and
5
-maximum ranks.
6
-"""
7
-from threading import Lock
8
-
9
-import pytest
10
-import torch
11
-
12
-# Enable custom op register
13
-import vllm.lora.ops.triton_ops # noqa: F401
14
-from vllm.lora.ops.torch_ops import (bgmv_expand, bgmv_expand_slice,
15
- bgmv_shrink, sgmv_expand,
16
- sgmv_expand_slice, sgmv_shrink)
17
-from vllm.lora.ops.triton_ops.utils import _LORA_A_PTR_DICT, _LORA_B_PTR_DICT
18
-from vllm.platforms import current_platform
19
-
20
-from .utils import (assert_close, generate_data,
21
- generate_data_for_expand_nslices,
22
- generate_data_for_nslices)
23
-
24
-HIDDEN_SIZES = [2049]
25
-
26
-BATCHES = [1, 4, 16, 32]
27
-NUM_LORA = [1, 8, 32, 128]
28
-DTYPES = [torch.float16, torch.bfloat16]
29
-MAX_RANKS = [1, 4, 8, 16, 32, 64, 128, 256]
30
-SCALES = [0.5]
31
-SEED = [0]
32
-DEVICES = [f"cuda:{0}"]
33
-
34
-_dict_lock = Lock()
35
-
36
-
37
-@pytest.mark.parametrize("batches", BATCHES)
38
-@pytest.mark.parametrize("num_loras", NUM_LORA)
39
-@pytest.mark.parametrize("rank", MAX_RANKS)
40
-@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
41
-@pytest.mark.parametrize("scaling", SCALES)
42
-@pytest.mark.parametrize("nslices", [1, 2, 3])
43
-@pytest.mark.parametrize("dtype", DTYPES)
44
-@pytest.mark.parametrize("op_type", ["shrink", "expand"])
45
-@pytest.mark.parametrize("seed", SEED)
46
-@pytest.mark.parametrize("device", DEVICES)
47
-def test_punica_sgmv(
48
- batches: int,
49
- num_loras: int,
50
- rank: int,
51
- hidden_size: int,
52
- scaling: float,
53
- nslices: int,
54
- dtype: torch.dtype,
55
- op_type: str,
56
- seed: int,
57
- device: str,
58
-):
59
- torch.set_default_device(device)
60
- current_platform.seed_everything(seed)
61
-
62
- seq_length = 128
63
- (
64
- inputs_tensor,
65
- lora_weights_lst,
66
- our_out_tensor,
67
- ref_out_tensor,
68
- b_seq_start_loc,
69
- lora_indices_tensor,
70
- seq_len_tensor,
71
- indices,
72
- ) = generate_data_for_nslices(
73
- batches,
74
- hidden_size,
75
- num_loras,
76
- rank,
77
- seq_length,
78
- nslices,
79
- dtype,
80
- op_type,
81
- device,
82
- )
83
- max_seq_length = seq_len_tensor.max()
84
- token_nums = seq_len_tensor.sum().item()
85
- if isinstance(max_seq_length, tuple):
86
- max_seq_length = max_seq_length[0].item()
87
- else:
88
- max_seq_length = max_seq_length.item()
89
- if op_type == "shrink":
90
- # Preventing cache error pointer.
91
- with _dict_lock:
92
- _LORA_A_PTR_DICT.clear()
93
- torch.ops.vllm.sgmv_shrink(
94
- inputs_tensor,
95
- lora_weights_lst,
96
- our_out_tensor,
97
- b_seq_start_loc,
98
- seq_len_tensor,
99
- lora_indices_tensor,
100
- batches,
101
- max_seq_length,
102
- token_nums,
103
- scaling,
104
- )
105
- for index in range(nslices):
106
- sgmv_shrink(
107
- inputs_tensor,
108
- lora_weights_lst[index],
109
- ref_out_tensor[index],
110
- b_seq_start_loc,
111
- seq_len_tensor,
112
- lora_indices_tensor,
113
- batches,
114
- max_seq_length,
115
- token_nums,
116
- scaling,
117
- )
118
-
119
- else:
120
- with _dict_lock:
121
- _LORA_B_PTR_DICT.clear()
122
- torch.ops.vllm.sgmv_expand(
123
- inputs_tensor,
124
- lora_weights_lst,
125
- our_out_tensor,
126
- b_seq_start_loc,
127
- seq_len_tensor,
128
- lora_indices_tensor,
129
- batches,
130
- max_seq_length,
131
- token_nums,
132
- offset_start=0,
133
- add_inputs=True,
134
- )
135
- slice_offset = 0
136
- if nslices == 1:
137
- # Verify the torch's sgmv_expand op
138
- sgmv_expand(
139
- inputs_tensor[0],
140
- lora_weights_lst[0],
141
- ref_out_tensor,
142
- b_seq_start_loc,
143
- seq_len_tensor,
144
- lora_indices_tensor,
145
- batches,
146
- max_seq_length,
147
- token_nums,
148
- add_inputs=True,
149
- )
150
- else:
151
- for index in range(nslices):
152
- lora_weights = lora_weights_lst[index]
153
- sgmv_expand_slice(
154
- inputs_tensor[index],
155
- lora_weights,
156
- ref_out_tensor,
157
- b_seq_start_loc,
158
- seq_len_tensor,
159
- lora_indices_tensor,
160
- batches,
161
- max_seq_length,
162
- token_nums,
163
- slice_offset,
164
- hidden_size,
165
- add_inputs=True,
166
- )
167
- slice_offset += hidden_size
168
-
169
- assert_close(our_out_tensor, ref_out_tensor)
170
-
171
-
172
-@pytest.mark.parametrize("batches", BATCHES)
173
-@pytest.mark.parametrize("num_loras", NUM_LORA)
174
-@pytest.mark.parametrize("rank", MAX_RANKS)
175
-@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
176
-@pytest.mark.parametrize("scaling", SCALES)
177
-@pytest.mark.parametrize("dtype", DTYPES)
178
-@pytest.mark.parametrize("op_type", ["shrink", "expand"])
179
-@pytest.mark.parametrize("seed", SEED)
180
-@pytest.mark.parametrize("device", DEVICES)
181
-def test_punica_bgmv(
182
- batches: int,
183
- num_loras: int,
184
- rank: int,
185
- hidden_size: int,
186
- scaling: float,
187
- dtype: torch.dtype,
188
- op_type: str,
189
- seed: int,
190
- device: str,
191
-):
192
- torch.set_default_device(device)
193
- current_platform.seed_everything(seed)
194
-
195
- seq_length = 1
196
- (
197
- inputs_tensor,
198
- lora_weights,
199
- our_out_tensor,
200
- ref_out_tensor,
201
- b_seq_start_loc,
202
- lora_indices_tensor,
203
- seq_len_tensor,
204
- indices,
205
- ) = generate_data(
206
- batches,
207
- hidden_size,
208
- num_loras,
209
- rank,
210
- seq_length,
211
- dtype,
212
- op_type,
213
- device,
214
- )
215
- if op_type == "shrink":
216
- torch.ops.vllm.bgmv_shrink(
217
- inputs_tensor,
218
- lora_weights,
219
- our_out_tensor,
220
- indices,
221
- scaling,
222
- )
223
-
224
- bgmv_shrink(
225
- inputs_tensor,
226
- lora_weights,
227
- ref_out_tensor,
228
- indices,
229
- scaling,
230
- )
231
-
232
- else:
233
- torch.ops.vllm.bgmv_expand(
234
- inputs_tensor,
235
- lora_weights,
236
- our_out_tensor,
237
- indices,
238
- add_inputs=True,
239
- )
240
- bgmv_expand(
241
- inputs_tensor,
242
- lora_weights,
243
- ref_out_tensor,
244
- indices,
245
- add_inputs=True,
246
- )
247
-
248
- if op_type == "shrink":
249
- ref_out_tensor = ref_out_tensor.to(torch.float32)
250
- assert_close(our_out_tensor, ref_out_tensor)
251
-
252
-
253
-@pytest.mark.parametrize("batches", BATCHES)
254
-@pytest.mark.parametrize("num_loras", NUM_LORA)
255
-@pytest.mark.parametrize("rank", MAX_RANKS)
256
-@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
257
-@pytest.mark.parametrize("nslices", [2, 3])
258
-@pytest.mark.parametrize("dtype", DTYPES)
259
-@pytest.mark.parametrize("seed", SEED)
260
-@pytest.mark.parametrize("device", DEVICES)
261
-def test_punica_bgmv_expand_nslices(
262
- batches: int,
263
- num_loras: int,
264
- rank: int,
265
- hidden_size: int,
266
- nslices: int,
267
- dtype: torch.dtype,
268
- seed: int,
269
- device: str,
270
-):
271
- torch.set_default_device(device)
272
- current_platform.seed_everything(seed)
273
-
274
- seq_length = 1
275
- (
276
- inputs_tensor,
277
- lora_weights_lst,
278
- our_outputs,
279
- ref_outputs,
280
- b_seq_start_loc,
281
- lora_indices_tensor,
282
- seq_len_tensor,
283
- indices,
284
- ) = generate_data_for_expand_nslices(
285
- batches,
286
- hidden_size,
287
- num_loras,
288
- rank,
289
- seq_length,
290
- dtype,
291
- nslices,
292
- device,
293
- )
294
- slice_offset = 0
295
- for index in range(nslices):
296
- lora_weights = lora_weights_lst[index]
297
- torch.ops.vllm.bgmv_expand_slice(
298
- inputs_tensor,
299
- lora_weights,
300
- our_outputs,
301
- indices,
302
- slice_offset,
303
- slice_size=hidden_size,
304
- add_inputs=True,
305
- )
306
- bgmv_expand_slice(
307
- inputs_tensor,
308
- lora_weights,
309
- ref_outputs,
310
- indices,
311
- slice_offset,
312
- slice_size=hidden_size,
313
- add_inputs=True,
314
- )
315
-
316
- slice_offset += hidden_size
317
- assert_close(our_outputs, ref_outputs)
tests/lora/utils.py
+34
-7
@@ -1,6 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
-from typing import Dict, List, Optional
3
+from dataclasses import dataclass
4
+from typing import Dict, List, Optional, Tuple, Union
5
6
import torch
7
@@ -106,6 +107,31 @@ def assert_close(a, b):
107
torch.testing.assert_close(a, b, rtol=rtol, atol=atol)
108
109
110
+@dataclass
111
+class PunicaTensors:
112
+ inputs_tensor: torch.Tensor
113
+ lora_weights: Union[torch.Tensor, List[torch.Tensor]]
114
+ our_out_tensor: torch.Tensor
115
+ ref_out_tensor: torch.Tensor
116
+ b_seq_start_loc: torch.Tensor
117
+ prompt_lora_mapping: torch.Tensor
118
+ seq_len_tensor: torch.Tensor
119
+ token_lora_mapping: torch.Tensor
120
+
121
+ def meta(self) -> Tuple[int, int]:
122
+ """
123
+ Infer max_seq_length and token_nums from the tensors
124
+ and return them.
125
+ """
126
+ max_seq_length = self.seq_len_tensor.max()
127
+ token_nums = self.seq_len_tensor.sum().item()
128
+ if isinstance(max_seq_length, tuple):
129
+ max_seq_length = max_seq_length[0].item()
130
+ else:
131
+ max_seq_length = max_seq_length.item()
132
+ return max_seq_length, token_nums
133
+
134
+
135
def generate_data(
136
batches,
137
hidden_size,
@@ -115,7 +141,7 @@ def generate_data(
141
dtype,
142
op_type,
143
device,
118
-):
144
+) -> PunicaTensors:
145
seq_len_tensor = torch.randint(seq_length, seq_length + 1,
146
(batches, )).to(device)
147
b_seq_start_loc = torch.cumsum(
@@ -164,7 +190,8 @@ def generate_data(
190
indices[current_offset:current_offset +
191
seq_len_tensor[b_id]].copy_(lora_index)
192
current_offset += seq_len_tensor[b_id].item()
167
- return (
193
+
194
+ return PunicaTensors(
195
inputs_tensor,
196
lora_weights,
197
our_out_tensor,
@@ -185,7 +212,7 @@ def generate_data_for_expand_nslices(
212
dtype,
213
nslices,
214
device,
188
-):
215
+) -> PunicaTensors:
216
seq_len_tensor = torch.randint(seq_length, seq_length + 1,
217
(batches, )).to(device)
218
b_seq_start_loc = torch.cumsum(
@@ -222,7 +249,7 @@ def generate_data_for_expand_nslices(
249
current_offset += seq_len_tensor[b_id].item()
250
251
lora_indices_tensor = lora_indices_tensor.to(device)
225
- return (
252
+ return PunicaTensors(
253
inputs_tensor,
254
lora_weights_lst,
255
our_out_tensor,
@@ -244,7 +271,7 @@ def generate_data_for_nslices(
271
dtype,
272
op_type,
273
device,
247
-):
274
+) -> PunicaTensors:
275
seq_len_tensor = torch.randint(seq_length, seq_length + 1,
276
(batches, )).to(device)
277
b_seq_start_loc = torch.cumsum(
@@ -302,7 +329,7 @@ def generate_data_for_nslices(
329
current_offset += seq_len_tensor[b_id].item()
330
331
lora_indices_tensor = lora_indices_tensor.to(device)
305
- return (
332
+ return PunicaTensors(
333
inputs_tensor,
334
lora_weights_lst,
335
our_out_tensor,