A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[Misc][LoRA] Move the implementation of lora bias to punica.py (#10829)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Dec 3, 2024 at 01:53 UTC
b45f0d79469f583736052b80bfc8b3bab29f50d8
4 files changed
+156
-175
tests/lora/test_llama_tp.py
+27
-33
@@ -55,15 +55,7 @@ def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:
55
return generated_texts
56
57
58
-@fork_new_process_for_each_test
59
-def test_llama_lora(sql_lora_files):
60
-
61
- llm = vllm.LLM(MODEL_PATH,
62
- enable_lora=True,
63
- max_num_seqs=16,
64
- max_loras=4,
65
- tensor_parallel_size=1)
66
-
58
+def generate_and_test(llm, sql_lora_files):
59
print("lora adapter created")
60
assert do_sample(llm, sql_lora_files, lora_id=0) == EXPECTED_NO_LORA_OUTPUT
61
@@ -79,6 +71,17 @@ def test_llama_lora(sql_lora_files):
71
print("removing lora")
72
73
74
+@fork_new_process_for_each_test
75
+def test_llama_lora(sql_lora_files):
76
+
77
+ llm = vllm.LLM(MODEL_PATH,
78
+ enable_lora=True,
79
+ max_num_seqs=16,
80
+ max_loras=4,
81
+ tensor_parallel_size=1)
82
+ generate_and_test(llm, sql_lora_files)
83
+
84
+
85
@fork_new_process_for_each_test
86
def test_llama_lora_warmup(sql_lora_files):
87
"""Test that the LLM initialization works with a warmup LORA path and
@@ -118,20 +121,7 @@ def test_llama_lora_tp4(sql_lora_files):
121
max_loras=4,
122
tensor_parallel_size=4,
123
)
121
-
122
- print("lora adapter created")
123
- assert do_sample(llm, sql_lora_files, lora_id=0) == EXPECTED_NO_LORA_OUTPUT
124
-
125
- print("lora 1")
126
- assert do_sample(llm, sql_lora_files, lora_id=1) == EXPECTED_LORA_OUTPUT
127
-
128
- print("no lora")
129
- assert do_sample(llm, sql_lora_files, lora_id=0) == EXPECTED_NO_LORA_OUTPUT
130
-
131
- print("lora 2")
132
- assert do_sample(llm, sql_lora_files, lora_id=2) == EXPECTED_LORA_OUTPUT
133
-
134
- print("removing lora")
124
+ generate_and_test(llm, sql_lora_files)
125
126
127
@multi_gpu_test(num_gpus=4)
@@ -146,16 +136,20 @@ def test_llama_lora_tp4_fully_sharded_loras(sql_lora_files):
136
tensor_parallel_size=4,
137
fully_sharded_loras=True,
138
)
149
- print("lora adapter created")
150
- assert do_sample(llm, sql_lora_files, lora_id=0) == EXPECTED_NO_LORA_OUTPUT
151
-
152
- print("lora 1")
153
- assert do_sample(llm, sql_lora_files, lora_id=1) == EXPECTED_LORA_OUTPUT
139
+ generate_and_test(llm, sql_lora_files)
140
155
- print("no lora")
156
- assert do_sample(llm, sql_lora_files, lora_id=0) == EXPECTED_NO_LORA_OUTPUT
141
158
- print("lora 2")
159
- assert do_sample(llm, sql_lora_files, lora_id=2) == EXPECTED_LORA_OUTPUT
142
+@multi_gpu_test(num_gpus=4)
143
+@fork_new_process_for_each_test
144
+def test_llama_lora_tp4_fully_sharded_enable_bias(sql_lora_files):
145
161
- print("removing lora")
146
+ llm = vllm.LLM(
147
+ MODEL_PATH,
148
+ enable_lora=True,
149
+ max_num_seqs=16,
150
+ max_loras=4,
151
+ tensor_parallel_size=4,
152
+ fully_sharded_loras=True,
153
+ enable_lora_bias=True,
154
+ )
155
+ generate_and_test(llm, sql_lora_files)
vllm/lora/fully_sharded_layers.py
+12
-29
@@ -73,6 +73,7 @@ class ColumnParallelLinearWithShardedLoRA(ColumnParallelLinearWithLoRA):
73
self.punica_wrapper.add_expand(output,
74
buffer,
75
self.lora_b_stacked,
76
+ self.bias_stacked,
77
add_input=True)
78
# now have column partitioned output
79
@@ -131,27 +132,14 @@ def _mcp_apply(x, bias, layer: QKVParallelLinearWithLora):
132
layer.lora_a_stacked[idx], 1.0)
133
134
buffers = tensor_model_parallel_all_gather(buffers)
134
- left_offset = 0
135
- for idx in range(n):
136
- shard_size = layer.lora_b_stacked[idx].shape[2]
137
-
138
- if layer.bias_stacked is not None:
139
- bias = layer.bias_stacked[idx]
140
- if bias is not None:
141
- bias = bias.view(-1, bias.shape[-1])
142
- bias = bias[layer.punica_wrapper.token_lora_indices]
143
- bias[layer.punica_wrapper.token_lora_indices == -1] = 0
144
- output[:, left_offset:left_offset + shard_size] += bias
145
-
146
- layer.punica_wrapper.add_expand_slice(
147
- output,
148
- buffers[idx],
149
- layer.lora_b_stacked[idx],
150
- left_offset,
151
- shard_size,
152
- add_input=True,
153
- )
154
- left_offset += shard_size
135
+ layer.punica_wrapper.add_expand_packed_nslice(
136
+ output,
137
+ buffers,
138
+ layer.lora_b_stacked,
139
+ layer.bias_stacked,
140
+ 1.0,
141
+ layer.output_slices,
142
+ )
143
144
output = output.view(*out_orig_shape)
145
# now have column partitioned and packed output
@@ -234,6 +222,7 @@ class QKVParallelLinearWithShardedLora(QKVParallelLinearWithLora):
222
self.punica_wrapper.add_expand(output,
223
buffer,
224
self.lora_b_stacked,
225
+ self.bias_all,
226
add_input=True)
227
# now have column partitioned output
228
output = output.view(*out_orig_shape)
@@ -350,15 +339,9 @@ class RowParallelLinearWithShardedLoRA(RowParallelLinearWithLoRA):
339
# reduced before being used
340
shard_size = self.lora_b_stacked.shape[2]
341
start_idx = self.tp_rank * shard_size
353
-
354
- if self.bias_stacked is not None:
355
- bias = self.bias_stacked.view(-1, self.bias_stacked.shape[-1])
356
- bias = bias[self.punica_wrapper.token_lora_indices]
357
- bias[self.punica_wrapper.token_lora_indices == -1] = 0
358
- output += bias
359
-
342
self.punica_wrapper.add_expand_slice(output, buffer,
361
- self.lora_b_stacked, start_idx,
343
+ self.lora_b_stacked,
344
+ self.bias_stacked, start_idx,
345
shard_size)
346
output = output.view(*out_orig_shape)
347
return output
vllm/lora/layers.py
+12
-101
@@ -67,63 +67,6 @@ def _not_fully_sharded_can_replace(can_replace):
67
return dec
68
69
70
-def apply_bias(
71
- indices: torch.Tensor,
72
- output: torch.Tensor,
73
- bias_stacked: torch.Tensor,
74
-):
75
- """Applies bias to output
76
-
77
- Input shapes:
78
- bias_stacked: (num_loras, output_dim)
79
- indices: (batch_size)
80
- output: (batch_size, output_dim)
81
- """
82
- org_output = output
83
- output = output.view(-1, output.shape[-1])
84
- indices = indices.view(-1)
85
-
86
- bias_stacked = bias_stacked.view(-1, bias_stacked.shape[-1])
87
- bias_stacked = bias_stacked[indices]
88
- bias_stacked[indices == -1] = 0
89
- output += bias_stacked
90
-
91
- return output.view_as(org_output)
92
-
93
-
94
-def apply_bias_packed_nslice(
95
- indices: torch.Tensor,
96
- output: torch.Tensor,
97
- output_slices: Tuple[int, ...],
98
- bias_stacked: Tuple[torch.Tensor, torch.Tensor, torch.Tensor],
99
-):
100
- """Applies bias to output
101
-
102
- Input shapes:
103
- bias_stacked: 3 element tuple of (num_loras, output_dim)
104
- indices: (batch_size)
105
- output: (batch_size, q_slice_size + 2*kv_slice_size)
106
- output_slices: n-1 element tuple of (slice_size...),
107
- where n is number of slices
108
- """
109
- org_output = output
110
- output = output.view(-1, output.shape[-1])
111
- indices = indices.view(-1)
112
-
113
- offset_left = 0
114
- for slice_idx, slice in enumerate(output_slices):
115
- bias = bias_stacked[slice_idx]
116
- if bias is not None:
117
- bias = bias.view(-1, bias.shape[-1])
118
- bias = bias[indices]
119
- bias[indices == -1] = 0
120
- output[:, offset_left:offset_left + slice] += bias
121
-
122
- offset_left += slice
123
-
124
- return output.view_as(org_output)
125
-
126
-
70
@dataclass
71
class LoRAMapping(AdapterMapping):
72
is_prefill: bool = False
@@ -311,6 +254,7 @@ class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA):
254
self.punica_wrapper.add_expand(full_output,
255
full_lora_a_embeddings,
256
self.lora_b_stacked,
257
+ bias_all=None,
258
add_input=True)
259
return full_output.view_as(full_output_org)
260
@@ -399,15 +343,9 @@ class ReplicatedLinearWithLoRA(BaseLayerWithLoRA):
343
def apply(self, x: torch.Tensor,
344
bias: Optional[torch.Tensor]) -> torch.Tensor:
345
output = self.base_layer.quant_method.apply(self.base_layer, x, bias)
402
- if self.bias_stacked is not None:
403
- self.indices = self.punica_wrapper.token_lora_indices
404
- output = apply_bias(
405
- self.indices,
406
- output,
407
- self.bias_stacked,
408
- )
346
self.punica_wrapper.add_lora(output, x, self.lora_a_stacked,
410
- self.lora_b_stacked, 1.0)
347
+ self.lora_b_stacked, self.bias_stacked,
348
+ 1.0)
349
return output
350
351
def forward(self, input_):
@@ -576,15 +514,9 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
514
def apply(self, x: torch.Tensor,
515
bias: Optional[torch.Tensor]) -> torch.Tensor:
516
output = self.base_layer.quant_method.apply(self.base_layer, x, bias)
579
- if self.bias_stacked is not None:
580
- self.indices = self.punica_wrapper.token_lora_indices
581
- output = apply_bias(
582
- self.indices,
583
- output,
584
- self.bias_stacked,
585
- )
517
self.punica_wrapper.add_lora(output, x, self.lora_a_stacked,
587
- self.lora_b_stacked, 1.0)
518
+ self.lora_b_stacked, self.bias_stacked,
519
+ 1.0)
520
return output
521
522
def forward(self, input_):
@@ -687,8 +619,8 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
619
) for _ in range(n_slices))
620
else:
621
self.bias_stacked = None
690
-
622
self.output_dim = self.lora_b_stacked[0].shape[2]
623
+ self.output_slices = (self.output_dim, self.output_dim)
624
625
def reset_lora(self, index: int):
626
self.lora_a_stacked[0][index] = 0
@@ -772,17 +704,9 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
704
def apply(self, x: torch.Tensor,
705
bias: Optional[torch.Tensor]) -> torch.Tensor:
706
output = self.base_layer.quant_method.apply(self.base_layer, x, bias)
775
- if self.bias_stacked is not None:
776
- self.indices = self.punica_wrapper.token_lora_indices
777
- output = apply_bias_packed_nslice(
778
- self.indices,
779
- output,
780
- (self.output_dim, self.output_dim),
781
- self.bias_stacked,
782
- )
707
self.punica_wrapper.add_lora_packed_nslice(
784
- output, x, self.lora_a_stacked, self.lora_b_stacked, 1.0,
785
- (self.output_dim, self.output_dim))
708
+ output, x, self.lora_a_stacked, self.lora_b_stacked,
709
+ self.bias_stacked, 1.0, (self.output_dim, self.output_dim))
710
return output
711
712
@classmethod
@@ -1129,17 +1053,10 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
1053
def apply(self, x: torch.Tensor,
1054
bias: Optional[torch.Tensor]) -> torch.Tensor:
1055
output = self.base_layer.quant_method.apply(self.base_layer, x, bias)
1132
- if self.bias_stacked is not None:
1133
- self.indices = self.punica_wrapper.token_lora_indices
1134
- output = apply_bias_packed_nslice(
1135
- self.indices,
1136
- output,
1137
- self.output_slices,
1138
- self.bias_stacked,
1139
- )
1056
self.punica_wrapper.add_lora_packed_nslice(output, x,
1057
self.lora_a_stacked,
1142
- self.lora_b_stacked, 1.0,
1058
+ self.lora_b_stacked,
1059
+ self.bias_stacked, 1.0,
1060
self.output_slices)
1061
return output
1062
@@ -1264,15 +1181,9 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
1181
1182
def apply(self, x: torch.Tensor) -> torch.Tensor:
1183
output = self.base_layer.quant_method.apply(self.base_layer, x)
1267
- if self.bias_stacked is not None:
1268
- self.indices = self.punica_wrapper.token_lora_indices
1269
- output = apply_bias(
1270
- self.indices,
1271
- output,
1272
- self.bias_stacked,
1273
- )
1184
self.punica_wrapper.add_lora(output, x, self.lora_a_stacked,
1275
- self.lora_b_stacked, 1.0)
1185
+ self.lora_b_stacked, self.bias_stacked,
1186
+ 1.0)
1187
return output
1188
1189
def forward(self, input_):
vllm/lora/punica.py
+105
-12
@@ -450,6 +450,62 @@ class PunicaWrapper:
450
bgmv_expand_slice(x, w_t_all, y, self.token_lora_indices, y_offset,
451
y_slice_size, add_input)
452
453
+ def apply_bias(
454
+ self,
455
+ indices: torch.Tensor,
456
+ output: torch.Tensor,
457
+ bias_stacked: torch.Tensor,
458
+ ):
459
+ """Applies bias to output
460
+
461
+ Input shapes:
462
+ bias_stacked: (num_loras, output_dim)
463
+ indices: (batch_size)
464
+ output: (batch_size, output_dim)
465
+ """
466
+ org_output = output
467
+ output = output.view(-1, output.shape[-1])
468
+ indices = indices.view(-1)
469
+
470
+ bias_stacked = bias_stacked.view(-1, bias_stacked.shape[-1])
471
+ bias_stacked = bias_stacked[indices]
472
+ bias_stacked[indices == -1] = 0
473
+ output += bias_stacked
474
+
475
+ return output.view_as(org_output)
476
+
477
+ def apply_bias_packed_nslice(
478
+ self,
479
+ indices: torch.Tensor,
480
+ output: torch.Tensor,
481
+ output_slices: Tuple[int, ...],
482
+ bias_stacked: Tuple[Optional[torch.Tensor], ...],
483
+ ):
484
+ """Applies bias to output
485
+
486
+ Input shapes:
487
+ bias_stacked: 3 element tuple of (num_loras, output_dim)
488
+ indices: (batch_size)
489
+ output: (batch_size, q_slice_size + 2*kv_slice_size)
490
+ output_slices: n-1 element tuple of (slice_size...),
491
+ where n is number of slices
492
+ """
493
+ org_output = output
494
+ output = output.view(-1, output.shape[-1])
495
+ indices = indices.view(-1)
496
+
497
+ offset_left = 0
498
+ for slice_idx, slice in enumerate(output_slices):
499
+ bias = bias_stacked[slice_idx]
500
+ if bias is not None:
501
+ bias = bias.view(-1, bias.shape[-1])
502
+ bias = bias[indices]
503
+ bias[indices == -1] = 0
504
+ output[:, offset_left:offset_left + slice] += bias
505
+ offset_left += slice
506
+
507
+ return output.view_as(org_output)
508
+
509
def add_shrink(
510
self,
511
y: torch.Tensor,
@@ -474,16 +530,19 @@ class PunicaWrapper:
530
y: torch.Tensor,
531
x: torch.Tensor,
532
w_t_all: torch.Tensor,
533
+ bias_all: Optional[torch.Tensor],
534
add_input: bool = True,
535
):
536
"""
480
- Perform the ` y+=x@w_t_all` computation, which is suitable for the
537
+ Perform the ` y+=x@w_t_all+bias` computation, which is suitable for the
538
GEMM of lora'b.
539
When `is_prefill` is true, it indicates that it is currently the
540
prefill stage, and the `expand_prefill` function should be called.
541
Otherwise, it is the decode stage, and the expand_decode function
542
should be called.
543
"""
544
+ if bias_all is not None:
545
+ y = self.apply_bias(self.token_lora_indices, y, bias_all)
546
547
expand_fun: Callable = (self.expand_prefill
548
if self.is_prefill else self.expand_decode)
@@ -493,23 +552,54 @@ class PunicaWrapper:
552
y: torch.Tensor,
553
x: torch.Tensor,
554
w_t_all: torch.Tensor,
555
+ bias_all: Optional[torch.Tensor],
556
y_offset: Optional[int],
557
y_slice_size: Optional[int],
558
add_input: bool = True):
559
"""
560
Similar to `add_expand`
561
"""
562
+ if bias_all is not None:
563
+ y = self.apply_bias(self.token_lora_indices, y, bias_all)
564
565
expand_slice_fun: Callable = (self.expand_slice_prefill
566
if self.is_prefill else
567
self.expand_slice_decode)
568
expand_slice_fun(y, x, w_t_all, y_offset, y_slice_size, add_input)
569
570
+ def add_expand_packed_nslice(self, y: torch.Tensor, x: torch.Tensor,
571
+ lora_b_stacked: Tuple[torch.Tensor, ...],
572
+ bias_stacked: Optional[Tuple[torch.Tensor,
573
+ ...]],
574
+ scale: float,
575
+ output_slices: Tuple[int, ...]) -> None:
576
+ """
577
+ Similar to `add_expand`
578
+ """
579
+ y_org = y
580
+ y = y.view(-1, y.shape[-1])
581
+ offset_left = 0
582
+ if bias_stacked is not None:
583
+ self.apply_bias_packed_nslice(self.token_lora_indices, y,
584
+ output_slices, bias_stacked)
585
+ for slice_idx in range(len(lora_b_stacked)):
586
+ self.add_expand_slice(y,
587
+ x[slice_idx],
588
+ lora_b_stacked[slice_idx],
589
+ None,
590
+ offset_left,
591
+ output_slices[slice_idx],
592
+ add_input=True)
593
+ offset_left += output_slices[slice_idx]
594
+
595
+ y = y.view_as(y_org)
596
+
597
def add_lora(self,
598
y: torch.Tensor,
599
x: torch.Tensor,
600
wa_t_all: torch.Tensor,
601
wb_t_all: torch.Tensor,
602
+ bias_all: Optional[torch.Tensor],
603
scale: float,
604
y_offset: Optional[int] = None,
605
y_slice_size: Optional[int] = None,
@@ -522,12 +612,13 @@ class PunicaWrapper:
612
@ wa_t_all[indices[i], layer_idx, :, :].transpose(-1, -2)
613
@ wb_t_all[indices[i], layer_idx, :, :].transpose(-1, -2)
614
* scale
525
- ).squeeze(0)
615
+ ).squeeze(0)+bias[i]
616
Args:
617
y (torch.Tensor): Output tensor. Will be changed in-place.
618
x (torch.Tensor): Input tensor
619
wa_t_all (torch.Tensor): lora_a's weight
620
wb_t_all (torch.Tensor): lora_b's weight
621
+ bias_all: (torch.Tensor): lora's bias
622
scale (float): Scaling factor.
623
y_offset (Optional[int], optional): Offset to apply to the starting
624
column of y.
@@ -544,27 +635,26 @@ class PunicaWrapper:
635
buffer = torch.zeros((x.size(0), r),
636
dtype=torch.float32,
637
device=x.device)
547
-
638
+ if bias_all is not None:
639
+ y = self.apply_bias(self.token_lora_indices, y, bias_all)
640
self.add_shrink(buffer, x, wa_t_all, scale)
641
if y_offset is None and y_slice_size is None:
550
- self.add_expand(y, buffer, wb_t_all, add_input=True)
642
+ self.add_expand(y, buffer, wb_t_all, bias_all=None, add_input=True)
643
else:
644
self.add_expand_slice(y,
645
buffer,
646
wb_t_all,
647
+ None,
648
y_offset,
649
y_slice_size,
650
add_input=True)
651
y = y.view_as(y_org)
652
653
def add_lora_packed_nslice(self, y: torch.Tensor, x: torch.Tensor,
561
- lora_a_stacked: Tuple[torch.Tensor,
562
- torch.Tensor,
563
- torch.Tensor],
564
- lora_b_stacked: Tuple[torch.Tensor,
565
- torch.Tensor,
566
- torch.Tensor],
567
- scale: float,
654
+ lora_a_stacked: Tuple[torch.Tensor, ...],
655
+ lora_b_stacked: Tuple[torch.Tensor, ...],
656
+ bias_all: Tuple[Optional[torch.Tensor],
657
+ ...], scale: float,
658
output_slices: Tuple[int, ...]) -> None:
659
"""
660
Applies lora to each input. Similar to add_lora, This method is
@@ -575,10 +665,13 @@ class PunicaWrapper:
665
x = x.view(-1, x.shape[-1])
666
y = y.view(-1, y.shape[-1])
667
offset_left = 0
668
+ if bias_all is not None:
669
+ y = self.apply_bias_packed_nslice(self.token_lora_indices, y,
670
+ output_slices, bias_all)
671
# TODO fuse these kernels
672
for slice_idx in range(len(output_slices)):
673
self.add_lora(y, x, lora_a_stacked[slice_idx],
581
- lora_b_stacked[slice_idx], scale, offset_left,
674
+ lora_b_stacked[slice_idx], None, scale, offset_left,
675
output_slices[slice_idx])
676
offset_left += output_slices[slice_idx]
677