A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[LoRA] Adds support for bias in LoRA (#5733)
Signed-off-by: Umesh Deshpande <udeshpa@us.ibm.com> Co-authored-by: Umesh Deshpande <udeshpa@us.ibm.com>
Umesh committed
Nov 12, 2024 at 11:08 UTC
8a06428c70657b3310a317b3caf3c562b0e042ae
10 files changed
+456
-20
tests/lora/conftest.py
+5
@@ -152,6 +152,11 @@ def sql_lora_files(sql_lora_huggingface_id):
152
return snapshot_download(repo_id=sql_lora_huggingface_id)
153
154
155
+@pytest.fixture(scope="session")
156
+def lora_bias_files():
157
+ return snapshot_download(repo_id="followumesh/granite-3b-lora8-bias")
158
+
159
+
160
@pytest.fixture(scope="session")
161
def mixtral_lora_files():
162
# Note: this module has incorrect adapter_config.json to test
tests/lora/test_lora_bias_e2e.py
new
+52
@@ -0,0 +1,52 @@
1
+from typing import List
2
+
3
+import pytest
4
+
5
+import vllm
6
+from vllm.lora.request import LoRARequest
7
+
8
+MODEL_PATH = "ibm-granite/granite-3b-code-base"
9
+
10
+
11
+def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:
12
+ prompts = [
13
+ "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE candidate (people_id VARCHAR, unsure_rate INTEGER); CREATE TABLE people (sex VARCHAR, people_id VARCHAR)\n\n question: which gender got the highest average uncertain ratio. [/user] [assistant]", # noqa: E501
14
+ "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_28138035_4 (womens_doubles VARCHAR, mens_singles VARCHAR)\n\n question: Name the women's doubles for werner schlager [/user] [assistant]" # noqa: E501
15
+ ]
16
+ sampling_params = vllm.SamplingParams(temperature=0,
17
+ max_tokens=256,
18
+ stop=["[/assistant]"])
19
+ outputs = llm.generate(
20
+ prompts,
21
+ sampling_params,
22
+ lora_request=LoRARequest(str(lora_id), lora_id, lora_path)
23
+ if lora_id else None)
24
+ generated_texts: List[str] = []
25
+ for output in outputs:
26
+ generated_text = output.outputs[0].text
27
+ generated_texts.append(generated_text)
28
+ return generated_texts
29
+
30
+
31
+@pytest.mark.parametrize("lora_bias", [True])
32
+@pytest.mark.parametrize("fully_sharded", [True, False])
33
+def test_lora_bias(lora_bias_files: str, lora_bias: bool, fully_sharded: bool):
34
+ llm = vllm.LLM(MODEL_PATH,
35
+ enable_lora=True,
36
+ max_num_seqs=16,
37
+ max_lora_rank=8,
38
+ max_loras=1,
39
+ enable_lora_bias=lora_bias,
40
+ tensor_parallel_size=1,
41
+ fully_sharded_loras=fully_sharded)
42
+
43
+ print("lora adapter created")
44
+ output1 = do_sample(llm, lora_bias_files, lora_id=0)
45
+
46
+ print("lora")
47
+ output2 = do_sample(llm, lora_bias_files, lora_id=1)
48
+
49
+ if lora_bias:
50
+ assert output1 != output2
51
+ else:
52
+ assert output1 == output2
tests/lora/test_utils.py
+9
-5
@@ -12,36 +12,40 @@ from vllm.utils import LRUCache
12
13
def test_parse_fine_tuned_lora_name_valid():
14
fixture = {
15
- ("base_model.model.lm_head.lora_A.weight", "lm_head", True),
16
- ("base_model.model.lm_head.lora_B.weight", "lm_head", False),
15
+ ("base_model.model.lm_head.lora_A.weight", "lm_head", True, False),
16
+ ("base_model.model.lm_head.lora_B.weight", "lm_head", False, False),
17
(
18
"base_model.model.model.embed_tokens.lora_embedding_A",
19
"model.embed_tokens",
20
True,
21
+ False,
22
),
23
(
24
"base_model.model.model.embed_tokens.lora_embedding_B",
25
"model.embed_tokens",
26
False,
27
+ False,
28
),
29
(
30
"base_model.model.model.layers.9.mlp.down_proj.lora_A.weight",
31
"model.layers.9.mlp.down_proj",
32
True,
33
+ False,
34
),
35
(
36
"base_model.model.model.layers.9.mlp.down_proj.lora_B.weight",
37
"model.layers.9.mlp.down_proj",
38
False,
39
+ False,
40
),
41
}
38
- for name, module_name, is_lora_a in fixture:
39
- assert (module_name, is_lora_a) == parse_fine_tuned_lora_name(name)
42
+ for name, module_name, is_lora_a, is_bias in fixture:
43
+ assert (module_name, is_lora_a,
44
+ is_bias) == parse_fine_tuned_lora_name(name)
45
46
47
def test_parse_fine_tuned_lora_name_invalid():
48
fixture = {
44
- "weight",
49
"base_model.weight",
50
"base_model.model.weight",
51
}
vllm/config.py
+1
@@ -1687,6 +1687,7 @@ class LoRAConfig:
1687
# This is a constant.
1688
lora_vocab_padding_size: ClassVar[int] = 256
1689
long_lora_scaling_factors: Optional[Tuple[float]] = None
1690
+ bias_enabled: bool = False
1691
1692
def __post_init__(self):
1693
# Setting the maximum rank to 256 should be able to satisfy the vast
vllm/engine/arg_utils.py
+5
@@ -143,6 +143,7 @@ class EngineArgs:
143
limit_mm_per_prompt: Optional[Mapping[str, int]] = None
144
mm_processor_kwargs: Optional[Dict[str, Any]] = None
145
enable_lora: bool = False
146
+ enable_lora_bias: bool = False
147
max_loras: int = 1
148
max_lora_rank: int = 16
149
enable_prompt_adapter: bool = False
@@ -584,6 +585,9 @@ class EngineArgs:
585
parser.add_argument('--enable-lora',
586
action='store_true',
587
help='If True, enable handling of LoRA adapters.')
588
+ parser.add_argument('--enable-lora-bias',
589
+ action='store_true',
590
+ help='If True, enable bias for LoRA adapters.')
591
parser.add_argument('--max-loras',
592
type=int,
593
default=EngineArgs.max_loras,
@@ -1148,6 +1152,7 @@ class EngineArgs:
1152
and parallel_config.use_ray),
1153
policy=self.scheduling_policy)
1154
lora_config = LoRAConfig(
1155
+ bias_enabled=self.enable_lora_bias,
1156
max_lora_rank=self.max_lora_rank,
1157
max_loras=self.max_loras,
1158
fully_sharded_loras=self.fully_sharded_loras,
vllm/lora/fully_sharded_layers.py
+33
@@ -70,6 +70,14 @@ class ColumnParallelLinearWithShardedLoRA(ColumnParallelLinearWithLoRA):
70
self.lora_b_stacked,
71
add_input=True)
72
# now have column partitioned output
73
+
74
+ if self.bias_stacked is not None:
75
+ self.bias_stacked = self.bias_stacked.view(
76
+ -1, self.bias_stacked.shape[-1])
77
+ self.bias_stacked = self.bias_stacked[
78
+ self.punica_wrapper.token_lora_indices]
79
+ output += self.bias_stacked
80
+
81
output = output.view(*out_orig_shape)
82
return output
83
@@ -121,6 +129,15 @@ def _mcp_apply(x, bias, layer: QKVParallelLinearWithLora):
129
left_offset = 0
130
for idx in range(n):
131
shard_size = layer.lora_b_stacked[idx].shape[2]
132
+
133
+ if layer.bias_stacked is not None:
134
+ bias = layer.bias_stacked[idx]
135
+ if bias is not None:
136
+ bias = bias.view(-1, bias.shape[-1])
137
+ bias = bias[layer.punica_wrapper.token_lora_indices]
138
+ bias[layer.punica_wrapper.token_lora_indices == -1] = 0
139
+ output[:, left_offset:left_offset + shard_size] += bias
140
+
141
layer.punica_wrapper.add_expand_slice(
142
output,
143
buffers[idx],
@@ -295,6 +312,15 @@ class RowParallelLinearWithShardedLoRA(RowParallelLinearWithLoRA):
312
lora_b = lora_b[:, start_idx:end_idx]
313
return lora_b
314
315
+ def slice_bias(self, bias: torch.Tensor) -> torch.Tensor:
316
+ if bias is None:
317
+ return bias
318
+ shard_size = self.bias_stacked.shape[2]
319
+ start_idx = self.tp_rank * shard_size
320
+ end_idx = (self.tp_rank + 1) * shard_size
321
+ bias = bias[start_idx:end_idx]
322
+ return bias
323
+
324
def apply(self, x: torch.Tensor) -> torch.Tensor:
325
output = self.base_layer.quant_method.apply(self.base_layer, x)
326
@@ -318,6 +344,13 @@ class RowParallelLinearWithShardedLoRA(RowParallelLinearWithLoRA):
344
# reduced before being used
345
shard_size = self.lora_b_stacked.shape[2]
346
start_idx = self.tp_rank * shard_size
347
+
348
+ if self.bias_stacked is not None:
349
+ bias = self.bias_stacked.view(-1, self.bias_stacked.shape[-1])
350
+ bias = bias[self.punica_wrapper.token_lora_indices]
351
+ bias[self.punica_wrapper.token_lora_indices == -1] = 0
352
+ output += bias
353
+
354
self.punica_wrapper.add_expand_slice(output, buffer,
355
self.lora_b_stacked, start_idx,
356
shard_size)
vllm/lora/layers.py
+295
-1
@@ -67,6 +67,63 @@ def _not_fully_sharded_can_replace(can_replace):
67
return dec
68
69
70
+def apply_bias(
71
+ indices: torch.Tensor,
72
+ output: torch.Tensor,
73
+ bias_stacked: torch.Tensor,
74
+):
75
+ """Applies bias to output
76
+
77
+ Input shapes:
78
+ bias_stacked: (num_loras, output_dim)
79
+ indices: (batch_size)
80
+ output: (batch_size, output_dim)
81
+ """
82
+ org_output = output
83
+ output = output.view(-1, output.shape[-1])
84
+ indices = indices.view(-1)
85
+
86
+ bias_stacked = bias_stacked.view(-1, bias_stacked.shape[-1])
87
+ bias_stacked = bias_stacked[indices]
88
+ bias_stacked[indices == -1] = 0
89
+ output += bias_stacked
90
+
91
+ return output.view_as(org_output)
92
+
93
+
94
+def apply_bias_packed_nslice(
95
+ indices: torch.Tensor,
96
+ output: torch.Tensor,
97
+ output_slices: Tuple[int, ...],
98
+ bias_stacked: Tuple[torch.Tensor, torch.Tensor, torch.Tensor],
99
+):
100
+ """Applies bias to output
101
+
102
+ Input shapes:
103
+ bias_stacked: 3 element tuple of (num_loras, output_dim)
104
+ indices: (batch_size)
105
+ output: (batch_size, q_slice_size + 2*kv_slice_size)
106
+ output_slices: n-1 element tuple of (slice_size...),
107
+ where n is number of slices
108
+ """
109
+ org_output = output
110
+ output = output.view(-1, output.shape[-1])
111
+ indices = indices.view(-1)
112
+
113
+ offset_left = 0
114
+ for slice_idx, slice in enumerate(output_slices):
115
+ bias = bias_stacked[slice_idx]
116
+ if bias is not None:
117
+ bias = bias.view(-1, bias.shape[-1])
118
+ bias = bias[indices]
119
+ bias[indices == -1] = 0
120
+ output[:, offset_left:offset_left + slice] += bias
121
+
122
+ offset_left += slice
123
+
124
+ return output.view_as(org_output)
125
+
126
+
127
@dataclass
128
class LoRAMapping(AdapterMapping):
129
is_prefill: bool = False
@@ -105,6 +162,7 @@ class BaseLayerWithLoRA(nn.Module):
162
lora_a: torch.Tensor,
163
lora_b: torch.Tensor,
164
embeddings_tensor: Optional[torch.Tensor],
165
+ bias: Optional[torch.Tensor] = None,
166
):
167
"""Overwrites lora tensors at index."""
168
...
@@ -203,6 +261,7 @@ class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA):
261
lora_a: torch.Tensor,
262
lora_b: torch.Tensor,
263
embeddings_tensor: Optional[torch.Tensor],
264
+ bias: Optional[torch.Tensor] = None,
265
):
266
self.reset_lora(index)
267
self.lora_a_stacked[index, :lora_a.shape[0], :lora_a.shape[1]].copy_(
@@ -299,10 +358,22 @@ class ReplicatedLinearWithLoRA(BaseLayerWithLoRA):
358
dtype=lora_config.lora_dtype,
359
device=self.device,
360
)
361
+ if lora_config.bias_enabled:
362
+ self.bias_stacked = torch.zeros(
363
+ max_loras,
364
+ 1,
365
+ self.output_size,
366
+ dtype=lora_config.lora_dtype,
367
+ device=self.device,
368
+ )
369
+ else:
370
+ self.bias_stacked = None
371
372
def reset_lora(self, index: int):
373
self.lora_a_stacked[index] = 0
374
self.lora_b_stacked[index] = 0
375
+ if self.lora_config.bias_enabled:
376
+ self.bias_stacked[index] = 0
377
378
def set_lora(
379
self,
@@ -310,6 +381,7 @@ class ReplicatedLinearWithLoRA(BaseLayerWithLoRA):
381
lora_a: torch.Tensor,
382
lora_b: torch.Tensor,
383
embeddings_tensor: Optional[torch.Tensor],
384
+ bias: Optional[torch.Tensor] = None,
385
):
386
self.reset_lora(index)
387
@@ -319,10 +391,21 @@ class ReplicatedLinearWithLoRA(BaseLayerWithLoRA):
391
self.lora_b_stacked[index,
392
0, :lora_b.shape[1], :lora_b.shape[0]].copy_(
393
lora_b.T, non_blocking=True)
394
+ if bias is not None:
395
+ self.bias_stacked[index,
396
+ 0, :bias.shape[0]].copy_(bias.T,
397
+ non_blocking=True)
398
399
def apply(self, x: torch.Tensor,
400
bias: Optional[torch.Tensor]) -> torch.Tensor:
401
output = self.base_layer.quant_method.apply(self.base_layer, x, bias)
402
+ if self.bias_stacked is not None:
403
+ self.indices = self.punica_wrapper.token_lora_indices
404
+ output = apply_bias(
405
+ self.indices,
406
+ output,
407
+ self.bias_stacked,
408
+ )
409
self.punica_wrapper.add_lora(output, x, self.lora_a_stacked,
410
self.lora_b_stacked, 1.0)
411
return output
@@ -401,11 +484,25 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
484
dtype=lora_config.lora_dtype,
485
device=self.device,
486
)
487
+
488
+ if lora_config.bias_enabled:
489
+ self.bias_stacked = torch.zeros(
490
+ max_loras,
491
+ 1,
492
+ self.output_size,
493
+ dtype=lora_config.lora_dtype,
494
+ device=self.device,
495
+ )
496
+ else:
497
+ self.bias_stacked = None
498
+
499
self.output_dim = self.lora_b_stacked.shape[2]
500
501
def reset_lora(self, index: int):
502
self.lora_a_stacked[index] = 0
503
self.lora_b_stacked[index] = 0
504
+ if self.lora_config.bias_enabled:
505
+ self.bias_stacked[index] = 0
506
507
def slice_lora_a(self, lora_a: torch.Tensor) -> torch.Tensor:
508
return lora_a
@@ -418,18 +515,30 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
515
lora_b = lora_b[:, start_idx:end_idx]
516
return lora_b
517
518
+ def slice_bias(self, bias: torch.Tensor) -> torch.Tensor:
519
+ if bias is None:
520
+ return bias
521
+ tensor_model_parallel_rank = get_tensor_model_parallel_rank()
522
+ shard_size = self.output_dim
523
+ start_idx = tensor_model_parallel_rank * shard_size
524
+ end_idx = (tensor_model_parallel_rank + 1) * shard_size
525
+ bias = bias[start_idx:end_idx]
526
+ return bias
527
+
528
def set_lora(
529
self,
530
index: int,
531
lora_a: torch.Tensor,
532
lora_b: torch.Tensor,
533
embeddings_tensor: Optional[torch.Tensor],
534
+ bias: Optional[torch.Tensor] = None,
535
):
536
self.reset_lora(index)
537
538
if self.tp_size > 1:
539
lora_a = self.slice_lora_a(lora_a)
540
lora_b = self.slice_lora_b(lora_b)
541
+ bias = self.slice_bias(bias)
542
543
self.lora_a_stacked[index,
544
0, :lora_a.shape[1], :lora_a.shape[0]].copy_(
@@ -437,10 +546,21 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
546
self.lora_b_stacked[index,
547
0, :lora_b.shape[1], :lora_b.shape[0]].copy_(
548
lora_b.T, non_blocking=True)
549
+ if bias is not None:
550
+ self.bias_stacked[index,
551
+ 0, :bias.shape[0]].copy_(bias.T,
552
+ non_blocking=True)
553
554
def apply(self, x: torch.Tensor,
555
bias: Optional[torch.Tensor]) -> torch.Tensor:
556
output = self.base_layer.quant_method.apply(self.base_layer, x, bias)
557
+ if self.bias_stacked is not None:
558
+ self.indices = self.punica_wrapper.token_lora_indices
559
+ output = apply_bias(
560
+ self.indices,
561
+ output,
562
+ self.bias_stacked,
563
+ )
564
self.punica_wrapper.add_lora(output, x, self.lora_a_stacked,
565
self.lora_b_stacked, 1.0)
566
return output
@@ -534,6 +654,17 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
654
dtype=lora_config.lora_dtype,
655
device=self.device,
656
) for _ in range(n_slices))
657
+ if lora_config.bias_enabled:
658
+ self.bias_stacked = tuple(
659
+ torch.zeros(
660
+ max_loras,
661
+ 1,
662
+ self.output_size // 2,
663
+ dtype=lora_config.lora_dtype,
664
+ device=self.device,
665
+ ) for _ in range(n_slices))
666
+ else:
667
+ self.bias_stacked = None
668
669
self.output_dim = self.lora_b_stacked[0].shape[2]
670
@@ -542,6 +673,9 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
673
self.lora_a_stacked[1][index] = 0
674
self.lora_b_stacked[0][index] = 0
675
self.lora_b_stacked[1][index] = 0
676
+ if self.lora_config.bias_enabled:
677
+ self.bias_stacked[0][index] = 0
678
+ self.bias_stacked[1][index] = 0
679
680
def slice_lora_a(
681
self, lora_a: List[Union[torch.Tensor, None]]
@@ -562,18 +696,32 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
696
]
697
return lora_b
698
699
+ def slice_bias(
700
+ self, bias: List[Union[torch.Tensor,
701
+ None]]) -> List[Union[torch.Tensor, None]]:
702
+ if bias[0] is None or bias[1] is None:
703
+ return bias
704
+ shard_size = self.output_dim
705
+ start_idx = self.tp_rank * shard_size
706
+ end_idx = (self.tp_rank + 1) * shard_size
707
+ bias = [bias[0][start_idx:end_idx], bias[1][start_idx:end_idx]]
708
+ return bias
709
+
710
def set_lora(
711
self,
712
index: int,
713
lora_a: torch.Tensor,
714
lora_b: torch.Tensor,
715
embeddings_tensor: Optional[torch.Tensor],
716
+ bias: Optional[torch.Tensor] = None,
717
):
718
self.reset_lora(index)
719
720
if self.tp_size > 1:
721
lora_a = self.slice_lora_a(lora_a)
722
lora_b = self.slice_lora_b(lora_b)
723
+ if bias is not None:
724
+ bias = self.slice_bias(bias)
725
726
if lora_a[0] is not None:
727
self.lora_a_stacked[0][
@@ -582,6 +730,10 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
730
self.lora_b_stacked[0][
731
index, 0, :lora_b[0].shape[1], :lora_b[0].shape[0]].copy_(
732
lora_b[0].T, non_blocking=True)
733
+ if bias is not None and bias[0] is not None:
734
+ self.bias_stacked[0][index,
735
+ 0, :bias[0].shape[0]].copy_(bias[0].T,
736
+ non_blocking=True)
737
if lora_a[1] is not None:
738
self.lora_a_stacked[1][
739
index, 0, :lora_a[1].shape[1], :lora_a[1].shape[0]].copy_(
@@ -589,10 +741,22 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
741
self.lora_b_stacked[1][
742
index, 0, :lora_b[1].shape[1], :lora_b[1].shape[0]].copy_(
743
lora_b[1].T, non_blocking=True)
744
+ if bias is not None and bias[1] is not None:
745
+ self.bias_stacked[1][index,
746
+ 0, :bias[1].shape[0]].copy_(bias[1].T,
747
+ non_blocking=True)
748
749
def apply(self, x: torch.Tensor,
750
bias: Optional[torch.Tensor]) -> torch.Tensor:
751
output = self.base_layer.quant_method.apply(self.base_layer, x, bias)
752
+ if self.bias_stacked is not None:
753
+ self.indices = self.punica_wrapper.token_lora_indices
754
+ output = apply_bias_packed_nslice(
755
+ self.indices,
756
+ output,
757
+ (self.output_dim, self.output_dim),
758
+ self.bias_stacked,
759
+ )
760
self.punica_wrapper.add_lora_packed_nslice(
761
output, x, self.lora_a_stacked, self.lora_b_stacked, 1.0,
762
(self.output_dim, self.output_dim))
@@ -654,17 +818,35 @@ class QKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
818
lora_b = torch.cat([lora_b_q, lora_b_k, lora_b_v], dim=1)
819
return lora_b
820
821
+ def slice_bias(self, bias: torch.Tensor) -> torch.Tensor:
822
+ bias_q = bias[self.q_proj_shard_size *
823
+ self.q_shard_id:self.q_proj_shard_size *
824
+ (self.q_shard_id + 1)]
825
+ k_offset = self.q_proj_total_size
826
+ bias_k = bias[k_offset +
827
+ self.kv_proj_shard_size * self.kv_shard_id:k_offset +
828
+ self.kv_proj_shard_size * (self.kv_shard_id + 1)]
829
+ v_offset = k_offset + self.kv_proj_total_size
830
+ bias_v = bias[v_offset +
831
+ self.kv_proj_shard_size * self.kv_shard_id:v_offset +
832
+ self.kv_proj_shard_size * (self.kv_shard_id + 1)]
833
+ bias = torch.cat([bias_q, bias_k, bias_v], dim=1)
834
+ return bias
835
+
836
def set_lora(
837
self,
838
index: int,
839
lora_a: torch.Tensor,
840
lora_b: torch.Tensor,
841
embeddings_tensor: Optional[torch.Tensor],
842
+ bias: Optional[torch.Tensor] = None,
843
):
844
self.reset_lora(index)
845
if self.tp_size > 1:
846
lora_a = self.slice_lora_a(lora_a)
847
lora_b = self.slice_lora_b(lora_b)
848
+ if bias is not None:
849
+ bias = self.slice_bias(bias)
850
851
self.lora_a_stacked[index,
852
0, :lora_a.shape[1], :lora_a.shape[0]].copy_(
@@ -672,6 +854,10 @@ class QKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
854
self.lora_b_stacked[index,
855
0, :lora_b.shape[1], :lora_b.shape[0]].copy_(
856
lora_b.T, non_blocking=True)
857
+ if bias is not None:
858
+ self.bias_stacked[index,
859
+ 0, :bias.shape[0]].copy_(bias.T,
860
+ non_blocking=True)
861
862
@classmethod
863
@_not_fully_sharded_can_replace
@@ -768,6 +954,32 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
954
device=self.device,
955
),
956
)
957
+ if lora_config.bias_enabled:
958
+ self.bias_stacked = (
959
+ torch.zeros(
960
+ max_loras,
961
+ 1,
962
+ self.q_proj_shard_size,
963
+ dtype=lora_config.lora_dtype,
964
+ device=self.device,
965
+ ),
966
+ torch.zeros(
967
+ max_loras,
968
+ 1,
969
+ self.kv_proj_shard_size,
970
+ dtype=lora_config.lora_dtype,
971
+ device=self.device,
972
+ ),
973
+ torch.zeros(
974
+ max_loras,
975
+ 1,
976
+ self.kv_proj_shard_size,
977
+ dtype=lora_config.lora_dtype,
978
+ device=self.device,
979
+ ),
980
+ )
981
+ else:
982
+ self.bias_stacked = None
983
984
self.output_slices = (
985
self.q_proj_shard_size,
@@ -787,6 +999,10 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
999
self.lora_b_stacked[1][index] = 0
1000
self.lora_a_stacked[2][index] = 0
1001
self.lora_b_stacked[2][index] = 0
1002
+ if self.lora_config.bias_enabled:
1003
+ self.bias_stacked[0][index] = 0
1004
+ self.bias_stacked[1][index] = 0
1005
+ self.bias_stacked[2][index] = 0
1006
1007
def slice_lora_a(
1008
self, lora_a: List[Union[torch.Tensor, None]]
@@ -812,18 +1028,40 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
1028
lora_b = [lora_b_q, lora_b_k, lora_b_v]
1029
return lora_b
1030
1031
+ def slice_bias(
1032
+ self, bias: List[Union[torch.Tensor,
1033
+ None]]) -> List[Union[torch.Tensor, None]]:
1034
+ bias_q, bias_k, bias_v = bias
1035
+ if bias_q is not None:
1036
+ bias_q = bias_q[self.q_proj_shard_size *
1037
+ self.q_shard_id:self.q_proj_shard_size *
1038
+ (self.q_shard_id + 1)]
1039
+ if bias_k is not None:
1040
+ bias_k = bias_k[self.kv_proj_shard_size *
1041
+ self.kv_shard_id:self.kv_proj_shard_size *
1042
+ (self.kv_shard_id + 1)]
1043
+ if bias_v is not None:
1044
+ bias_v = bias_v[self.kv_proj_shard_size *
1045
+ self.kv_shard_id:self.kv_proj_shard_size *
1046
+ (self.kv_shard_id + 1)]
1047
+ bias = [bias_q, bias_k, bias_v]
1048
+ return bias
1049
+
1050
def set_lora(
1051
self,
1052
index: int,
1053
lora_a: torch.Tensor,
1054
lora_b: torch.Tensor,
1055
embeddings_tensor: Optional[torch.Tensor],
1056
+ bias: Optional[torch.Tensor] = None,
1057
):
1058
self.reset_lora(index)
1059
1060
if self.tp_size > 1:
1061
lora_a = self.slice_lora_a(lora_a)
1062
lora_b = self.slice_lora_b(lora_b)
1063
+ if bias is not None:
1064
+ bias = self.slice_bias(bias)
1065
1066
if lora_b[0] is not None:
1067
lora_b_q = lora_b[0]
@@ -854,9 +1092,28 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
1092
index, 0, :lora_a[2].shape[1], :lora_a[2].shape[0]].copy_(
1093
lora_a[2].T, non_blocking=True)
1094
1095
+ if bias is not None:
1096
+ if bias[0] is not None:
1097
+ self.bias_stacked[0][index, 0, :bias[0].shape[0]].copy_(
1098
+ bias[0].T, non_blocking=True)
1099
+ if bias[1] is not None:
1100
+ self.bias_stacked[1][index, 0, :bias[1].shape[0]].copy_(
1101
+ bias[1].T, non_blocking=True)
1102
+ if bias[2] is not None:
1103
+ self.bias_stacked[2][index, 0, :bias[2].shape[0]].copy_(
1104
+ bias[2].T, non_blocking=True)
1105
+
1106
def apply(self, x: torch.Tensor,
1107
bias: Optional[torch.Tensor]) -> torch.Tensor:
1108
output = self.base_layer.quant_method.apply(self.base_layer, x, bias)
1109
+ if self.bias_stacked is not None:
1110
+ self.indices = self.punica_wrapper.token_lora_indices
1111
+ output = apply_bias_packed_nslice(
1112
+ self.indices,
1113
+ output,
1114
+ self.output_slices,
1115
+ self.bias_stacked,
1116
+ )
1117
self.punica_wrapper.add_lora_packed_nslice(output, x,
1118
self.lora_a_stacked,
1119
self.lora_b_stacked, 1.0,
@@ -919,9 +1176,27 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
1176
device=self.device,
1177
)
1178
1179
+ if lora_config.bias_enabled:
1180
+ self.bias_stacked = torch.zeros(
1181
+ (
1182
+ max_loras,
1183
+ 1,
1184
+ self.output_size,
1185
+ ),
1186
+ dtype=lora_config.lora_dtype,
1187
+ device=self.device,
1188
+ )
1189
+ else:
1190
+ self.bias_stacked = None
1191
+ # Lazily initialized
1192
+ self.indices: torch.Tensor
1193
+ self.indices_len: List[int]
1194
+
1195
def reset_lora(self, index: int):
1196
self.lora_a_stacked[index] = 0
1197
self.lora_b_stacked[index] = 0
1198
+ if self.lora_config.bias_enabled:
1199
+ self.bias_stacked[index] = 0
1200
1201
def slice_lora_a(self, lora_a: torch.Tensor) -> torch.Tensor:
1202
tensor_model_parallel_rank = get_tensor_model_parallel_rank()
@@ -934,18 +1209,24 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
1209
def slice_lora_b(self, lora_b: torch.Tensor) -> torch.Tensor:
1210
return lora_b
1211
1212
+ def slice_bias(self, bias: torch.Tensor) -> torch.Tensor:
1213
+ return bias
1214
+
1215
def set_lora(
1216
self,
1217
index: int,
1218
lora_a: torch.Tensor,
1219
lora_b: torch.Tensor,
1220
embeddings_tensor: Optional[torch.Tensor],
1221
+ bias: Optional[torch.Tensor] = None,
1222
):
1223
self.reset_lora(index)
1224
1225
if self.base_layer.tp_size > 1:
1226
lora_a = self.slice_lora_a(lora_a)
1227
lora_b = self.slice_lora_b(lora_b)
1228
+ if bias is not None:
1229
+ bias = self.slice_bias(bias)
1230
1231
self.lora_a_stacked[index,
1232
0, :lora_a.shape[1], :lora_a.shape[0]].copy_(
@@ -953,9 +1234,20 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
1234
self.lora_b_stacked[index,
1235
0, :lora_b.shape[1], :lora_b.shape[0]].copy_(
1236
lora_b.T, non_blocking=True)
1237
+ if bias is not None:
1238
+ self.bias_stacked[index,
1239
+ 0, :bias.shape[0]].copy_(bias.T,
1240
+ non_blocking=True)
1241
1242
def apply(self, x: torch.Tensor) -> torch.Tensor:
1243
output = self.base_layer.quant_method.apply(self.base_layer, x)
1244
+ if self.bias_stacked is not None:
1245
+ self.indices = self.punica_wrapper.token_lora_indices
1246
+ output = apply_bias(
1247
+ self.indices,
1248
+ output,
1249
+ self.bias_stacked,
1250
+ )
1251
self.punica_wrapper.add_lora(output, x, self.lora_a_stacked,
1252
self.lora_b_stacked, 1.0)
1253
return output
@@ -1132,6 +1424,7 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
1424
lora_a: torch.Tensor,
1425
lora_b: torch.Tensor,
1426
embeddings_tensor: Optional[torch.Tensor],
1427
+ bias: Optional[torch.Tensor] = None,
1428
):
1429
self.reset_lora(index)
1430
self.lora_a_stacked[index,
@@ -1199,7 +1492,7 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
1492
neginf=float("-inf")))
1493
logits[:,
1494
self.base_layer.org_vocab_size:self.base_layer.org_vocab_size +
1202
- lora_logits.shape[1], ] = lora_logits
1495
+ lora_logits.shape[1]] = lora_logits
1496
1497
# LogitsProcessorWithLoRA always using bgmv
1498
self.punica_wrapper.add_lora_logits(logits, hidden_states,
@@ -1276,6 +1569,7 @@ class LinearScalingRotaryEmbeddingWithLora(BaseLayerWithLoRA):
1569
lora_a: torch.Tensor,
1570
lora_b: torch.Tensor,
1571
embeddings_tensor: Optional[torch.Tensor],
1572
+ bias: Optional[torch.Tensor] = None,
1573
):
1574
...
1575
vllm/lora/lora.py
+16
-1
@@ -17,6 +17,7 @@ class LoRALayerWeights:
17
lora_alpha: int,
18
lora_a: torch.Tensor,
19
lora_b: torch.Tensor,
20
+ bias: Optional[torch.Tensor] = None,
21
embeddings_tensor: Optional[torch.Tensor] = None,
22
scaling: Optional[float] = None,
23
) -> None:
@@ -25,6 +26,7 @@ class LoRALayerWeights:
26
self.lora_alpha = lora_alpha
27
self.lora_a = lora_a
28
self.lora_b = lora_b
29
+ self.bias = bias
30
self.embeddings_tensor = embeddings_tensor
31
32
if scaling is None:
@@ -66,7 +68,8 @@ class LoRALayerWeights:
68
rank: int,
69
dtype: torch.dtype,
70
device: torch.types.Device,
69
- embeddings_tensor_dim: Optional[int] = None) -> "LoRALayerWeights":
71
+ embeddings_tensor_dim: Optional[int] = None,
72
+ bias_enabled: Optional[bool] = False) -> "LoRALayerWeights":
73
pin_memory = str(device) == "cpu" and is_pin_memory_available()
74
lora_a = torch.zeros([input_dim, rank],
75
dtype=dtype,
@@ -76,6 +79,14 @@ class LoRALayerWeights:
79
dtype=dtype,
80
device=device,
81
pin_memory=pin_memory)
82
+ if bias_enabled:
83
+ bias = torch.zeros([output_dim],
84
+ dtype=dtype,
85
+ device=device,
86
+ pin_memory=pin_memory)
87
+ else:
88
+ bias = None
89
+
90
embeddings_tensor = torch.rand(
91
10,
92
embeddings_tensor_dim,
@@ -88,6 +99,7 @@ class LoRALayerWeights:
99
lora_alpha=1,
100
lora_a=lora_a,
101
lora_b=lora_b,
102
+ bias=bias,
103
embeddings_tensor=embeddings_tensor,
104
)
105
@@ -102,6 +114,7 @@ class PackedLoRALayerWeights(LoRALayerWeights):
114
lora_alphas: List[Optional[int]],
115
lora_a: List[Optional[torch.Tensor]],
116
lora_b: List[Optional[torch.Tensor]],
117
+ bias: Optional[List[Optional[torch.Tensor]]] = None,
118
scaling: Optional[List[float]] = None,
119
) -> None:
120
super().__init__(
@@ -110,6 +123,7 @@ class PackedLoRALayerWeights(LoRALayerWeights):
123
lora_alpha=0,
124
lora_a=lora_a,
125
lora_b=lora_b,
126
+ bias=bias,
127
scaling=scaling, # type: ignore
128
embeddings_tensor=None,
129
)
@@ -141,6 +155,7 @@ class PackedLoRALayerWeights(LoRALayerWeights):
155
[lora.lora_alpha if lora is not None else None for lora in loras],
156
[lora.lora_a if lora is not None else None for lora in loras],
157
[lora.lora_b if lora is not None else None for lora in loras],
158
+ [lora.bias if lora is not None else None for lora in loras],
159
scaling=[
160
1 if lora is not None else None # type: ignore
161
for lora in loras
vllm/lora/models.py
+30
-6
@@ -4,7 +4,7 @@ import math
4
import os
5
import re
6
from dataclasses import dataclass, field
7
-from typing import Any, Callable, Dict, List, Optional, Type
7
+from typing import Any, Callable, Dict, List, Optional, Sequence, Type
8
9
import safetensors.torch
10
import torch
@@ -119,7 +119,8 @@ class LoRAModel(AdapterModel):
119
pin_memory = str(device) == "cpu" and is_pin_memory_available()
120
loras: Dict[str, LoRALayerWeights] = {}
121
for tensor_name, tensor in tensors.items():
122
- module_name, is_lora_a = parse_fine_tuned_lora_name(tensor_name)
122
+ module_name, is_lora_a, is_bias = parse_fine_tuned_lora_name(
123
+ tensor_name)
124
if module_name not in loras:
125
lora_embeddings_tensor = None
126
if embeddings:
@@ -136,8 +137,16 @@ class LoRAModel(AdapterModel):
137
lora_embeddings_tensor.pin_memory())
138
loras[module_name] = LoRALayerWeights(module_name, rank,
139
lora_alpha, None, None,
140
+ None,
141
lora_embeddings_tensor)
140
- if is_lora_a:
142
+ if is_bias:
143
+ loras[module_name].bias = tensor.to(device=device,
144
+ dtype=dtype).t()
145
+ bias = tensor.to(device=device, dtype=dtype).t()
146
+ if pin_memory:
147
+ bias = bias.pin_memory()
148
+ loras[module_name].bias = bias
149
+ elif is_lora_a:
150
loras[module_name].lora_a = tensor.to(device=device,
151
dtype=dtype).t()
152
if pin_memory:
@@ -215,7 +224,7 @@ class LoRAModel(AdapterModel):
224
with safetensors.safe_open(lora_tensor_path,
225
framework="pt") as f: # type: ignore
226
for lora_module in f.keys(): # noqa
218
- module_name, _ = parse_fine_tuned_lora_name(lora_module)
227
+ module_name, _, _ = parse_fine_tuned_lora_name(lora_module)
228
part_name = module_name.split(".")[-1]
229
if part_name not in expected_lora_modules:
230
unexpected_modules.append(module_name)
@@ -386,8 +395,19 @@ class LoRAModelManager(AdapterModelManager):
395
module_lora = lora_model.get_lora(module_name)
396
if module_lora:
397
module_lora.optimize()
398
+ # Bias is not explicitly enabled with the flag enable_lora_bias.
399
+ bias = module_lora.bias
400
+ if ((torch.is_tensor(bias) or
401
+ (isinstance(bias, Sequence) and any(b is not None
402
+ for b in bias)))
403
+ and not self.lora_config.bias_enabled):
404
+ module_lora.bias = None
405
+ raise ValueError(
406
+ f"Adapter bias cannot be used for {module_name}"
407
+ " without --enable-lora-bias.")
408
module.set_lora(index, module_lora.lora_a, module_lora.lora_b,
390
- module_lora.embeddings_tensor)
409
+ module_lora.embeddings_tensor,
410
+ module_lora.bias)
411
else:
412
module.reset_lora(index)
413
return True
@@ -509,6 +529,7 @@ class LoRAModelManager(AdapterModelManager):
529
"""Create zero-initialized LoRAModel for warmup."""
530
model = LoRAModel(lora_id, rank, {}, scaling_factor)
531
for module_name, module in self.model.named_modules():
532
+ bias_enabled = self.lora_config.bias_enabled
533
if (not self._match_target_modules(module_name)
534
or not isinstance(module, BaseLayerWithLoRA)
535
or isinstance(module, LinearScalingRotaryEmbeddingWithLora)
@@ -536,7 +557,8 @@ class LoRAModelManager(AdapterModelManager):
557
rank,
558
module.lora_a_stacked.dtype,
559
"cpu",
539
- embeddings_tensor_dim=embeddings_tensor_dim)
560
+ embeddings_tensor_dim=embeddings_tensor_dim,
561
+ bias_enabled=bias_enabled)
562
else:
563
lora = LoRALayerWeights.create_dummy_lora_weights(
564
module_name,
@@ -545,6 +567,7 @@ class LoRAModelManager(AdapterModelManager):
567
rank,
568
module.lora_a_stacked.dtype,
569
"cpu",
570
+ bias_enabled=bias_enabled,
571
)
572
lora.optimize()
573
else:
@@ -559,6 +582,7 @@ class LoRAModelManager(AdapterModelManager):
582
rank,
583
module.lora_a_stacked[i].dtype,
584
"cpu",
585
+ bias_enabled=bias_enabled,
586
)
587
lora.optimize()
588
subloras.append(lora)
vllm/lora/utils.py
+10
-7
@@ -91,7 +91,7 @@ def replace_submodule(model: nn.Module, module_name: str,
91
return new_module
92
93
94
-def parse_fine_tuned_lora_name(name: str) -> Tuple[str, bool]:
94
+def parse_fine_tuned_lora_name(name: str) -> Tuple[str, bool, bool]:
95
"""Parse the name of lora weights.
96
97
args:
@@ -101,15 +101,18 @@ def parse_fine_tuned_lora_name(name: str) -> Tuple[str, bool]:
101
Tuple(module_name, is_lora_a):
102
module_name: the name of the module, e.g. model.dense1,
103
is_lora_a whether the tensor is lora_a or lora_b.
104
+ is_bias whether the tensor is lora bias.
105
"""
106
parts = name.split(".")
107
+ if parts[-1] == "weight" and (parts[-2] == "lora_A"
108
+ or parts[-2] == "lora_B"):
109
+ return ".".join(parts[2:-2]), parts[-2] == "lora_A", False
110
107
- if len(parts) >= 2 and parts[0] == "base_model" and parts[1] == "model":
108
- if parts[-1] == "weight":
109
- if parts[-2] == "lora_A" or parts[-2] == "lora_B":
110
- return ".".join(parts[2:-2]), parts[-2] == "lora_A"
111
- elif parts[-1] == "lora_embedding_A" or parts[-1] == "lora_embedding_B":
112
- return ".".join(parts[2:-1]), parts[-1] == "lora_embedding_A"
111
+ if parts[-1] == "lora_embedding_A" or parts[-1] == "lora_embedding_B":
112
+ return ".".join(parts[2:-1]), parts[-1] == "lora_embedding_A", False
113
+
114
+ if parts[-1] == "bias":
115
+ return ".".join(parts[2:-2]), False, True
116
117
raise ValueError(f"{name} is unsupported LoRA weight")
118