@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8a06428c70
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[LoRA] Adds support for bias in LoRA (#5733)

Signed-off-by: Umesh Deshpande <udeshpa@us.ibm.com> Co-authored-by: Umesh Deshpande <udeshpa@us.ibm.com>

Umesh committed Nov 12, 2024 at 11:08 UTC 8a06428c70657b3310a317b3caf3c562b0e042ae
10 files changed +456 -20
tests/lora/conftest.py
+5
@@ -152,6 +152,11 @@ def sql_lora_files(sql_lora_huggingface_id):
152 return snapshot_download(repo_id=sql_lora_huggingface_id)
153
154
155 +@pytest.fixture(scope="session")
156 +def lora_bias_files():
157 + return snapshot_download(repo_id="followumesh/granite-3b-lora8-bias")
158 +
159 +
160 @pytest.fixture(scope="session")
161 def mixtral_lora_files():
162 # Note: this module has incorrect adapter_config.json to test
tests/lora/test_lora_bias_e2e.py new
+52
@@ -0,0 +1,52 @@
1 +from typing import List
2 +
3 +import pytest
4 +
5 +import vllm
6 +from vllm.lora.request import LoRARequest
7 +
8 +MODEL_PATH = "ibm-granite/granite-3b-code-base"
9 +
10 +
11 +def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:
12 + prompts = [
13 + "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE candidate (people_id VARCHAR, unsure_rate INTEGER); CREATE TABLE people (sex VARCHAR, people_id VARCHAR)\n\n question: which gender got the highest average uncertain ratio. [/user] [assistant]", # noqa: E501
14 + "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_28138035_4 (womens_doubles VARCHAR, mens_singles VARCHAR)\n\n question: Name the women's doubles for werner schlager [/user] [assistant]" # noqa: E501
15 + ]
16 + sampling_params = vllm.SamplingParams(temperature=0,
17 + max_tokens=256,
18 + stop=["[/assistant]"])
19 + outputs = llm.generate(
20 + prompts,
21 + sampling_params,
22 + lora_request=LoRARequest(str(lora_id), lora_id, lora_path)
23 + if lora_id else None)
24 + generated_texts: List[str] = []
25 + for output in outputs:
26 + generated_text = output.outputs[0].text
27 + generated_texts.append(generated_text)
28 + return generated_texts
29 +
30 +
31 +@pytest.mark.parametrize("lora_bias", [True])
32 +@pytest.mark.parametrize("fully_sharded", [True, False])
33 +def test_lora_bias(lora_bias_files: str, lora_bias: bool, fully_sharded: bool):
34 + llm = vllm.LLM(MODEL_PATH,
35 + enable_lora=True,
36 + max_num_seqs=16,
37 + max_lora_rank=8,
38 + max_loras=1,
39 + enable_lora_bias=lora_bias,
40 + tensor_parallel_size=1,
41 + fully_sharded_loras=fully_sharded)
42 +
43 + print("lora adapter created")
44 + output1 = do_sample(llm, lora_bias_files, lora_id=0)
45 +
46 + print("lora")
47 + output2 = do_sample(llm, lora_bias_files, lora_id=1)
48 +
49 + if lora_bias:
50 + assert output1 != output2
51 + else:
52 + assert output1 == output2
tests/lora/test_utils.py
+9 -5
@@ -12,36 +12,40 @@ from vllm.utils import LRUCache
12
13 def test_parse_fine_tuned_lora_name_valid():
14 fixture = {
15 - ("base_model.model.lm_head.lora_A.weight", "lm_head", True),
16 - ("base_model.model.lm_head.lora_B.weight", "lm_head", False),
15 + ("base_model.model.lm_head.lora_A.weight", "lm_head", True, False),
16 + ("base_model.model.lm_head.lora_B.weight", "lm_head", False, False),
17 (
18 "base_model.model.model.embed_tokens.lora_embedding_A",
19 "model.embed_tokens",
20 True,
21 + False,
22 ),
23 (
24 "base_model.model.model.embed_tokens.lora_embedding_B",
25 "model.embed_tokens",
26 False,
27 + False,
28 ),
29 (
30 "base_model.model.model.layers.9.mlp.down_proj.lora_A.weight",
31 "model.layers.9.mlp.down_proj",
32 True,
33 + False,
34 ),
35 (
36 "base_model.model.model.layers.9.mlp.down_proj.lora_B.weight",
37 "model.layers.9.mlp.down_proj",
38 False,
39 + False,
40 ),
41 }
38 - for name, module_name, is_lora_a in fixture:
39 - assert (module_name, is_lora_a) == parse_fine_tuned_lora_name(name)
42 + for name, module_name, is_lora_a, is_bias in fixture:
43 + assert (module_name, is_lora_a,
44 + is_bias) == parse_fine_tuned_lora_name(name)
45
46
47 def test_parse_fine_tuned_lora_name_invalid():
48 fixture = {
44 - "weight",
49 "base_model.weight",
50 "base_model.model.weight",
51 }
vllm/config.py
+1
@@ -1687,6 +1687,7 @@ class LoRAConfig:
1687 # This is a constant.
1688 lora_vocab_padding_size: ClassVar[int] = 256
1689 long_lora_scaling_factors: Optional[Tuple[float]] = None
1690 + bias_enabled: bool = False
1691
1692 def __post_init__(self):
1693 # Setting the maximum rank to 256 should be able to satisfy the vast
vllm/engine/arg_utils.py
+5
@@ -143,6 +143,7 @@ class EngineArgs:
143 limit_mm_per_prompt: Optional[Mapping[str, int]] = None
144 mm_processor_kwargs: Optional[Dict[str, Any]] = None
145 enable_lora: bool = False
146 + enable_lora_bias: bool = False
147 max_loras: int = 1
148 max_lora_rank: int = 16
149 enable_prompt_adapter: bool = False
@@ -584,6 +585,9 @@ class EngineArgs:
585 parser.add_argument('--enable-lora',
586 action='store_true',
587 help='If True, enable handling of LoRA adapters.')
588 + parser.add_argument('--enable-lora-bias',
589 + action='store_true',
590 + help='If True, enable bias for LoRA adapters.')
591 parser.add_argument('--max-loras',
592 type=int,
593 default=EngineArgs.max_loras,
@@ -1148,6 +1152,7 @@ class EngineArgs:
1152 and parallel_config.use_ray),
1153 policy=self.scheduling_policy)
1154 lora_config = LoRAConfig(
1155 + bias_enabled=self.enable_lora_bias,
1156 max_lora_rank=self.max_lora_rank,
1157 max_loras=self.max_loras,
1158 fully_sharded_loras=self.fully_sharded_loras,
vllm/lora/fully_sharded_layers.py
+33
@@ -70,6 +70,14 @@ class ColumnParallelLinearWithShardedLoRA(ColumnParallelLinearWithLoRA):
70 self.lora_b_stacked,
71 add_input=True)
72 # now have column partitioned output
73 +
74 + if self.bias_stacked is not None:
75 + self.bias_stacked = self.bias_stacked.view(
76 + -1, self.bias_stacked.shape[-1])
77 + self.bias_stacked = self.bias_stacked[
78 + self.punica_wrapper.token_lora_indices]
79 + output += self.bias_stacked
80 +
81 output = output.view(*out_orig_shape)
82 return output
83
@@ -121,6 +129,15 @@ def _mcp_apply(x, bias, layer: QKVParallelLinearWithLora):
129 left_offset = 0
130 for idx in range(n):
131 shard_size = layer.lora_b_stacked[idx].shape[2]
132 +
133 + if layer.bias_stacked is not None:
134 + bias = layer.bias_stacked[idx]
135 + if bias is not None:
136 + bias = bias.view(-1, bias.shape[-1])
137 + bias = bias[layer.punica_wrapper.token_lora_indices]
138 + bias[layer.punica_wrapper.token_lora_indices == -1] = 0
139 + output[:, left_offset:left_offset + shard_size] += bias
140 +
141 layer.punica_wrapper.add_expand_slice(
142 output,
143 buffers[idx],
@@ -295,6 +312,15 @@ class RowParallelLinearWithShardedLoRA(RowParallelLinearWithLoRA):
312 lora_b = lora_b[:, start_idx:end_idx]
313 return lora_b
314
315 + def slice_bias(self, bias: torch.Tensor) -> torch.Tensor:
316 + if bias is None:
317 + return bias
318 + shard_size = self.bias_stacked.shape[2]
319 + start_idx = self.tp_rank * shard_size
320 + end_idx = (self.tp_rank + 1) * shard_size
321 + bias = bias[start_idx:end_idx]
322 + return bias
323 +
324 def apply(self, x: torch.Tensor) -> torch.Tensor:
325 output = self.base_layer.quant_method.apply(self.base_layer, x)
326
@@ -318,6 +344,13 @@ class RowParallelLinearWithShardedLoRA(RowParallelLinearWithLoRA):
344 # reduced before being used
345 shard_size = self.lora_b_stacked.shape[2]
346 start_idx = self.tp_rank * shard_size
347 +
348 + if self.bias_stacked is not None:
349 + bias = self.bias_stacked.view(-1, self.bias_stacked.shape[-1])
350 + bias = bias[self.punica_wrapper.token_lora_indices]
351 + bias[self.punica_wrapper.token_lora_indices == -1] = 0
352 + output += bias
353 +
354 self.punica_wrapper.add_expand_slice(output, buffer,
355 self.lora_b_stacked, start_idx,
356 shard_size)
vllm/lora/layers.py
+295 -1
@@ -67,6 +67,63 @@ def _not_fully_sharded_can_replace(can_replace):
67 return dec
68
69
70 +def apply_bias(
71 + indices: torch.Tensor,
72 + output: torch.Tensor,
73 + bias_stacked: torch.Tensor,
74 +):
75 + """Applies bias to output
76 +
77 + Input shapes:
78 + bias_stacked: (num_loras, output_dim)
79 + indices: (batch_size)
80 + output: (batch_size, output_dim)
81 + """
82 + org_output = output
83 + output = output.view(-1, output.shape[-1])
84 + indices = indices.view(-1)
85 +
86 + bias_stacked = bias_stacked.view(-1, bias_stacked.shape[-1])
87 + bias_stacked = bias_stacked[indices]
88 + bias_stacked[indices == -1] = 0
89 + output += bias_stacked
90 +
91 + return output.view_as(org_output)
92 +
93 +
94 +def apply_bias_packed_nslice(
95 + indices: torch.Tensor,
96 + output: torch.Tensor,
97 + output_slices: Tuple[int, ...],
98 + bias_stacked: Tuple[torch.Tensor, torch.Tensor, torch.Tensor],
99 +):
100 + """Applies bias to output
101 +
102 + Input shapes:
103 + bias_stacked: 3 element tuple of (num_loras, output_dim)
104 + indices: (batch_size)
105 + output: (batch_size, q_slice_size + 2*kv_slice_size)
106 + output_slices: n-1 element tuple of (slice_size...),
107 + where n is number of slices
108 + """
109 + org_output = output
110 + output = output.view(-1, output.shape[-1])
111 + indices = indices.view(-1)
112 +
113 + offset_left = 0
114 + for slice_idx, slice in enumerate(output_slices):
115 + bias = bias_stacked[slice_idx]
116 + if bias is not None:
117 + bias = bias.view(-1, bias.shape[-1])
118 + bias = bias[indices]
119 + bias[indices == -1] = 0
120 + output[:, offset_left:offset_left + slice] += bias
121 +
122 + offset_left += slice
123 +
124 + return output.view_as(org_output)
125 +
126 +
127 @dataclass
128 class LoRAMapping(AdapterMapping):
129 is_prefill: bool = False
@@ -105,6 +162,7 @@ class BaseLayerWithLoRA(nn.Module):
162 lora_a: torch.Tensor,
163 lora_b: torch.Tensor,
164 embeddings_tensor: Optional[torch.Tensor],
165 + bias: Optional[torch.Tensor] = None,
166 ):
167 """Overwrites lora tensors at index."""
168 ...
@@ -203,6 +261,7 @@ class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA):
261 lora_a: torch.Tensor,
262 lora_b: torch.Tensor,
263 embeddings_tensor: Optional[torch.Tensor],
264 + bias: Optional[torch.Tensor] = None,
265 ):
266 self.reset_lora(index)
267 self.lora_a_stacked[index, :lora_a.shape[0], :lora_a.shape[1]].copy_(
@@ -299,10 +358,22 @@ class ReplicatedLinearWithLoRA(BaseLayerWithLoRA):
358 dtype=lora_config.lora_dtype,
359 device=self.device,
360 )
361 + if lora_config.bias_enabled:
362 + self.bias_stacked = torch.zeros(
363 + max_loras,
364 + 1,
365 + self.output_size,
366 + dtype=lora_config.lora_dtype,
367 + device=self.device,
368 + )
369 + else:
370 + self.bias_stacked = None
371
372 def reset_lora(self, index: int):
373 self.lora_a_stacked[index] = 0
374 self.lora_b_stacked[index] = 0
375 + if self.lora_config.bias_enabled:
376 + self.bias_stacked[index] = 0
377
378 def set_lora(
379 self,
@@ -310,6 +381,7 @@ class ReplicatedLinearWithLoRA(BaseLayerWithLoRA):
381 lora_a: torch.Tensor,
382 lora_b: torch.Tensor,
383 embeddings_tensor: Optional[torch.Tensor],
384 + bias: Optional[torch.Tensor] = None,
385 ):
386 self.reset_lora(index)
387
@@ -319,10 +391,21 @@ class ReplicatedLinearWithLoRA(BaseLayerWithLoRA):
391 self.lora_b_stacked[index,
392 0, :lora_b.shape[1], :lora_b.shape[0]].copy_(
393 lora_b.T, non_blocking=True)
394 + if bias is not None:
395 + self.bias_stacked[index,
396 + 0, :bias.shape[0]].copy_(bias.T,
397 + non_blocking=True)
398
399 def apply(self, x: torch.Tensor,
400 bias: Optional[torch.Tensor]) -> torch.Tensor:
401 output = self.base_layer.quant_method.apply(self.base_layer, x, bias)
402 + if self.bias_stacked is not None:
403 + self.indices = self.punica_wrapper.token_lora_indices
404 + output = apply_bias(
405 + self.indices,
406 + output,
407 + self.bias_stacked,
408 + )
409 self.punica_wrapper.add_lora(output, x, self.lora_a_stacked,
410 self.lora_b_stacked, 1.0)
411 return output
@@ -401,11 +484,25 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
484 dtype=lora_config.lora_dtype,
485 device=self.device,
486 )
487 +
488 + if lora_config.bias_enabled:
489 + self.bias_stacked = torch.zeros(
490 + max_loras,
491 + 1,
492 + self.output_size,
493 + dtype=lora_config.lora_dtype,
494 + device=self.device,
495 + )
496 + else:
497 + self.bias_stacked = None
498 +
499 self.output_dim = self.lora_b_stacked.shape[2]
500
501 def reset_lora(self, index: int):
502 self.lora_a_stacked[index] = 0
503 self.lora_b_stacked[index] = 0
504 + if self.lora_config.bias_enabled:
505 + self.bias_stacked[index] = 0
506
507 def slice_lora_a(self, lora_a: torch.Tensor) -> torch.Tensor:
508 return lora_a
@@ -418,18 +515,30 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
515 lora_b = lora_b[:, start_idx:end_idx]
516 return lora_b
517
518 + def slice_bias(self, bias: torch.Tensor) -> torch.Tensor:
519 + if bias is None:
520 + return bias
521 + tensor_model_parallel_rank = get_tensor_model_parallel_rank()
522 + shard_size = self.output_dim
523 + start_idx = tensor_model_parallel_rank * shard_size
524 + end_idx = (tensor_model_parallel_rank + 1) * shard_size
525 + bias = bias[start_idx:end_idx]
526 + return bias
527 +
528 def set_lora(
529 self,
530 index: int,
531 lora_a: torch.Tensor,
532 lora_b: torch.Tensor,
533 embeddings_tensor: Optional[torch.Tensor],
534 + bias: Optional[torch.Tensor] = None,
535 ):
536 self.reset_lora(index)
537
538 if self.tp_size > 1:
539 lora_a = self.slice_lora_a(lora_a)
540 lora_b = self.slice_lora_b(lora_b)
541 + bias = self.slice_bias(bias)
542
543 self.lora_a_stacked[index,
544 0, :lora_a.shape[1], :lora_a.shape[0]].copy_(
@@ -437,10 +546,21 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
546 self.lora_b_stacked[index,
547 0, :lora_b.shape[1], :lora_b.shape[0]].copy_(
548 lora_b.T, non_blocking=True)
549 + if bias is not None:
550 + self.bias_stacked[index,
551 + 0, :bias.shape[0]].copy_(bias.T,
552 + non_blocking=True)
553
554 def apply(self, x: torch.Tensor,
555 bias: Optional[torch.Tensor]) -> torch.Tensor:
556 output = self.base_layer.quant_method.apply(self.base_layer, x, bias)
557 + if self.bias_stacked is not None:
558 + self.indices = self.punica_wrapper.token_lora_indices
559 + output = apply_bias(
560 + self.indices,
561 + output,
562 + self.bias_stacked,
563 + )
564 self.punica_wrapper.add_lora(output, x, self.lora_a_stacked,
565 self.lora_b_stacked, 1.0)
566 return output
@@ -534,6 +654,17 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
654 dtype=lora_config.lora_dtype,
655 device=self.device,
656 ) for _ in range(n_slices))
657 + if lora_config.bias_enabled:
658 + self.bias_stacked = tuple(
659 + torch.zeros(
660 + max_loras,
661 + 1,
662 + self.output_size // 2,
663 + dtype=lora_config.lora_dtype,
664 + device=self.device,
665 + ) for _ in range(n_slices))
666 + else:
667 + self.bias_stacked = None
668
669 self.output_dim = self.lora_b_stacked[0].shape[2]
670
@@ -542,6 +673,9 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
673 self.lora_a_stacked[1][index] = 0
674 self.lora_b_stacked[0][index] = 0
675 self.lora_b_stacked[1][index] = 0
676 + if self.lora_config.bias_enabled:
677 + self.bias_stacked[0][index] = 0
678 + self.bias_stacked[1][index] = 0
679
680 def slice_lora_a(
681 self, lora_a: List[Union[torch.Tensor, None]]
@@ -562,18 +696,32 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
696 ]
697 return lora_b
698
699 + def slice_bias(
700 + self, bias: List[Union[torch.Tensor,
701 + None]]) -> List[Union[torch.Tensor, None]]:
702 + if bias[0] is None or bias[1] is None:
703 + return bias
704 + shard_size = self.output_dim
705 + start_idx = self.tp_rank * shard_size
706 + end_idx = (self.tp_rank + 1) * shard_size
707 + bias = [bias[0][start_idx:end_idx], bias[1][start_idx:end_idx]]
708 + return bias
709 +
710 def set_lora(
711 self,
712 index: int,
713 lora_a: torch.Tensor,
714 lora_b: torch.Tensor,
715 embeddings_tensor: Optional[torch.Tensor],
716 + bias: Optional[torch.Tensor] = None,
717 ):
718 self.reset_lora(index)
719
720 if self.tp_size > 1:
721 lora_a = self.slice_lora_a(lora_a)
722 lora_b = self.slice_lora_b(lora_b)
723 + if bias is not None:
724 + bias = self.slice_bias(bias)
725
726 if lora_a[0] is not None:
727 self.lora_a_stacked[0][
@@ -582,6 +730,10 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
730 self.lora_b_stacked[0][
731 index, 0, :lora_b[0].shape[1], :lora_b[0].shape[0]].copy_(
732 lora_b[0].T, non_blocking=True)
733 + if bias is not None and bias[0] is not None:
734 + self.bias_stacked[0][index,
735 + 0, :bias[0].shape[0]].copy_(bias[0].T,
736 + non_blocking=True)
737 if lora_a[1] is not None:
738 self.lora_a_stacked[1][
739 index, 0, :lora_a[1].shape[1], :lora_a[1].shape[0]].copy_(
@@ -589,10 +741,22 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
741 self.lora_b_stacked[1][
742 index, 0, :lora_b[1].shape[1], :lora_b[1].shape[0]].copy_(
743 lora_b[1].T, non_blocking=True)
744 + if bias is not None and bias[1] is not None:
745 + self.bias_stacked[1][index,
746 + 0, :bias[1].shape[0]].copy_(bias[1].T,
747 + non_blocking=True)
748
749 def apply(self, x: torch.Tensor,
750 bias: Optional[torch.Tensor]) -> torch.Tensor:
751 output = self.base_layer.quant_method.apply(self.base_layer, x, bias)
752 + if self.bias_stacked is not None:
753 + self.indices = self.punica_wrapper.token_lora_indices
754 + output = apply_bias_packed_nslice(
755 + self.indices,
756 + output,
757 + (self.output_dim, self.output_dim),
758 + self.bias_stacked,
759 + )
760 self.punica_wrapper.add_lora_packed_nslice(
761 output, x, self.lora_a_stacked, self.lora_b_stacked, 1.0,
762 (self.output_dim, self.output_dim))
@@ -654,17 +818,35 @@ class QKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
818 lora_b = torch.cat([lora_b_q, lora_b_k, lora_b_v], dim=1)
819 return lora_b
820
821 + def slice_bias(self, bias: torch.Tensor) -> torch.Tensor:
822 + bias_q = bias[self.q_proj_shard_size *
823 + self.q_shard_id:self.q_proj_shard_size *
824 + (self.q_shard_id + 1)]
825 + k_offset = self.q_proj_total_size
826 + bias_k = bias[k_offset +
827 + self.kv_proj_shard_size * self.kv_shard_id:k_offset +
828 + self.kv_proj_shard_size * (self.kv_shard_id + 1)]
829 + v_offset = k_offset + self.kv_proj_total_size
830 + bias_v = bias[v_offset +
831 + self.kv_proj_shard_size * self.kv_shard_id:v_offset +
832 + self.kv_proj_shard_size * (self.kv_shard_id + 1)]
833 + bias = torch.cat([bias_q, bias_k, bias_v], dim=1)
834 + return bias
835 +
836 def set_lora(
837 self,
838 index: int,
839 lora_a: torch.Tensor,
840 lora_b: torch.Tensor,
841 embeddings_tensor: Optional[torch.Tensor],
842 + bias: Optional[torch.Tensor] = None,
843 ):
844 self.reset_lora(index)
845 if self.tp_size > 1:
846 lora_a = self.slice_lora_a(lora_a)
847 lora_b = self.slice_lora_b(lora_b)
848 + if bias is not None:
849 + bias = self.slice_bias(bias)
850
851 self.lora_a_stacked[index,
852 0, :lora_a.shape[1], :lora_a.shape[0]].copy_(
@@ -672,6 +854,10 @@ class QKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
854 self.lora_b_stacked[index,
855 0, :lora_b.shape[1], :lora_b.shape[0]].copy_(
856 lora_b.T, non_blocking=True)
857 + if bias is not None:
858 + self.bias_stacked[index,
859 + 0, :bias.shape[0]].copy_(bias.T,
860 + non_blocking=True)
861
862 @classmethod
863 @_not_fully_sharded_can_replace
@@ -768,6 +954,32 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
954 device=self.device,
955 ),
956 )
957 + if lora_config.bias_enabled:
958 + self.bias_stacked = (
959 + torch.zeros(
960 + max_loras,
961 + 1,
962 + self.q_proj_shard_size,
963 + dtype=lora_config.lora_dtype,
964 + device=self.device,
965 + ),
966 + torch.zeros(
967 + max_loras,
968 + 1,
969 + self.kv_proj_shard_size,
970 + dtype=lora_config.lora_dtype,
971 + device=self.device,
972 + ),
973 + torch.zeros(
974 + max_loras,
975 + 1,
976 + self.kv_proj_shard_size,
977 + dtype=lora_config.lora_dtype,
978 + device=self.device,
979 + ),
980 + )
981 + else:
982 + self.bias_stacked = None
983
984 self.output_slices = (
985 self.q_proj_shard_size,
@@ -787,6 +999,10 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
999 self.lora_b_stacked[1][index] = 0
1000 self.lora_a_stacked[2][index] = 0
1001 self.lora_b_stacked[2][index] = 0
1002 + if self.lora_config.bias_enabled:
1003 + self.bias_stacked[0][index] = 0
1004 + self.bias_stacked[1][index] = 0
1005 + self.bias_stacked[2][index] = 0
1006
1007 def slice_lora_a(
1008 self, lora_a: List[Union[torch.Tensor, None]]
@@ -812,18 +1028,40 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
1028 lora_b = [lora_b_q, lora_b_k, lora_b_v]
1029 return lora_b
1030
1031 + def slice_bias(
1032 + self, bias: List[Union[torch.Tensor,
1033 + None]]) -> List[Union[torch.Tensor, None]]:
1034 + bias_q, bias_k, bias_v = bias
1035 + if bias_q is not None:
1036 + bias_q = bias_q[self.q_proj_shard_size *
1037 + self.q_shard_id:self.q_proj_shard_size *
1038 + (self.q_shard_id + 1)]
1039 + if bias_k is not None:
1040 + bias_k = bias_k[self.kv_proj_shard_size *
1041 + self.kv_shard_id:self.kv_proj_shard_size *
1042 + (self.kv_shard_id + 1)]
1043 + if bias_v is not None:
1044 + bias_v = bias_v[self.kv_proj_shard_size *
1045 + self.kv_shard_id:self.kv_proj_shard_size *
1046 + (self.kv_shard_id + 1)]
1047 + bias = [bias_q, bias_k, bias_v]
1048 + return bias
1049 +
1050 def set_lora(
1051 self,
1052 index: int,
1053 lora_a: torch.Tensor,
1054 lora_b: torch.Tensor,
1055 embeddings_tensor: Optional[torch.Tensor],
1056 + bias: Optional[torch.Tensor] = None,
1057 ):
1058 self.reset_lora(index)
1059
1060 if self.tp_size > 1:
1061 lora_a = self.slice_lora_a(lora_a)
1062 lora_b = self.slice_lora_b(lora_b)
1063 + if bias is not None:
1064 + bias = self.slice_bias(bias)
1065
1066 if lora_b[0] is not None:
1067 lora_b_q = lora_b[0]
@@ -854,9 +1092,28 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
1092 index, 0, :lora_a[2].shape[1], :lora_a[2].shape[0]].copy_(
1093 lora_a[2].T, non_blocking=True)
1094
1095 + if bias is not None:
1096 + if bias[0] is not None:
1097 + self.bias_stacked[0][index, 0, :bias[0].shape[0]].copy_(
1098 + bias[0].T, non_blocking=True)
1099 + if bias[1] is not None:
1100 + self.bias_stacked[1][index, 0, :bias[1].shape[0]].copy_(
1101 + bias[1].T, non_blocking=True)
1102 + if bias[2] is not None:
1103 + self.bias_stacked[2][index, 0, :bias[2].shape[0]].copy_(
1104 + bias[2].T, non_blocking=True)
1105 +
1106 def apply(self, x: torch.Tensor,
1107 bias: Optional[torch.Tensor]) -> torch.Tensor:
1108 output = self.base_layer.quant_method.apply(self.base_layer, x, bias)
1109 + if self.bias_stacked is not None:
1110 + self.indices = self.punica_wrapper.token_lora_indices
1111 + output = apply_bias_packed_nslice(
1112 + self.indices,
1113 + output,
1114 + self.output_slices,
1115 + self.bias_stacked,
1116 + )
1117 self.punica_wrapper.add_lora_packed_nslice(output, x,
1118 self.lora_a_stacked,
1119 self.lora_b_stacked, 1.0,
@@ -919,9 +1176,27 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
1176 device=self.device,
1177 )
1178
1179 + if lora_config.bias_enabled:
1180 + self.bias_stacked = torch.zeros(
1181 + (
1182 + max_loras,
1183 + 1,
1184 + self.output_size,
1185 + ),
1186 + dtype=lora_config.lora_dtype,
1187 + device=self.device,
1188 + )
1189 + else:
1190 + self.bias_stacked = None
1191 + # Lazily initialized
1192 + self.indices: torch.Tensor
1193 + self.indices_len: List[int]
1194 +
1195 def reset_lora(self, index: int):
1196 self.lora_a_stacked[index] = 0
1197 self.lora_b_stacked[index] = 0
1198 + if self.lora_config.bias_enabled:
1199 + self.bias_stacked[index] = 0
1200
1201 def slice_lora_a(self, lora_a: torch.Tensor) -> torch.Tensor:
1202 tensor_model_parallel_rank = get_tensor_model_parallel_rank()
@@ -934,18 +1209,24 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
1209 def slice_lora_b(self, lora_b: torch.Tensor) -> torch.Tensor:
1210 return lora_b
1211
1212 + def slice_bias(self, bias: torch.Tensor) -> torch.Tensor:
1213 + return bias
1214 +
1215 def set_lora(
1216 self,
1217 index: int,
1218 lora_a: torch.Tensor,
1219 lora_b: torch.Tensor,
1220 embeddings_tensor: Optional[torch.Tensor],
1221 + bias: Optional[torch.Tensor] = None,
1222 ):
1223 self.reset_lora(index)
1224
1225 if self.base_layer.tp_size > 1:
1226 lora_a = self.slice_lora_a(lora_a)
1227 lora_b = self.slice_lora_b(lora_b)
1228 + if bias is not None:
1229 + bias = self.slice_bias(bias)
1230
1231 self.lora_a_stacked[index,
1232 0, :lora_a.shape[1], :lora_a.shape[0]].copy_(
@@ -953,9 +1234,20 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
1234 self.lora_b_stacked[index,
1235 0, :lora_b.shape[1], :lora_b.shape[0]].copy_(
1236 lora_b.T, non_blocking=True)
1237 + if bias is not None:
1238 + self.bias_stacked[index,
1239 + 0, :bias.shape[0]].copy_(bias.T,
1240 + non_blocking=True)
1241
1242 def apply(self, x: torch.Tensor) -> torch.Tensor:
1243 output = self.base_layer.quant_method.apply(self.base_layer, x)
1244 + if self.bias_stacked is not None:
1245 + self.indices = self.punica_wrapper.token_lora_indices
1246 + output = apply_bias(
1247 + self.indices,
1248 + output,
1249 + self.bias_stacked,
1250 + )
1251 self.punica_wrapper.add_lora(output, x, self.lora_a_stacked,
1252 self.lora_b_stacked, 1.0)
1253 return output
@@ -1132,6 +1424,7 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
1424 lora_a: torch.Tensor,
1425 lora_b: torch.Tensor,
1426 embeddings_tensor: Optional[torch.Tensor],
1427 + bias: Optional[torch.Tensor] = None,
1428 ):
1429 self.reset_lora(index)
1430 self.lora_a_stacked[index,
@@ -1199,7 +1492,7 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
1492 neginf=float("-inf")))
1493 logits[:,
1494 self.base_layer.org_vocab_size:self.base_layer.org_vocab_size +
1202 - lora_logits.shape[1], ] = lora_logits
1495 + lora_logits.shape[1]] = lora_logits
1496
1497 # LogitsProcessorWithLoRA always using bgmv
1498 self.punica_wrapper.add_lora_logits(logits, hidden_states,
@@ -1276,6 +1569,7 @@ class LinearScalingRotaryEmbeddingWithLora(BaseLayerWithLoRA):
1569 lora_a: torch.Tensor,
1570 lora_b: torch.Tensor,
1571 embeddings_tensor: Optional[torch.Tensor],
1572 + bias: Optional[torch.Tensor] = None,
1573 ):
1574 ...
1575
vllm/lora/lora.py
+16 -1
@@ -17,6 +17,7 @@ class LoRALayerWeights:
17 lora_alpha: int,
18 lora_a: torch.Tensor,
19 lora_b: torch.Tensor,
20 + bias: Optional[torch.Tensor] = None,
21 embeddings_tensor: Optional[torch.Tensor] = None,
22 scaling: Optional[float] = None,
23 ) -> None:
@@ -25,6 +26,7 @@ class LoRALayerWeights:
26 self.lora_alpha = lora_alpha
27 self.lora_a = lora_a
28 self.lora_b = lora_b
29 + self.bias = bias
30 self.embeddings_tensor = embeddings_tensor
31
32 if scaling is None:
@@ -66,7 +68,8 @@ class LoRALayerWeights:
68 rank: int,
69 dtype: torch.dtype,
70 device: torch.types.Device,
69 - embeddings_tensor_dim: Optional[int] = None) -> "LoRALayerWeights":
71 + embeddings_tensor_dim: Optional[int] = None,
72 + bias_enabled: Optional[bool] = False) -> "LoRALayerWeights":
73 pin_memory = str(device) == "cpu" and is_pin_memory_available()
74 lora_a = torch.zeros([input_dim, rank],
75 dtype=dtype,
@@ -76,6 +79,14 @@ class LoRALayerWeights:
79 dtype=dtype,
80 device=device,
81 pin_memory=pin_memory)
82 + if bias_enabled:
83 + bias = torch.zeros([output_dim],
84 + dtype=dtype,
85 + device=device,
86 + pin_memory=pin_memory)
87 + else:
88 + bias = None
89 +
90 embeddings_tensor = torch.rand(
91 10,
92 embeddings_tensor_dim,
@@ -88,6 +99,7 @@ class LoRALayerWeights:
99 lora_alpha=1,
100 lora_a=lora_a,
101 lora_b=lora_b,
102 + bias=bias,
103 embeddings_tensor=embeddings_tensor,
104 )
105
@@ -102,6 +114,7 @@ class PackedLoRALayerWeights(LoRALayerWeights):
114 lora_alphas: List[Optional[int]],
115 lora_a: List[Optional[torch.Tensor]],
116 lora_b: List[Optional[torch.Tensor]],
117 + bias: Optional[List[Optional[torch.Tensor]]] = None,
118 scaling: Optional[List[float]] = None,
119 ) -> None:
120 super().__init__(
@@ -110,6 +123,7 @@ class PackedLoRALayerWeights(LoRALayerWeights):
123 lora_alpha=0,
124 lora_a=lora_a,
125 lora_b=lora_b,
126 + bias=bias,
127 scaling=scaling, # type: ignore
128 embeddings_tensor=None,
129 )
@@ -141,6 +155,7 @@ class PackedLoRALayerWeights(LoRALayerWeights):
155 [lora.lora_alpha if lora is not None else None for lora in loras],
156 [lora.lora_a if lora is not None else None for lora in loras],
157 [lora.lora_b if lora is not None else None for lora in loras],
158 + [lora.bias if lora is not None else None for lora in loras],
159 scaling=[
160 1 if lora is not None else None # type: ignore
161 for lora in loras
vllm/lora/models.py
+30 -6
@@ -4,7 +4,7 @@ import math
4 import os
5 import re
6 from dataclasses import dataclass, field
7 -from typing import Any, Callable, Dict, List, Optional, Type
7 +from typing import Any, Callable, Dict, List, Optional, Sequence, Type
8
9 import safetensors.torch
10 import torch
@@ -119,7 +119,8 @@ class LoRAModel(AdapterModel):
119 pin_memory = str(device) == "cpu" and is_pin_memory_available()
120 loras: Dict[str, LoRALayerWeights] = {}
121 for tensor_name, tensor in tensors.items():
122 - module_name, is_lora_a = parse_fine_tuned_lora_name(tensor_name)
122 + module_name, is_lora_a, is_bias = parse_fine_tuned_lora_name(
123 + tensor_name)
124 if module_name not in loras:
125 lora_embeddings_tensor = None
126 if embeddings:
@@ -136,8 +137,16 @@ class LoRAModel(AdapterModel):
137 lora_embeddings_tensor.pin_memory())
138 loras[module_name] = LoRALayerWeights(module_name, rank,
139 lora_alpha, None, None,
140 + None,
141 lora_embeddings_tensor)
140 - if is_lora_a:
142 + if is_bias:
143 + loras[module_name].bias = tensor.to(device=device,
144 + dtype=dtype).t()
145 + bias = tensor.to(device=device, dtype=dtype).t()
146 + if pin_memory:
147 + bias = bias.pin_memory()
148 + loras[module_name].bias = bias
149 + elif is_lora_a:
150 loras[module_name].lora_a = tensor.to(device=device,
151 dtype=dtype).t()
152 if pin_memory:
@@ -215,7 +224,7 @@ class LoRAModel(AdapterModel):
224 with safetensors.safe_open(lora_tensor_path,
225 framework="pt") as f: # type: ignore
226 for lora_module in f.keys(): # noqa
218 - module_name, _ = parse_fine_tuned_lora_name(lora_module)
227 + module_name, _, _ = parse_fine_tuned_lora_name(lora_module)
228 part_name = module_name.split(".")[-1]
229 if part_name not in expected_lora_modules:
230 unexpected_modules.append(module_name)
@@ -386,8 +395,19 @@ class LoRAModelManager(AdapterModelManager):
395 module_lora = lora_model.get_lora(module_name)
396 if module_lora:
397 module_lora.optimize()
398 + # Bias is not explicitly enabled with the flag enable_lora_bias.
399 + bias = module_lora.bias
400 + if ((torch.is_tensor(bias) or
401 + (isinstance(bias, Sequence) and any(b is not None
402 + for b in bias)))
403 + and not self.lora_config.bias_enabled):
404 + module_lora.bias = None
405 + raise ValueError(
406 + f"Adapter bias cannot be used for {module_name}"
407 + " without --enable-lora-bias.")
408 module.set_lora(index, module_lora.lora_a, module_lora.lora_b,
390 - module_lora.embeddings_tensor)
409 + module_lora.embeddings_tensor,
410 + module_lora.bias)
411 else:
412 module.reset_lora(index)
413 return True
@@ -509,6 +529,7 @@ class LoRAModelManager(AdapterModelManager):
529 """Create zero-initialized LoRAModel for warmup."""
530 model = LoRAModel(lora_id, rank, {}, scaling_factor)
531 for module_name, module in self.model.named_modules():
532 + bias_enabled = self.lora_config.bias_enabled
533 if (not self._match_target_modules(module_name)
534 or not isinstance(module, BaseLayerWithLoRA)
535 or isinstance(module, LinearScalingRotaryEmbeddingWithLora)
@@ -536,7 +557,8 @@ class LoRAModelManager(AdapterModelManager):
557 rank,
558 module.lora_a_stacked.dtype,
559 "cpu",
539 - embeddings_tensor_dim=embeddings_tensor_dim)
560 + embeddings_tensor_dim=embeddings_tensor_dim,
561 + bias_enabled=bias_enabled)
562 else:
563 lora = LoRALayerWeights.create_dummy_lora_weights(
564 module_name,
@@ -545,6 +567,7 @@ class LoRAModelManager(AdapterModelManager):
567 rank,
568 module.lora_a_stacked.dtype,
569 "cpu",
570 + bias_enabled=bias_enabled,
571 )
572 lora.optimize()
573 else:
@@ -559,6 +582,7 @@ class LoRAModelManager(AdapterModelManager):
582 rank,
583 module.lora_a_stacked[i].dtype,
584 "cpu",
585 + bias_enabled=bias_enabled,
586 )
587 lora.optimize()
588 subloras.append(lora)
vllm/lora/utils.py
+10 -7
@@ -91,7 +91,7 @@ def replace_submodule(model: nn.Module, module_name: str,
91 return new_module
92
93
94 -def parse_fine_tuned_lora_name(name: str) -> Tuple[str, bool]:
94 +def parse_fine_tuned_lora_name(name: str) -> Tuple[str, bool, bool]:
95 """Parse the name of lora weights.
96
97 args:
@@ -101,15 +101,18 @@ def parse_fine_tuned_lora_name(name: str) -> Tuple[str, bool]:
101 Tuple(module_name, is_lora_a):
102 module_name: the name of the module, e.g. model.dense1,
103 is_lora_a whether the tensor is lora_a or lora_b.
104 + is_bias whether the tensor is lora bias.
105 """
106 parts = name.split(".")
107 + if parts[-1] == "weight" and (parts[-2] == "lora_A"
108 + or parts[-2] == "lora_B"):
109 + return ".".join(parts[2:-2]), parts[-2] == "lora_A", False
110
107 - if len(parts) >= 2 and parts[0] == "base_model" and parts[1] == "model":
108 - if parts[-1] == "weight":
109 - if parts[-2] == "lora_A" or parts[-2] == "lora_B":
110 - return ".".join(parts[2:-2]), parts[-2] == "lora_A"
111 - elif parts[-1] == "lora_embedding_A" or parts[-1] == "lora_embedding_B":
112 - return ".".join(parts[2:-1]), parts[-1] == "lora_embedding_A"
111 + if parts[-1] == "lora_embedding_A" or parts[-1] == "lora_embedding_B":
112 + return ".".join(parts[2:-1]), parts[-1] == "lora_embedding_A", False
113 +
114 + if parts[-1] == "bias":
115 + return ".".join(parts[2:-2]), False, True
116
117 raise ValueError(f"{name} is unsupported LoRA weight")
118