A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[Misc][LoRA] Refactor and clean MergedQKVParallelLinearWithLora implementation (#10958)
Signed-off-by: Isotr0py <2037008807@qq.com>
Isotr0py committed
Dec 7, 2024 at 18:33 UTC
b26b4cd03c5468c68c3ce328ea6498a5d816870d
1 file changed
+60
-263
vllm/lora/layers.py
+60
-263
@@ -542,10 +542,20 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
542
Both slices must have the same size.
543
"""
544
545
- def __init__(self, base_layer: MergedColumnParallelLinear) -> None:
545
+ def __init__(
546
+ self, base_layer: Union[MergedColumnParallelLinear,
547
+ QKVParallelLinear]) -> None:
548
super().__init__(base_layer)
549
# There are two LoRA layers
548
- self.n_slices = len(self.base_layer.output_sizes)
550
+ self.tp_size = get_tensor_model_parallel_world_size()
551
+ self.tp_rank = get_tensor_model_parallel_rank()
552
+ # the output_sizes in MergedColumnParallelLinear is not sharded by tp
553
+ # we need to divide it by the tp_size to get correct slices size
554
+ output_sizes = self.base_layer.output_sizes
555
+ self.output_slices = tuple(
556
+ divide(output_size, self.tp_size) for output_size in output_sizes)
557
+ self.n_slices = len(self.output_slices)
558
+ self.output_ids = (self.tp_rank, ) * self.n_slices
559
560
def create_lora_weights(
561
self,
@@ -559,15 +569,6 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
569
"""
570
self.lora_config = lora_config
571
562
- if not (len(self.base_layer.output_sizes) == self.n_slices == 2
563
- and self.base_layer.output_sizes[0]
564
- == self.base_layer.output_sizes[1]):
565
- raise ValueError(
566
- "LoRAColumnParallelLinear2Slice requires 2 slices with "
567
- "the same size.")
568
- self.tp_size = get_tensor_model_parallel_world_size()
569
- self.tp_rank = get_tensor_model_parallel_rank()
570
-
572
lora_a_output_size_per_partition = (
573
lora_config.max_lora_rank if not lora_config.fully_sharded_loras
574
else divide(lora_config.max_lora_rank, self.tp_size))
@@ -585,22 +586,20 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
586
torch.zeros(
587
max_loras,
588
1,
588
- self.output_size // 2,
589
+ output_size,
590
lora_config.max_lora_rank,
591
dtype=lora_config.lora_dtype,
592
device=self.device,
592
- ) for _ in range(self.n_slices))
593
+ ) for output_size in self.output_slices)
594
if lora_config.bias_enabled:
595
self.lora_bias_stacked = tuple(
596
torch.zeros(
597
max_loras,
598
1,
598
- self.output_size // 2,
599
+ output_size,
600
dtype=lora_config.lora_dtype,
601
device=self.device,
601
- ) for _ in range(self.n_slices))
602
- self.output_dim = self.lora_b_stacked[0].shape[2]
603
- self.output_slices = (self.output_dim, self.output_dim)
602
+ ) for output_size in self.output_slices)
603
604
def slice_lora_a(
605
self, lora_a: List[Union[torch.Tensor, None]]
@@ -610,27 +609,21 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
609
def slice_lora_b(
610
self, lora_b: List[Union[torch.Tensor, None]]
611
) -> List[Union[torch.Tensor, None]]:
613
- #NOTE: lora_b contains 2 subloras, and each sublora could be None.
614
- shard_size = self.output_dim
615
- start_idx = self.tp_rank * shard_size
616
- end_idx = (self.tp_rank + 1) * shard_size
617
- lora_b = [
618
- lora_b[0][:, start_idx:end_idx] if lora_b[0] is not None else None,
619
- lora_b[1][:, start_idx:end_idx] if lora_b[1] is not None else None,
620
- ]
612
+ for i, (shard_id, shard_size) in enumerate(
613
+ zip(self.output_ids, self.output_slices)):
614
+ if (lora_b_i := lora_b[i]) is not None:
615
+ lora_b[i] = lora_b_i[:, shard_size * shard_id:shard_size *
616
+ (shard_id + 1)]
617
return lora_b
618
619
def slice_bias(
620
self, bias: List[Union[torch.Tensor,
621
None]]) -> List[Union[torch.Tensor, None]]:
626
- # NOTE : each bias could be None.
627
- shard_size = self.output_dim
628
- start_idx = self.tp_rank * shard_size
629
- end_idx = (self.tp_rank + 1) * shard_size
630
- bias = [
631
- bias[0][start_idx:end_idx] if bias[0] is not None else None,
632
- bias[1][start_idx:end_idx] if bias[1] is not None else None
633
- ]
622
+ for i, (shard_id, shard_size) in enumerate(
623
+ zip(self.output_ids, self.output_slices)):
624
+ if (bias_i := bias[i]) is not None:
625
+ bias[i] = bias_i[shard_size * shard_id:shard_size *
626
+ (shard_id + 1)]
627
return bias
628
629
def set_lora(
@@ -649,30 +642,25 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
642
if lora_bias is not None:
643
lora_bias = self.slice_bias(lora_bias)
644
652
- if lora_a[0] is not None:
653
- self.lora_a_stacked[0][
654
- index, 0, :lora_a[0].shape[1], :lora_a[0].shape[0]].copy_(
655
- lora_a[0].T, non_blocking=True)
656
- self.lora_b_stacked[0][
657
- index, 0, :lora_b[0].shape[1], :lora_b[0].shape[0]].copy_(
658
- lora_b[0].T, non_blocking=True)
659
- if lora_bias is not None and lora_bias[0] is not None:
660
- self.lora_bias_stacked = cast(Tuple[torch.Tensor, ...],
661
- self.lora_bias_stacked)
662
- self.lora_bias_stacked[0][index, 0, :lora_bias[0].shape[0]].copy_(
663
- lora_bias[0].T, non_blocking=True)
664
- if lora_a[1] is not None:
665
- self.lora_a_stacked[1][
666
- index, 0, :lora_a[1].shape[1], :lora_a[1].shape[0]].copy_(
667
- lora_a[1].T, non_blocking=True)
668
- self.lora_b_stacked[1][
669
- index, 0, :lora_b[1].shape[1], :lora_b[1].shape[0]].copy_(
670
- lora_b[1].T, non_blocking=True)
671
- if lora_bias is not None and lora_bias[1] is not None:
645
+ for i in range(self.n_slices):
646
+ if (lora_a_i := lora_a[i]) is not None:
647
+ self.lora_a_stacked[i][
648
+ index, 0, :lora_a_i.shape[1], :lora_a_i.shape[0]].copy_(
649
+ lora_a_i.T, non_blocking=True)
650
+ if (lora_b_i := lora_b[i]) is not None:
651
+ self.lora_b_stacked[i][
652
+ index, 0, :lora_b_i.shape[1], :lora_b_i.shape[0]].copy_(
653
+ lora_b_i.T, non_blocking=True)
654
+
655
+ if lora_bias is not None:
656
self.lora_bias_stacked = cast(Tuple[torch.Tensor, ...],
657
self.lora_bias_stacked)
674
- self.lora_bias_stacked[1][index, 0, :lora_bias[1].shape[0]].copy_(
675
- lora_bias[1].T, non_blocking=True)
658
+ for i in range(self.n_slices):
659
+ if (lora_bias_i := lora_bias[i]) is not None:
660
+ self.lora_bias_stacked[i][index,
661
+ 0, :lora_bias_i.shape[0]].copy_(
662
+ lora_bias_i.T,
663
+ non_blocking=True)
664
665
@classmethod
666
@_not_fully_sharded_can_replace
@@ -755,8 +743,8 @@ class QKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
743
packed_modules_list) == 1
744
745
758
-class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
759
- """ColumnParallelLinear layer that is composed of 3 sublayers (slices)
746
+class MergedQKVParallelLinearWithLora(MergedColumnParallelLinearWithLoRA):
747
+ """MergedColumnParallelLinear layer that is composed of 3 sublayers (slices)
748
packed together in qkv proj fashion
749
(q_proj + k_proj + v_proj -> qkv_proj).
750
@@ -773,22 +761,6 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
761
self.tp_size = get_tensor_model_parallel_world_size()
762
self.tp_rank = get_tensor_model_parallel_rank()
763
776
- def create_lora_weights(
777
- self,
778
- max_loras: int,
779
- lora_config: LoRAConfig,
780
- model_config: Optional[PretrainedConfig] = None,
781
- ) -> None:
782
- """
783
- The main reason for overloading this function is to handle inconsistent
784
- weight dimensions in qkv lora.
785
- """
786
- self.lora_config = lora_config
787
-
788
- if not (len(self.base_layer.output_sizes) == self.n_slices == 3):
789
- raise ValueError(
790
- "LoRAColumnParallelLinear3Slice requires 3 slices.")
791
-
764
self.q_proj_shard_size = (self.base_layer.num_heads *
765
self.base_layer.head_size)
766
self.kv_proj_shard_size = (self.base_layer.num_kv_heads *
@@ -796,203 +768,28 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
768
self.q_shard_id = self.tp_rank
769
self.kv_shard_id = self.tp_rank // self.base_layer.num_kv_head_replicas
770
799
- lora_a_output_size_per_partition = (
800
- lora_config.max_lora_rank if not lora_config.fully_sharded_loras
801
- else divide(lora_config.max_lora_rank, self.tp_size))
802
- # q, k, v
803
- self.lora_a_stacked = (
804
- torch.zeros(
805
- max_loras,
806
- 1,
807
- lora_a_output_size_per_partition,
808
- self.input_size,
809
- dtype=lora_config.lora_dtype,
810
- device=self.device,
811
- ),
812
- torch.zeros(
813
- max_loras,
814
- 1,
815
- lora_a_output_size_per_partition,
816
- self.input_size,
817
- dtype=lora_config.lora_dtype,
818
- device=self.device,
819
- ),
820
- torch.zeros(
821
- max_loras,
822
- 1,
823
- lora_a_output_size_per_partition,
824
- self.input_size,
825
- dtype=lora_config.lora_dtype,
826
- device=self.device,
827
- ),
828
- )
829
- self.lora_b_stacked = (
830
- torch.zeros(
831
- max_loras,
832
- 1,
833
- self.q_proj_shard_size,
834
- lora_config.max_lora_rank,
835
- dtype=lora_config.lora_dtype,
836
- device=self.device,
837
- ),
838
- torch.zeros(
839
- max_loras,
840
- 1,
841
- self.kv_proj_shard_size,
842
- lora_config.max_lora_rank,
843
- dtype=lora_config.lora_dtype,
844
- device=self.device,
845
- ),
846
- torch.zeros(
847
- max_loras,
848
- 1,
849
- self.kv_proj_shard_size,
850
- lora_config.max_lora_rank,
851
- dtype=lora_config.lora_dtype,
852
- device=self.device,
853
- ),
854
- )
855
- if lora_config.bias_enabled:
856
- self.lora_bias_stacked = (
857
- torch.zeros(
858
- max_loras,
859
- 1,
860
- self.q_proj_shard_size,
861
- dtype=lora_config.lora_dtype,
862
- device=self.device,
863
- ),
864
- torch.zeros(
865
- max_loras,
866
- 1,
867
- self.kv_proj_shard_size,
868
- dtype=lora_config.lora_dtype,
869
- device=self.device,
870
- ),
871
- torch.zeros(
872
- max_loras,
873
- 1,
874
- self.kv_proj_shard_size,
875
- dtype=lora_config.lora_dtype,
876
- device=self.device,
877
- ),
878
- )
771
self.output_slices = (
772
self.q_proj_shard_size,
773
self.kv_proj_shard_size,
774
self.kv_proj_shard_size,
775
)
884
- self.packed_indices: Optional[torch.Tensor] = None
885
- self.standard_indices: Optional[torch.Tensor] = None
886
- # lazily initialized.
887
- self.indices: torch.Tensor
888
- self.indices_len: List[int]
889
-
890
- def slice_lora_a(
891
- self, lora_a: List[Union[torch.Tensor, None]]
892
- ) -> List[Union[torch.Tensor, None]]:
893
- return lora_a
894
-
895
- def slice_lora_b(
896
- self, lora_b: List[Union[torch.Tensor, None]]
897
- ) -> List[Union[torch.Tensor, None]]:
898
- lora_b_q, lora_b_k, lora_b_v = None, None, None
899
- if lora_b[0] is not None:
900
- lora_b_q = lora_b[0][:, self.q_proj_shard_size *
901
- self.q_shard_id:self.q_proj_shard_size *
902
- (self.q_shard_id + 1), ]
903
- if lora_b[1] is not None:
904
- lora_b_k = lora_b[1][:, self.kv_proj_shard_size *
905
- self.kv_shard_id:self.kv_proj_shard_size *
906
- (self.kv_shard_id + 1), ]
907
- if lora_b[2] is not None:
908
- lora_b_v = lora_b[2][:, self.kv_proj_shard_size *
909
- self.kv_shard_id:self.kv_proj_shard_size *
910
- (self.kv_shard_id + 1), ]
911
- lora_b = [lora_b_q, lora_b_k, lora_b_v]
912
- return lora_b
913
-
914
- def slice_bias(
915
- self, bias: List[Union[torch.Tensor,
916
- None]]) -> List[Union[torch.Tensor, None]]:
917
- bias_q, bias_k, bias_v = bias
918
- if bias_q is not None:
919
- bias_q = bias_q[self.q_proj_shard_size *
920
- self.q_shard_id:self.q_proj_shard_size *
921
- (self.q_shard_id + 1)]
922
- if bias_k is not None:
923
- bias_k = bias_k[self.kv_proj_shard_size *
924
- self.kv_shard_id:self.kv_proj_shard_size *
925
- (self.kv_shard_id + 1)]
926
- if bias_v is not None:
927
- bias_v = bias_v[self.kv_proj_shard_size *
928
- self.kv_shard_id:self.kv_proj_shard_size *
929
- (self.kv_shard_id + 1)]
930
- bias = [bias_q, bias_k, bias_v]
931
- return bias
776
+ self.output_ids = (
777
+ self.q_shard_id,
778
+ self.kv_shard_id,
779
+ self.kv_shard_id,
780
+ )
781
933
- def set_lora(
782
+ def create_lora_weights(
783
self,
935
- index: int,
936
- lora_a: torch.Tensor,
937
- lora_b: torch.Tensor,
938
- embeddings_tensor: Optional[torch.Tensor],
939
- lora_bias: Optional[torch.Tensor] = None,
940
- ):
941
- self.reset_lora(index)
942
-
943
- if self.tp_size > 1:
944
- lora_a = self.slice_lora_a(lora_a)
945
- lora_b = self.slice_lora_b(lora_b)
946
- if lora_bias is not None:
947
- lora_bias = self.slice_bias(lora_bias)
948
-
949
- if lora_b[0] is not None:
950
- lora_b_q = lora_b[0]
951
- self.lora_b_stacked[0][
952
- index, 0, :lora_b_q.shape[1], :lora_b_q.shape[0]].copy_(
953
- lora_b_q.T, non_blocking=True)
954
- if lora_b[1] is not None:
955
- lora_b_k = lora_b[1]
956
- self.lora_b_stacked[1][
957
- index, 0, :lora_b_k.shape[1], :lora_b_k.shape[0]].copy_(
958
- lora_b_k.T, non_blocking=True)
959
- if lora_b[2] is not None:
960
- lora_b_v = lora_b[2]
961
- self.lora_b_stacked[2][
962
- index, 0, :lora_b_v.shape[1], :lora_b_v.shape[0]].copy_(
963
- lora_b_v.T, non_blocking=True)
964
-
965
- if lora_a[0] is not None:
966
- self.lora_a_stacked[0][
967
- index, 0, :lora_a[0].shape[1], :lora_a[0].shape[0]].copy_(
968
- lora_a[0].T, non_blocking=True)
969
- if lora_a[1] is not None:
970
- self.lora_a_stacked[1][
971
- index, 0, :lora_a[1].shape[1], :lora_a[1].shape[0]].copy_(
972
- lora_a[1].T, non_blocking=True)
973
- if lora_a[2] is not None:
974
- self.lora_a_stacked[2][
975
- index, 0, :lora_a[2].shape[1], :lora_a[2].shape[0]].copy_(
976
- lora_a[2].T, non_blocking=True)
977
-
978
- if lora_bias is not None:
979
- self.lora_bias_stacked = cast(Tuple[torch.Tensor, ...],
980
- self.lora_bias_stacked)
981
- if lora_bias[0] is not None:
982
- self.lora_bias_stacked[0][index,
983
- 0, :lora_bias[0].shape[0]].copy_(
984
- lora_bias[0].T,
985
- non_blocking=True)
986
- if lora_bias[1] is not None:
987
- self.lora_bias_stacked[1][index,
988
- 0, :lora_bias[1].shape[0]].copy_(
989
- lora_bias[1].T,
990
- non_blocking=True)
991
- if lora_bias[2] is not None:
992
- self.lora_bias_stacked[2][index,
993
- 0, :lora_bias[2].shape[0]].copy_(
994
- lora_bias[2].T,
995
- non_blocking=True)
784
+ max_loras: int,
785
+ lora_config: LoRAConfig,
786
+ model_config: Optional[PretrainedConfig] = None,
787
+ ) -> None:
788
+ """
789
+ The main reason for overloading this function is to handle inconsistent
790
+ weight dimensions in qkv lora.
791
+ """
792
+ super().create_lora_weights(max_loras, lora_config, model_config)
793
794
@classmethod
795
@_not_fully_sharded_can_replace