@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b26b4cd03c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Misc][LoRA] Refactor and clean MergedQKVParallelLinearWithLora implementation (#10958)

Signed-off-by: Isotr0py <2037008807@qq.com>

Isotr0py committed Dec 7, 2024 at 18:33 UTC b26b4cd03c5468c68c3ce328ea6498a5d816870d
1 file changed +60 -263
vllm/lora/layers.py
+60 -263
@@ -542,10 +542,20 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
542 Both slices must have the same size.
543 """
544
545 - def __init__(self, base_layer: MergedColumnParallelLinear) -> None:
545 + def __init__(
546 + self, base_layer: Union[MergedColumnParallelLinear,
547 + QKVParallelLinear]) -> None:
548 super().__init__(base_layer)
549 # There are two LoRA layers
548 - self.n_slices = len(self.base_layer.output_sizes)
550 + self.tp_size = get_tensor_model_parallel_world_size()
551 + self.tp_rank = get_tensor_model_parallel_rank()
552 + # the output_sizes in MergedColumnParallelLinear is not sharded by tp
553 + # we need to divide it by the tp_size to get correct slices size
554 + output_sizes = self.base_layer.output_sizes
555 + self.output_slices = tuple(
556 + divide(output_size, self.tp_size) for output_size in output_sizes)
557 + self.n_slices = len(self.output_slices)
558 + self.output_ids = (self.tp_rank, ) * self.n_slices
559
560 def create_lora_weights(
561 self,
@@ -559,15 +569,6 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
569 """
570 self.lora_config = lora_config
571
562 - if not (len(self.base_layer.output_sizes) == self.n_slices == 2
563 - and self.base_layer.output_sizes[0]
564 - == self.base_layer.output_sizes[1]):
565 - raise ValueError(
566 - "LoRAColumnParallelLinear2Slice requires 2 slices with "
567 - "the same size.")
568 - self.tp_size = get_tensor_model_parallel_world_size()
569 - self.tp_rank = get_tensor_model_parallel_rank()
570 -
572 lora_a_output_size_per_partition = (
573 lora_config.max_lora_rank if not lora_config.fully_sharded_loras
574 else divide(lora_config.max_lora_rank, self.tp_size))
@@ -585,22 +586,20 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
586 torch.zeros(
587 max_loras,
588 1,
588 - self.output_size // 2,
589 + output_size,
590 lora_config.max_lora_rank,
591 dtype=lora_config.lora_dtype,
592 device=self.device,
592 - ) for _ in range(self.n_slices))
593 + ) for output_size in self.output_slices)
594 if lora_config.bias_enabled:
595 self.lora_bias_stacked = tuple(
596 torch.zeros(
597 max_loras,
598 1,
598 - self.output_size // 2,
599 + output_size,
600 dtype=lora_config.lora_dtype,
601 device=self.device,
601 - ) for _ in range(self.n_slices))
602 - self.output_dim = self.lora_b_stacked[0].shape[2]
603 - self.output_slices = (self.output_dim, self.output_dim)
602 + ) for output_size in self.output_slices)
603
604 def slice_lora_a(
605 self, lora_a: List[Union[torch.Tensor, None]]
@@ -610,27 +609,21 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
609 def slice_lora_b(
610 self, lora_b: List[Union[torch.Tensor, None]]
611 ) -> List[Union[torch.Tensor, None]]:
613 - #NOTE: lora_b contains 2 subloras, and each sublora could be None.
614 - shard_size = self.output_dim
615 - start_idx = self.tp_rank * shard_size
616 - end_idx = (self.tp_rank + 1) * shard_size
617 - lora_b = [
618 - lora_b[0][:, start_idx:end_idx] if lora_b[0] is not None else None,
619 - lora_b[1][:, start_idx:end_idx] if lora_b[1] is not None else None,
620 - ]
612 + for i, (shard_id, shard_size) in enumerate(
613 + zip(self.output_ids, self.output_slices)):
614 + if (lora_b_i := lora_b[i]) is not None:
615 + lora_b[i] = lora_b_i[:, shard_size * shard_id:shard_size *
616 + (shard_id + 1)]
617 return lora_b
618
619 def slice_bias(
620 self, bias: List[Union[torch.Tensor,
621 None]]) -> List[Union[torch.Tensor, None]]:
626 - # NOTE : each bias could be None.
627 - shard_size = self.output_dim
628 - start_idx = self.tp_rank * shard_size
629 - end_idx = (self.tp_rank + 1) * shard_size
630 - bias = [
631 - bias[0][start_idx:end_idx] if bias[0] is not None else None,
632 - bias[1][start_idx:end_idx] if bias[1] is not None else None
633 - ]
622 + for i, (shard_id, shard_size) in enumerate(
623 + zip(self.output_ids, self.output_slices)):
624 + if (bias_i := bias[i]) is not None:
625 + bias[i] = bias_i[shard_size * shard_id:shard_size *
626 + (shard_id + 1)]
627 return bias
628
629 def set_lora(
@@ -649,30 +642,25 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
642 if lora_bias is not None:
643 lora_bias = self.slice_bias(lora_bias)
644
652 - if lora_a[0] is not None:
653 - self.lora_a_stacked[0][
654 - index, 0, :lora_a[0].shape[1], :lora_a[0].shape[0]].copy_(
655 - lora_a[0].T, non_blocking=True)
656 - self.lora_b_stacked[0][
657 - index, 0, :lora_b[0].shape[1], :lora_b[0].shape[0]].copy_(
658 - lora_b[0].T, non_blocking=True)
659 - if lora_bias is not None and lora_bias[0] is not None:
660 - self.lora_bias_stacked = cast(Tuple[torch.Tensor, ...],
661 - self.lora_bias_stacked)
662 - self.lora_bias_stacked[0][index, 0, :lora_bias[0].shape[0]].copy_(
663 - lora_bias[0].T, non_blocking=True)
664 - if lora_a[1] is not None:
665 - self.lora_a_stacked[1][
666 - index, 0, :lora_a[1].shape[1], :lora_a[1].shape[0]].copy_(
667 - lora_a[1].T, non_blocking=True)
668 - self.lora_b_stacked[1][
669 - index, 0, :lora_b[1].shape[1], :lora_b[1].shape[0]].copy_(
670 - lora_b[1].T, non_blocking=True)
671 - if lora_bias is not None and lora_bias[1] is not None:
645 + for i in range(self.n_slices):
646 + if (lora_a_i := lora_a[i]) is not None:
647 + self.lora_a_stacked[i][
648 + index, 0, :lora_a_i.shape[1], :lora_a_i.shape[0]].copy_(
649 + lora_a_i.T, non_blocking=True)
650 + if (lora_b_i := lora_b[i]) is not None:
651 + self.lora_b_stacked[i][
652 + index, 0, :lora_b_i.shape[1], :lora_b_i.shape[0]].copy_(
653 + lora_b_i.T, non_blocking=True)
654 +
655 + if lora_bias is not None:
656 self.lora_bias_stacked = cast(Tuple[torch.Tensor, ...],
657 self.lora_bias_stacked)
674 - self.lora_bias_stacked[1][index, 0, :lora_bias[1].shape[0]].copy_(
675 - lora_bias[1].T, non_blocking=True)
658 + for i in range(self.n_slices):
659 + if (lora_bias_i := lora_bias[i]) is not None:
660 + self.lora_bias_stacked[i][index,
661 + 0, :lora_bias_i.shape[0]].copy_(
662 + lora_bias_i.T,
663 + non_blocking=True)
664
665 @classmethod
666 @_not_fully_sharded_can_replace
@@ -755,8 +743,8 @@ class QKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
743 packed_modules_list) == 1
744
745
758 -class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
759 - """ColumnParallelLinear layer that is composed of 3 sublayers (slices)
746 +class MergedQKVParallelLinearWithLora(MergedColumnParallelLinearWithLoRA):
747 + """MergedColumnParallelLinear layer that is composed of 3 sublayers (slices)
748 packed together in qkv proj fashion
749 (q_proj + k_proj + v_proj -> qkv_proj).
750
@@ -773,22 +761,6 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
761 self.tp_size = get_tensor_model_parallel_world_size()
762 self.tp_rank = get_tensor_model_parallel_rank()
763
776 - def create_lora_weights(
777 - self,
778 - max_loras: int,
779 - lora_config: LoRAConfig,
780 - model_config: Optional[PretrainedConfig] = None,
781 - ) -> None:
782 - """
783 - The main reason for overloading this function is to handle inconsistent
784 - weight dimensions in qkv lora.
785 - """
786 - self.lora_config = lora_config
787 -
788 - if not (len(self.base_layer.output_sizes) == self.n_slices == 3):
789 - raise ValueError(
790 - "LoRAColumnParallelLinear3Slice requires 3 slices.")
791 -
764 self.q_proj_shard_size = (self.base_layer.num_heads *
765 self.base_layer.head_size)
766 self.kv_proj_shard_size = (self.base_layer.num_kv_heads *
@@ -796,203 +768,28 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
768 self.q_shard_id = self.tp_rank
769 self.kv_shard_id = self.tp_rank // self.base_layer.num_kv_head_replicas
770
799 - lora_a_output_size_per_partition = (
800 - lora_config.max_lora_rank if not lora_config.fully_sharded_loras
801 - else divide(lora_config.max_lora_rank, self.tp_size))
802 - # q, k, v
803 - self.lora_a_stacked = (
804 - torch.zeros(
805 - max_loras,
806 - 1,
807 - lora_a_output_size_per_partition,
808 - self.input_size,
809 - dtype=lora_config.lora_dtype,
810 - device=self.device,
811 - ),
812 - torch.zeros(
813 - max_loras,
814 - 1,
815 - lora_a_output_size_per_partition,
816 - self.input_size,
817 - dtype=lora_config.lora_dtype,
818 - device=self.device,
819 - ),
820 - torch.zeros(
821 - max_loras,
822 - 1,
823 - lora_a_output_size_per_partition,
824 - self.input_size,
825 - dtype=lora_config.lora_dtype,
826 - device=self.device,
827 - ),
828 - )
829 - self.lora_b_stacked = (
830 - torch.zeros(
831 - max_loras,
832 - 1,
833 - self.q_proj_shard_size,
834 - lora_config.max_lora_rank,
835 - dtype=lora_config.lora_dtype,
836 - device=self.device,
837 - ),
838 - torch.zeros(
839 - max_loras,
840 - 1,
841 - self.kv_proj_shard_size,
842 - lora_config.max_lora_rank,
843 - dtype=lora_config.lora_dtype,
844 - device=self.device,
845 - ),
846 - torch.zeros(
847 - max_loras,
848 - 1,
849 - self.kv_proj_shard_size,
850 - lora_config.max_lora_rank,
851 - dtype=lora_config.lora_dtype,
852 - device=self.device,
853 - ),
854 - )
855 - if lora_config.bias_enabled:
856 - self.lora_bias_stacked = (
857 - torch.zeros(
858 - max_loras,
859 - 1,
860 - self.q_proj_shard_size,
861 - dtype=lora_config.lora_dtype,
862 - device=self.device,
863 - ),
864 - torch.zeros(
865 - max_loras,
866 - 1,
867 - self.kv_proj_shard_size,
868 - dtype=lora_config.lora_dtype,
869 - device=self.device,
870 - ),
871 - torch.zeros(
872 - max_loras,
873 - 1,
874 - self.kv_proj_shard_size,
875 - dtype=lora_config.lora_dtype,
876 - device=self.device,
877 - ),
878 - )
771 self.output_slices = (
772 self.q_proj_shard_size,
773 self.kv_proj_shard_size,
774 self.kv_proj_shard_size,
775 )
884 - self.packed_indices: Optional[torch.Tensor] = None
885 - self.standard_indices: Optional[torch.Tensor] = None
886 - # lazily initialized.
887 - self.indices: torch.Tensor
888 - self.indices_len: List[int]
889 -
890 - def slice_lora_a(
891 - self, lora_a: List[Union[torch.Tensor, None]]
892 - ) -> List[Union[torch.Tensor, None]]:
893 - return lora_a
894 -
895 - def slice_lora_b(
896 - self, lora_b: List[Union[torch.Tensor, None]]
897 - ) -> List[Union[torch.Tensor, None]]:
898 - lora_b_q, lora_b_k, lora_b_v = None, None, None
899 - if lora_b[0] is not None:
900 - lora_b_q = lora_b[0][:, self.q_proj_shard_size *
901 - self.q_shard_id:self.q_proj_shard_size *
902 - (self.q_shard_id + 1), ]
903 - if lora_b[1] is not None:
904 - lora_b_k = lora_b[1][:, self.kv_proj_shard_size *
905 - self.kv_shard_id:self.kv_proj_shard_size *
906 - (self.kv_shard_id + 1), ]
907 - if lora_b[2] is not None:
908 - lora_b_v = lora_b[2][:, self.kv_proj_shard_size *
909 - self.kv_shard_id:self.kv_proj_shard_size *
910 - (self.kv_shard_id + 1), ]
911 - lora_b = [lora_b_q, lora_b_k, lora_b_v]
912 - return lora_b
913 -
914 - def slice_bias(
915 - self, bias: List[Union[torch.Tensor,
916 - None]]) -> List[Union[torch.Tensor, None]]:
917 - bias_q, bias_k, bias_v = bias
918 - if bias_q is not None:
919 - bias_q = bias_q[self.q_proj_shard_size *
920 - self.q_shard_id:self.q_proj_shard_size *
921 - (self.q_shard_id + 1)]
922 - if bias_k is not None:
923 - bias_k = bias_k[self.kv_proj_shard_size *
924 - self.kv_shard_id:self.kv_proj_shard_size *
925 - (self.kv_shard_id + 1)]
926 - if bias_v is not None:
927 - bias_v = bias_v[self.kv_proj_shard_size *
928 - self.kv_shard_id:self.kv_proj_shard_size *
929 - (self.kv_shard_id + 1)]
930 - bias = [bias_q, bias_k, bias_v]
931 - return bias
776 + self.output_ids = (
777 + self.q_shard_id,
778 + self.kv_shard_id,
779 + self.kv_shard_id,
780 + )
781
933 - def set_lora(
782 + def create_lora_weights(
783 self,
935 - index: int,
936 - lora_a: torch.Tensor,
937 - lora_b: torch.Tensor,
938 - embeddings_tensor: Optional[torch.Tensor],
939 - lora_bias: Optional[torch.Tensor] = None,
940 - ):
941 - self.reset_lora(index)
942 -
943 - if self.tp_size > 1:
944 - lora_a = self.slice_lora_a(lora_a)
945 - lora_b = self.slice_lora_b(lora_b)
946 - if lora_bias is not None:
947 - lora_bias = self.slice_bias(lora_bias)
948 -
949 - if lora_b[0] is not None:
950 - lora_b_q = lora_b[0]
951 - self.lora_b_stacked[0][
952 - index, 0, :lora_b_q.shape[1], :lora_b_q.shape[0]].copy_(
953 - lora_b_q.T, non_blocking=True)
954 - if lora_b[1] is not None:
955 - lora_b_k = lora_b[1]
956 - self.lora_b_stacked[1][
957 - index, 0, :lora_b_k.shape[1], :lora_b_k.shape[0]].copy_(
958 - lora_b_k.T, non_blocking=True)
959 - if lora_b[2] is not None:
960 - lora_b_v = lora_b[2]
961 - self.lora_b_stacked[2][
962 - index, 0, :lora_b_v.shape[1], :lora_b_v.shape[0]].copy_(
963 - lora_b_v.T, non_blocking=True)
964 -
965 - if lora_a[0] is not None:
966 - self.lora_a_stacked[0][
967 - index, 0, :lora_a[0].shape[1], :lora_a[0].shape[0]].copy_(
968 - lora_a[0].T, non_blocking=True)
969 - if lora_a[1] is not None:
970 - self.lora_a_stacked[1][
971 - index, 0, :lora_a[1].shape[1], :lora_a[1].shape[0]].copy_(
972 - lora_a[1].T, non_blocking=True)
973 - if lora_a[2] is not None:
974 - self.lora_a_stacked[2][
975 - index, 0, :lora_a[2].shape[1], :lora_a[2].shape[0]].copy_(
976 - lora_a[2].T, non_blocking=True)
977 -
978 - if lora_bias is not None:
979 - self.lora_bias_stacked = cast(Tuple[torch.Tensor, ...],
980 - self.lora_bias_stacked)
981 - if lora_bias[0] is not None:
982 - self.lora_bias_stacked[0][index,
983 - 0, :lora_bias[0].shape[0]].copy_(
984 - lora_bias[0].T,
985 - non_blocking=True)
986 - if lora_bias[1] is not None:
987 - self.lora_bias_stacked[1][index,
988 - 0, :lora_bias[1].shape[0]].copy_(
989 - lora_bias[1].T,
990 - non_blocking=True)
991 - if lora_bias[2] is not None:
992 - self.lora_bias_stacked[2][index,
993 - 0, :lora_bias[2].shape[0]].copy_(
994 - lora_bias[2].T,
995 - non_blocking=True)
784 + max_loras: int,
785 + lora_config: LoRAConfig,
786 + model_config: Optional[PretrainedConfig] = None,
787 + ) -> None:
788 + """
789 + The main reason for overloading this function is to handle inconsistent
790 + weight dimensions in qkv lora.
791 + """
792 + super().create_lora_weights(max_loras, lora_config, model_config)
793
794 @classmethod
795 @_not_fully_sharded_can_replace