@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 67005a07bc
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Bugfix] Add fully sharded layer for QKVParallelLinearWithLora (#5665)

Co-authored-by: Antoni Baum <antoni.baum@protonmail.com>

Jee Li committed Jun 21, 2024 at 12:46 UTC 67005a07bc0991211ba2acccb3e56c72a47f9def
5 files changed +93 -26
tests/lora/test_baichuan.py
+9 -5
@@ -64,7 +64,8 @@ def test_baichuan_lora(baichuan_lora_files):
64
65
66 @pytest.mark.skip("Requires multiple GPUs")
67 -def test_baichuan_tensor_parallel_equality(baichuan_lora_files):
67 +@pytest.mark.parametrize("fully_sharded", [True, False])
68 +def test_baichuan_tensor_parallel_equality(baichuan_lora_files, fully_sharded):
69 # Cannot use as it will initialize torch.cuda too early...
70 # if torch.cuda.device_count() < 4:
71 # pytest.skip(f"Not enough GPUs for tensor parallelism {4}")
@@ -75,7 +76,8 @@ def test_baichuan_tensor_parallel_equality(baichuan_lora_files):
76 max_loras=4,
77 max_lora_rank=64,
78 tensor_parallel_size=1,
78 - trust_remote_code=True)
79 + trust_remote_code=True,
80 + fully_sharded_loras=fully_sharded)
81 output_tp1 = do_sample(llm_tp1, baichuan_lora_files, lora_id=1)
82
83 del llm_tp1
@@ -87,7 +89,8 @@ def test_baichuan_tensor_parallel_equality(baichuan_lora_files):
89 max_loras=4,
90 max_lora_rank=64,
91 tensor_parallel_size=2,
90 - trust_remote_code=True)
92 + trust_remote_code=True,
93 + fully_sharded_loras=fully_sharded)
94 output_tp2 = do_sample(llm_tp2, baichuan_lora_files, lora_id=2)
95
96 del llm_tp2
@@ -101,10 +104,11 @@ def test_baichuan_tensor_parallel_equality(baichuan_lora_files):
104 max_loras=4,
105 max_lora_rank=64,
106 tensor_parallel_size=4,
104 - trust_remote_code=True)
107 + trust_remote_code=True,
108 + fully_sharded_loras=fully_sharded)
109 output_tp4 = do_sample(llm_tp4, baichuan_lora_files, lora_id=2)
110
111 del llm_tp4
112 cleanup()
113
110 - assert output_tp1 == output_tp4
\ No newline at end of file
114 + assert output_tp1 == output_tp4
tests/lora/test_layers.py
+5 -2
@@ -12,7 +12,8 @@ from vllm.config import LoRAConfig
12 from vllm.lora.fully_sharded_layers import (
13 ColumnParallelLinearWithShardedLoRA,
14 MergedColumnParallelLinearWithShardedLoRA,
15 - MergedQKVParallelLinearWithShardedLora, RowParallelLinearWithShardedLoRA)
15 + MergedQKVParallelLinearWithShardedLora, QKVParallelLinearWithShardedLora,
16 + RowParallelLinearWithShardedLoRA)
17 # yapf conflicts with isort for this block
18 # yapf: disable
19 from vllm.lora.layers import (BaseLayerWithLoRA, ColumnParallelLinearWithLoRA,
@@ -684,7 +685,9 @@ def test_column_parallel_packed(dist_init, num_loras, repeats, fully_shard,
685 bias=False,
686 params_dtype=torch.float16)
687 linear.weight.data = torch.rand_like(linear.weight.data)
687 - lora_linear = QKVParallelLinearWithLora(linear)
688 + lora_linear = QKVParallelLinearWithLora(
689 + linear
690 + ) if not fully_shard else QKVParallelLinearWithShardedLora(linear)
691
692 @dataclass
693 class FakeConfig:
vllm/lora/fully_sharded_layers.py
+55 -3
@@ -12,6 +12,7 @@ from vllm.distributed.parallel_state import get_tensor_model_parallel_rank
12 from vllm.lora.layers import (ColumnParallelLinearWithLoRA,
13 MergedColumnParallelLinearWithLoRA,
14 MergedQKVParallelLinearWithLora,
15 + QKVParallelLinearWithLora,
16 RowParallelLinearWithLoRA)
17 from vllm.lora.punica import bgmv, dispatch_bgmv_low_level
18
@@ -90,11 +91,11 @@ class ColumnParallelLinearWithShardedLoRA(ColumnParallelLinearWithLoRA):
91 def _mcp_apply(x, bias, layer):
92 """
93 MergedColumnParallelLinearWithShardedLoRA and
93 - QKVParallelLinearWithShardedLora share the same
94 + MergedQKVParallelLinearWithShardedLora share the same
95 LoRa weight application method.
96
97 The main difference is the step by shard_size for lora_b which can
97 - vary for QKVParallelLinearWithShardedLora but is constant for
98 + vary for MergedQKVParallelLinearWithShardedLora but is constant for
99 MergedColumnParallelLinearWithShardedLoRA.
100 """
101 # expecting 2 for column parallel and 3 for qkv
@@ -167,7 +168,7 @@ class MergedColumnParallelLinearWithShardedLoRA(
168 )
169
170
170 -class MergedQKVParallelLinearWithShardedLora(MergedQKVParallelLinearWithLora):
171 +class QKVParallelLinearWithShardedLora(QKVParallelLinearWithLora):
172 """
173 Differs from QKVParallelLinearWithLora by slicing the
174 LoRA A's also.
@@ -175,6 +176,57 @@ class MergedQKVParallelLinearWithShardedLora(MergedQKVParallelLinearWithLora):
176 Based on S-LoRA, slicing happens along the rank dim.
177 """
178
179 + def slice_lora_a(self, lora_a: torch.Tensor) -> torch.Tensor:
180 + tp_rank = get_tensor_model_parallel_rank()
181 + shard_size = self.lora_a_stacked.shape[2]
182 + start_idx = tp_rank * shard_size
183 + lora_a = lora_a[:, start_idx:start_idx + shard_size]
184 + return lora_a
185 +
186 + def apply(self, x: torch.Tensor,
187 + bias: Optional[torch.Tensor]) -> torch.Tensor:
188 + output = self.base_layer.quant_method.apply(self.base_layer, x, bias)
189 +
190 + x = x.view(-1, x.shape[-1])
191 + output, out_orig_shape = output.view(-1,
192 + output.shape[-1]), output.shape
193 + buffer = torch.zeros((x.shape[0], self.lora_a_stacked.shape[2]),
194 + dtype=torch.float32,
195 + device=x.device)
196 +
197 + bgmv(buffer, x, self.lora_a_stacked,
198 + self.indices[:self.indices_len[0]], 0, 1.0)
199 + buffer = tensor_model_parallel_all_gather(buffer)
200 + bgmv(output, buffer, self.lora_b_stacked,
201 + self.indices[:self.indices_len[0]], 0, 1.0)
202 + # now have column partitioned output
203 +
204 + output = output.view(*out_orig_shape)
205 + return output
206 +
207 + @classmethod
208 + @_fully_sharded_can_replace
209 + def can_replace_layer(cls, source_layer: nn.Module,
210 + lora_config: LoRAConfig, packed_modules_list: List,
211 + model_config: Optional[PretrainedConfig]) -> bool:
212 + # specifying kwargs so they can be easily accessed in decorator
213 + return super().can_replace_layer(
214 + source_layer=source_layer,
215 + lora_config=lora_config,
216 + packed_modules_list=packed_modules_list,
217 + model_config=model_config,
218 + decorate=False,
219 + )
220 +
221 +
222 +class MergedQKVParallelLinearWithShardedLora(MergedQKVParallelLinearWithLora):
223 + """
224 + Differs from MergedQKVParallelLinearWithLora by slicing the
225 + LoRA A's also.
226 +
227 + Based on S-LoRA, slicing happens along the rank dim.
228 + """
229 +
230 def slice_lora_a(
231 self, lora_a: List[Union[torch.Tensor, None]]
232 ) -> List[Union[torch.Tensor, None]]:
vllm/lora/layers.py
+21 -15
@@ -641,6 +641,24 @@ class QKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
641 self.kv_proj_total_size = (self.base_layer.total_num_kv_heads *
642 self.base_layer.head_size)
643
644 + def slice_lora_b(self, lora_b: torch.Tensor) -> torch.Tensor:
645 + tp_rank = get_tensor_model_parallel_rank()
646 + self.q_shard_id = tp_rank
647 + self.kv_shard_id = tp_rank // self.base_layer.num_kv_head_replicas
648 + lora_b_q = lora_b[:, self.q_proj_shard_size *
649 + self.q_shard_id:self.q_proj_shard_size *
650 + (self.q_shard_id + 1)]
651 + k_offset = self.q_proj_total_size
652 + lora_b_k = lora_b[:, k_offset +
653 + self.kv_proj_shard_size * self.kv_shard_id:k_offset +
654 + self.kv_proj_shard_size * (self.kv_shard_id + 1)]
655 + v_offset = k_offset + self.kv_proj_total_size
656 + lora_b_v = lora_b[:, v_offset +
657 + self.kv_proj_shard_size * self.kv_shard_id:v_offset +
658 + self.kv_proj_shard_size * (self.kv_shard_id + 1)]
659 + lora_b = torch.cat([lora_b_q, lora_b_k, lora_b_v], dim=1)
660 + return lora_b
661 +
662 def set_lora(
663 self,
664 index: int,
@@ -650,21 +668,8 @@ class QKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
668 ):
669 self.reset_lora(index)
670 if self.tp_size > 1:
653 - tp_rank = get_tensor_model_parallel_rank()
654 - self.q_shard_id = tp_rank
655 - self.kv_shard_id = tp_rank // self.base_layer.num_kv_head_replicas
656 - lora_b_q = lora_b[:, self.q_proj_shard_size *
657 - self.q_shard_id:self.q_proj_shard_size *
658 - (self.q_shard_id + 1)]
659 - k_offset = self.q_proj_total_size
660 - lora_b_k = lora_b[:, k_offset + self.kv_proj_shard_size *
661 - self.kv_shard_id:k_offset +
662 - self.kv_proj_shard_size * (self.kv_shard_id + 1)]
663 - v_offset = k_offset + self.kv_proj_total_size
664 - lora_b_v = lora_b[:, v_offset + self.kv_proj_shard_size *
665 - self.kv_shard_id:v_offset +
666 - self.kv_proj_shard_size * (self.kv_shard_id + 1)]
667 - lora_b = torch.cat([lora_b_q, lora_b_k, lora_b_v], dim=1)
671 + lora_a = self.slice_lora_a(lora_a)
672 + lora_b = self.slice_lora_b(lora_b)
673
674 self.lora_a_stacked[index,
675 0, :lora_a.shape[1], :lora_a.shape[0]].copy_(
@@ -674,6 +679,7 @@ class QKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
679 lora_b.T, non_blocking=True)
680
681 @classmethod
682 + @_not_fully_sharded_can_replace
683 def can_replace_layer(cls, source_layer: nn.Module,
684 lora_config: LoRAConfig, packed_modules_list: List,
685 model_config: Optional[PretrainedConfig]) -> bool:
vllm/lora/utils.py
+3 -1
@@ -8,7 +8,8 @@ from vllm.logger import init_logger
8 from vllm.lora.fully_sharded_layers import (
9 ColumnParallelLinearWithShardedLoRA,
10 MergedColumnParallelLinearWithShardedLoRA,
11 - MergedQKVParallelLinearWithShardedLora, RowParallelLinearWithShardedLoRA)
11 + MergedQKVParallelLinearWithShardedLora, QKVParallelLinearWithShardedLora,
12 + RowParallelLinearWithShardedLoRA)
13 # being imported for _all_lora_classes below
14 # yapf conflicts with isort for this block
15 # yapf: disable
@@ -35,6 +36,7 @@ _all_lora_classes: Set[Type[BaseLayerWithLoRA]] = {
36 RowParallelLinearWithLoRA,
37 LogitsProcessorWithLoRA,
38 ColumnParallelLinearWithShardedLoRA,
39 + QKVParallelLinearWithShardedLora,
40 MergedColumnParallelLinearWithShardedLoRA,
41 MergedQKVParallelLinearWithShardedLora,
42 RowParallelLinearWithShardedLoRA,