A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Bugfix] Add fully sharded layer for QKVParallelLinearWithLora (#5665)
Co-authored-by: Antoni Baum <antoni.baum@protonmail.com>
Jee Li committed
Jun 21, 2024 at 12:46 UTC
67005a07bc0991211ba2acccb3e56c72a47f9def
5 files changed
+93
-26
tests/lora/test_baichuan.py
+9
-5
@@ -64,7 +64,8 @@ def test_baichuan_lora(baichuan_lora_files):
64
65
66
@pytest.mark.skip("Requires multiple GPUs")
67
-def test_baichuan_tensor_parallel_equality(baichuan_lora_files):
67
+@pytest.mark.parametrize("fully_sharded", [True, False])
68
+def test_baichuan_tensor_parallel_equality(baichuan_lora_files, fully_sharded):
69
# Cannot use as it will initialize torch.cuda too early...
70
# if torch.cuda.device_count() < 4:
71
# pytest.skip(f"Not enough GPUs for tensor parallelism {4}")
@@ -75,7 +76,8 @@ def test_baichuan_tensor_parallel_equality(baichuan_lora_files):
76
max_loras=4,
77
max_lora_rank=64,
78
tensor_parallel_size=1,
78
- trust_remote_code=True)
79
+ trust_remote_code=True,
80
+ fully_sharded_loras=fully_sharded)
81
output_tp1 = do_sample(llm_tp1, baichuan_lora_files, lora_id=1)
82
83
del llm_tp1
@@ -87,7 +89,8 @@ def test_baichuan_tensor_parallel_equality(baichuan_lora_files):
89
max_loras=4,
90
max_lora_rank=64,
91
tensor_parallel_size=2,
90
- trust_remote_code=True)
92
+ trust_remote_code=True,
93
+ fully_sharded_loras=fully_sharded)
94
output_tp2 = do_sample(llm_tp2, baichuan_lora_files, lora_id=2)
95
96
del llm_tp2
@@ -101,10 +104,11 @@ def test_baichuan_tensor_parallel_equality(baichuan_lora_files):
104
max_loras=4,
105
max_lora_rank=64,
106
tensor_parallel_size=4,
104
- trust_remote_code=True)
107
+ trust_remote_code=True,
108
+ fully_sharded_loras=fully_sharded)
109
output_tp4 = do_sample(llm_tp4, baichuan_lora_files, lora_id=2)
110
111
del llm_tp4
112
cleanup()
113
110
- assert output_tp1 == output_tp4
\ No newline at end of file
114
+ assert output_tp1 == output_tp4
tests/lora/test_layers.py
+5
-2
@@ -12,7 +12,8 @@ from vllm.config import LoRAConfig
12
from vllm.lora.fully_sharded_layers import (
13
ColumnParallelLinearWithShardedLoRA,
14
MergedColumnParallelLinearWithShardedLoRA,
15
- MergedQKVParallelLinearWithShardedLora, RowParallelLinearWithShardedLoRA)
15
+ MergedQKVParallelLinearWithShardedLora, QKVParallelLinearWithShardedLora,
16
+ RowParallelLinearWithShardedLoRA)
17
# yapf conflicts with isort for this block
18
# yapf: disable
19
from vllm.lora.layers import (BaseLayerWithLoRA, ColumnParallelLinearWithLoRA,
@@ -684,7 +685,9 @@ def test_column_parallel_packed(dist_init, num_loras, repeats, fully_shard,
685
bias=False,
686
params_dtype=torch.float16)
687
linear.weight.data = torch.rand_like(linear.weight.data)
687
- lora_linear = QKVParallelLinearWithLora(linear)
688
+ lora_linear = QKVParallelLinearWithLora(
689
+ linear
690
+ ) if not fully_shard else QKVParallelLinearWithShardedLora(linear)
691
692
@dataclass
693
class FakeConfig:
vllm/lora/fully_sharded_layers.py
+55
-3
@@ -12,6 +12,7 @@ from vllm.distributed.parallel_state import get_tensor_model_parallel_rank
12
from vllm.lora.layers import (ColumnParallelLinearWithLoRA,
13
MergedColumnParallelLinearWithLoRA,
14
MergedQKVParallelLinearWithLora,
15
+ QKVParallelLinearWithLora,
16
RowParallelLinearWithLoRA)
17
from vllm.lora.punica import bgmv, dispatch_bgmv_low_level
18
@@ -90,11 +91,11 @@ class ColumnParallelLinearWithShardedLoRA(ColumnParallelLinearWithLoRA):
91
def _mcp_apply(x, bias, layer):
92
"""
93
MergedColumnParallelLinearWithShardedLoRA and
93
- QKVParallelLinearWithShardedLora share the same
94
+ MergedQKVParallelLinearWithShardedLora share the same
95
LoRa weight application method.
96
97
The main difference is the step by shard_size for lora_b which can
97
- vary for QKVParallelLinearWithShardedLora but is constant for
98
+ vary for MergedQKVParallelLinearWithShardedLora but is constant for
99
MergedColumnParallelLinearWithShardedLoRA.
100
"""
101
# expecting 2 for column parallel and 3 for qkv
@@ -167,7 +168,7 @@ class MergedColumnParallelLinearWithShardedLoRA(
168
)
169
170
170
-class MergedQKVParallelLinearWithShardedLora(MergedQKVParallelLinearWithLora):
171
+class QKVParallelLinearWithShardedLora(QKVParallelLinearWithLora):
172
"""
173
Differs from QKVParallelLinearWithLora by slicing the
174
LoRA A's also.
@@ -175,6 +176,57 @@ class MergedQKVParallelLinearWithShardedLora(MergedQKVParallelLinearWithLora):
176
Based on S-LoRA, slicing happens along the rank dim.
177
"""
178
179
+ def slice_lora_a(self, lora_a: torch.Tensor) -> torch.Tensor:
180
+ tp_rank = get_tensor_model_parallel_rank()
181
+ shard_size = self.lora_a_stacked.shape[2]
182
+ start_idx = tp_rank * shard_size
183
+ lora_a = lora_a[:, start_idx:start_idx + shard_size]
184
+ return lora_a
185
+
186
+ def apply(self, x: torch.Tensor,
187
+ bias: Optional[torch.Tensor]) -> torch.Tensor:
188
+ output = self.base_layer.quant_method.apply(self.base_layer, x, bias)
189
+
190
+ x = x.view(-1, x.shape[-1])
191
+ output, out_orig_shape = output.view(-1,
192
+ output.shape[-1]), output.shape
193
+ buffer = torch.zeros((x.shape[0], self.lora_a_stacked.shape[2]),
194
+ dtype=torch.float32,
195
+ device=x.device)
196
+
197
+ bgmv(buffer, x, self.lora_a_stacked,
198
+ self.indices[:self.indices_len[0]], 0, 1.0)
199
+ buffer = tensor_model_parallel_all_gather(buffer)
200
+ bgmv(output, buffer, self.lora_b_stacked,
201
+ self.indices[:self.indices_len[0]], 0, 1.0)
202
+ # now have column partitioned output
203
+
204
+ output = output.view(*out_orig_shape)
205
+ return output
206
+
207
+ @classmethod
208
+ @_fully_sharded_can_replace
209
+ def can_replace_layer(cls, source_layer: nn.Module,
210
+ lora_config: LoRAConfig, packed_modules_list: List,
211
+ model_config: Optional[PretrainedConfig]) -> bool:
212
+ # specifying kwargs so they can be easily accessed in decorator
213
+ return super().can_replace_layer(
214
+ source_layer=source_layer,
215
+ lora_config=lora_config,
216
+ packed_modules_list=packed_modules_list,
217
+ model_config=model_config,
218
+ decorate=False,
219
+ )
220
+
221
+
222
+class MergedQKVParallelLinearWithShardedLora(MergedQKVParallelLinearWithLora):
223
+ """
224
+ Differs from MergedQKVParallelLinearWithLora by slicing the
225
+ LoRA A's also.
226
+
227
+ Based on S-LoRA, slicing happens along the rank dim.
228
+ """
229
+
230
def slice_lora_a(
231
self, lora_a: List[Union[torch.Tensor, None]]
232
) -> List[Union[torch.Tensor, None]]:
vllm/lora/layers.py
+21
-15
@@ -641,6 +641,24 @@ class QKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
641
self.kv_proj_total_size = (self.base_layer.total_num_kv_heads *
642
self.base_layer.head_size)
643
644
+ def slice_lora_b(self, lora_b: torch.Tensor) -> torch.Tensor:
645
+ tp_rank = get_tensor_model_parallel_rank()
646
+ self.q_shard_id = tp_rank
647
+ self.kv_shard_id = tp_rank // self.base_layer.num_kv_head_replicas
648
+ lora_b_q = lora_b[:, self.q_proj_shard_size *
649
+ self.q_shard_id:self.q_proj_shard_size *
650
+ (self.q_shard_id + 1)]
651
+ k_offset = self.q_proj_total_size
652
+ lora_b_k = lora_b[:, k_offset +
653
+ self.kv_proj_shard_size * self.kv_shard_id:k_offset +
654
+ self.kv_proj_shard_size * (self.kv_shard_id + 1)]
655
+ v_offset = k_offset + self.kv_proj_total_size
656
+ lora_b_v = lora_b[:, v_offset +
657
+ self.kv_proj_shard_size * self.kv_shard_id:v_offset +
658
+ self.kv_proj_shard_size * (self.kv_shard_id + 1)]
659
+ lora_b = torch.cat([lora_b_q, lora_b_k, lora_b_v], dim=1)
660
+ return lora_b
661
+
662
def set_lora(
663
self,
664
index: int,
@@ -650,21 +668,8 @@ class QKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
668
):
669
self.reset_lora(index)
670
if self.tp_size > 1:
653
- tp_rank = get_tensor_model_parallel_rank()
654
- self.q_shard_id = tp_rank
655
- self.kv_shard_id = tp_rank // self.base_layer.num_kv_head_replicas
656
- lora_b_q = lora_b[:, self.q_proj_shard_size *
657
- self.q_shard_id:self.q_proj_shard_size *
658
- (self.q_shard_id + 1)]
659
- k_offset = self.q_proj_total_size
660
- lora_b_k = lora_b[:, k_offset + self.kv_proj_shard_size *
661
- self.kv_shard_id:k_offset +
662
- self.kv_proj_shard_size * (self.kv_shard_id + 1)]
663
- v_offset = k_offset + self.kv_proj_total_size
664
- lora_b_v = lora_b[:, v_offset + self.kv_proj_shard_size *
665
- self.kv_shard_id:v_offset +
666
- self.kv_proj_shard_size * (self.kv_shard_id + 1)]
667
- lora_b = torch.cat([lora_b_q, lora_b_k, lora_b_v], dim=1)
671
+ lora_a = self.slice_lora_a(lora_a)
672
+ lora_b = self.slice_lora_b(lora_b)
673
674
self.lora_a_stacked[index,
675
0, :lora_a.shape[1], :lora_a.shape[0]].copy_(
@@ -674,6 +679,7 @@ class QKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
679
lora_b.T, non_blocking=True)
680
681
@classmethod
682
+ @_not_fully_sharded_can_replace
683
def can_replace_layer(cls, source_layer: nn.Module,
684
lora_config: LoRAConfig, packed_modules_list: List,
685
model_config: Optional[PretrainedConfig]) -> bool:
vllm/lora/utils.py
+3
-1
@@ -8,7 +8,8 @@ from vllm.logger import init_logger
8
from vllm.lora.fully_sharded_layers import (
9
ColumnParallelLinearWithShardedLoRA,
10
MergedColumnParallelLinearWithShardedLoRA,
11
- MergedQKVParallelLinearWithShardedLora, RowParallelLinearWithShardedLoRA)
11
+ MergedQKVParallelLinearWithShardedLora, QKVParallelLinearWithShardedLora,
12
+ RowParallelLinearWithShardedLoRA)
13
# being imported for _all_lora_classes below
14
# yapf conflicts with isort for this block
15
# yapf: disable
@@ -35,6 +36,7 @@ _all_lora_classes: Set[Type[BaseLayerWithLoRA]] = {
36
RowParallelLinearWithLoRA,
37
LogitsProcessorWithLoRA,
38
ColumnParallelLinearWithShardedLoRA,
39
+ QKVParallelLinearWithShardedLora,
40
MergedColumnParallelLinearWithShardedLoRA,
41
MergedQKVParallelLinearWithShardedLora,
42
RowParallelLinearWithShardedLoRA,