A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[Misc][LoRA] Clean up the function interface of Punica (#10917)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Dec 5, 2024 at 21:22 UTC
571da8fc431ec36427ee1034a7779b23229b015e
5 files changed
+497
-631
tests/lora/test_layers.py
+32
-10
@@ -565,7 +565,9 @@ def test_lm_head_logits_processor(dist_init, num_loras, device, vocab_size,
565
@pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
566
@pytest.mark.parametrize("device", CUDA_DEVICES)
567
@pytest.mark.parametrize("stage", STAGES)
568
-def test_linear_replicated(dist_init, num_loras, device, stage) -> None:
568
+@pytest.mark.parametrize("bias_enabled", [True, False])
569
+def test_linear_replicated(dist_init, num_loras, device, stage,
570
+ bias_enabled) -> None:
571
572
torch.cuda.set_device(device)
573
torch.set_default_device(device)
@@ -573,7 +575,8 @@ def test_linear_replicated(dist_init, num_loras, device, stage) -> None:
575
max_loras = 8
576
lora_config = LoRAConfig(max_loras=max_loras,
577
max_lora_rank=8,
576
- lora_dtype=torch.float16)
578
+ lora_dtype=torch.float16,
579
+ bias_enabled=bias_enabled)
580
581
def create_random_linear_replicated_layer():
582
@@ -585,7 +588,12 @@ def test_linear_replicated(dist_init, num_loras, device, stage) -> None:
588
lora_linear = ReplicatedLinearWithLoRA(linear)
589
590
lora_linear.create_lora_weights(max_loras, lora_config)
588
-
591
+ assert (lora_linear.n_slices == len(lora_linear.lora_a_stacked) == len(
592
+ lora_linear.lora_b_stacked) == 1)
593
+ if bias_enabled:
594
+ assert len(lora_linear.lora_bias_stacked) == lora_linear.n_slices
595
+ else:
596
+ assert lora_linear.lora_bias_stacked is None
597
return linear, lora_linear
598
599
for i in range(10):
@@ -669,8 +677,9 @@ def test_linear_replicated(dist_init, num_loras, device, stage) -> None:
677
@pytest.mark.parametrize("fully_shard", [True, False])
678
@pytest.mark.parametrize("device", CUDA_DEVICES)
679
@pytest.mark.parametrize("stage", STAGES)
680
+@pytest.mark.parametrize("bias_enabled", [True, False])
681
def test_linear_parallel(dist_init, num_loras, orientation, fully_shard,
673
- device, stage) -> None:
682
+ device, stage, bias_enabled) -> None:
683
684
torch.cuda.set_device(device)
685
torch.set_default_device(device)
@@ -679,7 +688,8 @@ def test_linear_parallel(dist_init, num_loras, orientation, fully_shard,
688
lora_config = LoRAConfig(max_loras=max_loras,
689
max_lora_rank=8,
690
fully_sharded_loras=fully_shard,
682
- lora_dtype=torch.float16)
691
+ lora_dtype=torch.float16,
692
+ bias_enabled=bias_enabled)
693
694
def create_random_linear_parallel_layer():
695
if orientation == "row":
@@ -700,7 +710,12 @@ def test_linear_parallel(dist_init, num_loras, orientation, fully_shard,
710
if not fully_shard else
711
ColumnParallelLinearWithShardedLoRA(linear))
712
lora_linear.create_lora_weights(max_loras, lora_config)
703
-
713
+ assert (lora_linear.n_slices == len(lora_linear.lora_a_stacked) == len(
714
+ lora_linear.lora_b_stacked) == 1)
715
+ if bias_enabled:
716
+ assert len(lora_linear.lora_bias_stacked) == lora_linear.n_slices
717
+ else:
718
+ assert lora_linear.lora_bias_stacked is None
719
return linear, lora_linear
720
721
for i in range(10):
@@ -784,8 +799,9 @@ def test_linear_parallel(dist_init, num_loras, orientation, fully_shard,
799
@pytest.mark.parametrize("fully_shard", [True, False])
800
@pytest.mark.parametrize("device", CUDA_DEVICES)
801
@pytest.mark.parametrize("stage", STAGES)
802
+@pytest.mark.parametrize("bias_enabled", [True, False])
803
def test_column_parallel_packed(dist_init, num_loras, repeats, fully_shard,
788
- device, stage) -> None:
804
+ device, stage, bias_enabled) -> None:
805
806
torch.cuda.set_device(device)
807
torch.set_default_device(device)
@@ -794,7 +810,8 @@ def test_column_parallel_packed(dist_init, num_loras, repeats, fully_shard,
810
lora_config = LoRAConfig(max_loras=max_loras,
811
max_lora_rank=8,
812
fully_sharded_loras=fully_shard,
797
- lora_dtype=torch.float16)
813
+ lora_dtype=torch.float16,
814
+ bias_enabled=bias_enabled)
815
816
def create_column_parallel_packed_layer():
817
if repeats == 2:
@@ -832,10 +849,16 @@ def test_column_parallel_packed(dist_init, num_loras, repeats, fully_shard,
849
num_key_value_heads = 32
850
num_attention_heads = 32
851
852
+ n_slices = repeats
853
lora_linear.create_lora_weights(max_loras,
854
lora_config,
855
model_config=FakeConfig())
838
-
856
+ assert (lora_linear.n_slices == len(lora_linear.lora_a_stacked) == len(
857
+ lora_linear.lora_b_stacked) == n_slices)
858
+ if bias_enabled:
859
+ assert len(lora_linear.lora_bias_stacked) == lora_linear.n_slices
860
+ else:
861
+ assert lora_linear.lora_bias_stacked is None
862
return linear, lora_linear
863
864
for i in range(10):
@@ -911,7 +934,6 @@ def test_column_parallel_packed(dist_init, num_loras, repeats, fully_shard,
934
512,
935
lora_config.lora_extra_vocab_size,
936
)
914
- # lora_linear.set_mapping(*mapping_info)
937
938
lora_result = lora_linear(torch.cat(inputs))[0]
939
expected_result = linear(torch.cat(inputs))[0]
vllm/lora/fully_sharded_layers.py
+75
-100
@@ -1,5 +1,5 @@
1
# pylint: disable=unused-argument
2
-from typing import TYPE_CHECKING, List, Optional, Union
2
+from typing import TYPE_CHECKING, List, Optional, Tuple, Union, cast
3
4
import torch
5
import torch.nn as nn
@@ -32,6 +32,44 @@ def _fully_sharded_can_replace(can_replace):
32
return dec
33
34
35
+def _mcp_apply(x, bias, layer: ColumnParallelLinearWithLoRA):
36
+ """
37
+ For `ColumnParallelLinearWithLoRA` or classes that inherit from
38
+ `ColumnParallelLinearWithLoRA`, they share the same `apply` logic.
39
+ """
40
+ assert (layer.n_slices == len(layer.lora_a_stacked) == len(
41
+ layer.lora_b_stacked) == len(layer.output_slices))
42
+ if layer.lora_bias_stacked is not None:
43
+ assert layer.n_slices == len(layer.lora_bias_stacked)
44
+
45
+ output = layer.base_layer.quant_method.apply(layer.base_layer, x, bias)
46
+
47
+ x = x.view(-1, x.shape[-1])
48
+ output, out_orig_shape = output.view(-1, output.shape[-1]), output.shape
49
+
50
+ # Since communication is needed, the buffer is directly initialized as a
51
+ # tensor rather than a tuple of tensor.
52
+ buffers = torch.zeros(
53
+ (layer.n_slices, x.shape[0], layer.lora_a_stacked[0].shape[2]),
54
+ dtype=torch.float32,
55
+ device=x.device,
56
+ )
57
+
58
+ layer.punica_wrapper.add_shrink(buffers, x, layer.lora_a_stacked, 1.0)
59
+ buffers = tensor_model_parallel_all_gather(buffers)
60
+ layer.punica_wrapper.add_expand(output,
61
+ buffers,
62
+ layer.lora_b_stacked,
63
+ layer.lora_bias_stacked,
64
+ layer.output_slices,
65
+ offset_start=0,
66
+ add_input=True)
67
+
68
+ output = output.view(*out_orig_shape)
69
+ # now have column partitioned and packed output
70
+ return output
71
+
72
+
73
# these layers are based on the tensor parallelism strategy given in
74
# Y. Sheng et al., S-LoRA: Serving Thousands of Concurrent LoRA Adapters. 2023,
75
# https://arxiv.org/abs/2311.03285.
@@ -51,34 +89,15 @@ class ColumnParallelLinearWithShardedLoRA(ColumnParallelLinearWithLoRA):
89
# gather operation.
90
def slice_lora_a(self, lora_a: torch.Tensor) -> torch.Tensor:
91
tp_rank = get_tensor_model_parallel_rank()
54
- shard_size = self.lora_a_stacked.shape[2]
92
+ shard_size = self.lora_a_stacked[0].shape[2]
93
start_idx = tp_rank * shard_size
94
lora_a = lora_a[:, start_idx:start_idx + shard_size]
95
return lora_a
96
59
- def apply(self, x: torch.Tensor,
60
- bias: Optional[torch.Tensor]) -> torch.Tensor:
61
- output = self.base_layer.quant_method.apply(self.base_layer, x, bias)
62
-
63
- x = x.view(-1, x.shape[-1])
64
- output, out_orig_shape = output.view(-1,
65
- output.shape[-1]), output.shape
66
- buffer = torch.zeros(
67
- (x.shape[0], self.lora_a_stacked.shape[2]),
68
- dtype=torch.float32,
69
- device=x.device,
70
- )
71
- self.punica_wrapper.add_shrink(buffer, x, self.lora_a_stacked, 1.0)
72
- buffer = tensor_model_parallel_all_gather(buffer)
73
- self.punica_wrapper.add_expand(output,
74
- buffer,
75
- self.lora_b_stacked,
76
- self.bias_stacked,
77
- add_input=True)
78
- # now have column partitioned output
79
-
80
- output = output.view(*out_orig_shape)
81
- return output
97
+ def apply(self,
98
+ x: torch.Tensor,
99
+ bias: Optional[torch.Tensor] = None) -> torch.Tensor:
100
+ return _mcp_apply(x, bias, self)
101
102
@classmethod
103
@_fully_sharded_can_replace
@@ -99,46 +118,6 @@ class ColumnParallelLinearWithShardedLoRA(ColumnParallelLinearWithLoRA):
118
)
119
120
102
-def _mcp_apply(x, bias, layer: QKVParallelLinearWithLora):
103
- """
104
- MergedColumnParallelLinearWithShardedLoRA and
105
- MergedQKVParallelLinearWithShardedLora share the same
106
- LoRa weight application method.
107
-
108
- The main difference is the step by shard_size for lora_b which can
109
- vary for MergedQKVParallelLinearWithShardedLora but is constant for
110
- MergedColumnParallelLinearWithShardedLoRA.
111
- """
112
- # expecting 2 for column parallel and 3 for qkv
113
- n = len(layer.lora_a_stacked)
114
- output = layer.base_layer.quant_method.apply(layer.base_layer, x, bias)
115
-
116
- x = x.view(-1, x.shape[-1])
117
- output, out_orig_shape = output.view(-1, output.shape[-1]), output.shape
118
- buffers = torch.zeros(
119
- (n, x.shape[0], layer.lora_a_stacked[0].shape[2]),
120
- dtype=torch.float32,
121
- device=x.device,
122
- )
123
- for idx in range(n):
124
- layer.punica_wrapper.add_shrink(buffers[idx], x,
125
- layer.lora_a_stacked[idx], 1.0)
126
-
127
- buffers = tensor_model_parallel_all_gather(buffers)
128
- layer.punica_wrapper.add_expand_packed_nslice(
129
- output,
130
- buffers,
131
- layer.lora_b_stacked,
132
- layer.bias_stacked,
133
- 1.0,
134
- layer.output_slices,
135
- )
136
-
137
- output = output.view(*out_orig_shape)
138
- # now have column partitioned and packed output
139
- return output
140
-
141
-
121
class MergedColumnParallelLinearWithShardedLoRA(
122
MergedColumnParallelLinearWithLoRA):
123
"""
@@ -162,8 +141,9 @@ class MergedColumnParallelLinearWithShardedLoRA(
141
]
142
return lora_a
143
165
- def apply(self, x: torch.Tensor,
166
- bias: Optional[torch.Tensor]) -> torch.Tensor:
144
+ def apply(self,
145
+ x: torch.Tensor,
146
+ bias: Optional[torch.Tensor] = None) -> torch.Tensor:
147
return _mcp_apply(x, bias, self)
148
149
@classmethod
@@ -195,31 +175,15 @@ class QKVParallelLinearWithShardedLora(QKVParallelLinearWithLora):
175
176
def slice_lora_a(self, lora_a: torch.Tensor) -> torch.Tensor:
177
tp_rank = get_tensor_model_parallel_rank()
198
- shard_size = self.lora_a_stacked.shape[2]
178
+ shard_size = self.lora_a_stacked[0].shape[2]
179
start_idx = tp_rank * shard_size
180
lora_a = lora_a[:, start_idx:start_idx + shard_size]
181
return lora_a
182
203
- def apply(self, x: torch.Tensor,
204
- bias: Optional[torch.Tensor]) -> torch.Tensor:
205
- output = self.base_layer.quant_method.apply(self.base_layer, x, bias)
206
-
207
- x = x.view(-1, x.shape[-1])
208
- output, out_orig_shape = output.view(-1,
209
- output.shape[-1]), output.shape
210
- buffer = torch.zeros((x.shape[0], self.lora_a_stacked.shape[2]),
211
- dtype=torch.float32,
212
- device=x.device)
213
- self.punica_wrapper.add_shrink(buffer, x, self.lora_a_stacked, 1.0)
214
- buffer = tensor_model_parallel_all_gather(buffer)
215
- self.punica_wrapper.add_expand(output,
216
- buffer,
217
- self.lora_b_stacked,
218
- self.bias_stacked,
219
- add_input=True)
220
- # now have column partitioned output
221
- output = output.view(*out_orig_shape)
222
- return output
183
+ def apply(self,
184
+ x: torch.Tensor,
185
+ bias: Optional[torch.Tensor] = None) -> torch.Tensor:
186
+ return _mcp_apply(x, bias, self)
187
188
@classmethod
189
@_fully_sharded_can_replace
@@ -260,8 +224,9 @@ class MergedQKVParallelLinearWithShardedLora(MergedQKVParallelLinearWithLora):
224
]
225
return lora_a
226
263
- def apply(self, x: torch.Tensor,
264
- bias: Optional[torch.Tensor]) -> torch.Tensor:
227
+ def apply(self,
228
+ x: torch.Tensor,
229
+ bias: Optional[torch.Tensor] = None) -> torch.Tensor:
230
return _mcp_apply(x, bias, self)
231
232
@classmethod
@@ -294,7 +259,7 @@ class RowParallelLinearWithShardedLoRA(RowParallelLinearWithLoRA):
259
"""
260
261
def slice_lora_b(self, lora_b: torch.Tensor) -> torch.Tensor:
297
- shard_size = self.lora_b_stacked.shape[2]
262
+ shard_size = self.lora_b_stacked[0].shape[2]
263
start_idx = self.tp_rank * shard_size
264
end_idx = (self.tp_rank + 1) * shard_size
265
lora_b = lora_b[:, start_idx:end_idx]
@@ -303,20 +268,24 @@ class RowParallelLinearWithShardedLoRA(RowParallelLinearWithLoRA):
268
def slice_bias(self, bias: torch.Tensor) -> torch.Tensor:
269
if bias is None:
270
return bias
306
- shard_size = self.bias_stacked.shape[2]
271
+ self.lora_bias_stacked = cast(Tuple[torch.Tensor, ...],
272
+ self.lora_bias_stacked)
273
+ shard_size = self.lora_bias_stacked[0].shape[2]
274
start_idx = self.tp_rank * shard_size
275
end_idx = (self.tp_rank + 1) * shard_size
276
bias = bias[start_idx:end_idx]
277
return bias
278
312
- def apply(self, x: torch.Tensor) -> torch.Tensor:
279
+ def apply(self,
280
+ x: torch.Tensor,
281
+ bias: Optional[torch.Tensor] = None) -> torch.Tensor:
282
output = self.base_layer.quant_method.apply(self.base_layer, x)
283
284
x = x.view(-1, x.shape[-1])
285
output, out_orig_shape = output.view(-1,
286
output.shape[-1]), output.shape
287
buffer = torch.zeros(
319
- (x.shape[0], self.lora_a_stacked.shape[2]),
288
+ (self.n_slices, x.shape[0], self.lora_a_stacked[0].shape[2]),
289
dtype=torch.float32,
290
device=x.device,
291
)
@@ -330,12 +299,18 @@ class RowParallelLinearWithShardedLoRA(RowParallelLinearWithLoRA):
299
# remains is a standard all_reduce. User should be aware though that
300
# the output is not the same as a normal row_parallel, it should be
301
# reduced before being used
333
- shard_size = self.lora_b_stacked.shape[2]
334
- start_idx = self.tp_rank * shard_size
335
- self.punica_wrapper.add_expand_slice(output, buffer,
336
- self.lora_b_stacked,
337
- self.bias_stacked, start_idx,
338
- shard_size)
302
+ # NOTE offset are based on the rank.
303
+ shard_size = self.lora_b_stacked[0].shape[2]
304
+ offset_start = self.tp_rank * shard_size
305
+ self.punica_wrapper.add_expand(
306
+ output,
307
+ buffer,
308
+ self.lora_b_stacked,
309
+ self.lora_bias_stacked,
310
+ self.output_slices,
311
+ offset_start=offset_start,
312
+ add_input=True,
313
+ )
314
output = output.view(*out_orig_shape)
315
return output
316
vllm/lora/layers.py
+193
-345
@@ -1,7 +1,7 @@
1
# pylint: disable=unused-argument
2
import math
3
from dataclasses import dataclass
4
-from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union
4
+from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union, cast
5
6
import torch
7
import torch.nn as nn
@@ -18,11 +18,14 @@ from vllm.distributed import (get_tensor_model_parallel_rank,
18
tensor_model_parallel_gather)
19
from vllm.distributed.utils import divide
20
from vllm.lora.punica import PunicaWrapper
21
+# yapf: disable
22
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
23
+ LinearBase,
24
MergedColumnParallelLinear,
25
QKVParallelLinear,
26
ReplicatedLinear,
27
RowParallelLinear)
28
+# yapf: enable
29
from vllm.model_executor.layers.logits_processor import LogitsProcessor
30
from vllm.model_executor.layers.rotary_embedding import (
31
LinearScalingRotaryEmbedding, RotaryEmbedding)
@@ -249,13 +252,10 @@ class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA):
252
full_lora_a_embeddings.shape[1],
253
-1,
254
)
252
-
253
- # Embedding layer only need expand op
254
- self.punica_wrapper.add_expand(full_output,
255
- full_lora_a_embeddings,
256
- self.lora_b_stacked,
257
- bias_all=None,
258
- add_input=True)
255
+ self.punica_wrapper.add_lora_embedding(full_output,
256
+ full_lora_a_embeddings,
257
+ self.lora_b_stacked,
258
+ add_input=True)
259
return full_output.view_as(full_output_org)
260
261
@classmethod
@@ -269,14 +269,19 @@ class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA):
269
return type(source_layer) is VocabParallelEmbedding
270
271
272
-class ReplicatedLinearWithLoRA(BaseLayerWithLoRA):
272
+class BaseLinearLayerWithLoRA(BaseLayerWithLoRA):
273
274
- def __init__(self, base_layer: ReplicatedLinear) -> None:
274
+ def __init__(self, base_layer: LinearBase):
275
super().__init__()
276
self.base_layer = base_layer
277
self.input_size = self.base_layer.input_size
278
- self.output_size = self.base_layer.output_size
278
self.device = _get_lora_device(self.base_layer)
279
+ self.lora_bias_stacked: Optional[Tuple[torch.Tensor, ...]] = None
280
+
281
+ self.output_slices: Tuple[int, ...]
282
+ self.tp_size: int
283
+ self.output_size: int
284
+ self.n_slices: int
285
286
def create_lora_weights(
287
self,
@@ -285,39 +290,64 @@ class ReplicatedLinearWithLoRA(BaseLayerWithLoRA):
290
model_config: Optional[PretrainedConfig] = None,
291
) -> None:
292
self.lora_config = lora_config
288
- lora_a_output_size = lora_config.max_lora_rank
289
- self.lora_a_stacked = torch.zeros(
290
- max_loras,
291
- 1,
292
- lora_a_output_size,
293
- self.input_size,
294
- dtype=lora_config.lora_dtype,
295
- device=self.device,
296
- )
297
- self.lora_b_stacked = torch.zeros(
298
- max_loras,
299
- 1,
300
- self.output_size,
301
- lora_config.max_lora_rank,
302
- dtype=lora_config.lora_dtype,
303
- device=self.device,
304
- )
305
- if lora_config.bias_enabled:
306
- self.bias_stacked = torch.zeros(
293
+ #
294
+ if isinstance(self.base_layer, ReplicatedLinear):
295
+ lora_a_out_size = lora_config.max_lora_rank
296
+ lora_b_out_size = self.output_size
297
+
298
+ elif isinstance(self.base_layer, ColumnParallelLinear):
299
+ lora_a_out_size = (lora_config.max_lora_rank if
300
+ not lora_config.fully_sharded_loras else divide(
301
+ lora_config.max_lora_rank, self.tp_size))
302
+ lora_b_out_size = self.output_size
303
+
304
+ elif isinstance(self.base_layer, RowParallelLinear):
305
+ lora_a_out_size = lora_config.max_lora_rank
306
+ lora_b_out_size = (self.output_size if
307
+ not lora_config.fully_sharded_loras else divide(
308
+ self.output_size, self.tp_size))
309
+ else:
310
+ raise NotImplementedError
311
+
312
+ self.lora_a_stacked = tuple(
313
+ torch.zeros(
314
max_loras,
315
1,
309
- self.output_size,
316
+ lora_a_out_size,
317
+ self.input_size,
318
dtype=lora_config.lora_dtype,
319
device=self.device,
312
- )
313
- else:
314
- self.bias_stacked = None
320
+ ) for _ in range(self.n_slices))
321
+ self.lora_b_stacked = tuple(
322
+ torch.zeros(
323
+ max_loras,
324
+ 1,
325
+ lora_b_out_size,
326
+ lora_config.max_lora_rank,
327
+ dtype=lora_config.lora_dtype,
328
+ device=self.device,
329
+ ) for _ in range(self.n_slices))
330
+ if lora_config.bias_enabled:
331
+ lora_bias_out_size = lora_b_out_size
332
+ self.lora_bias_stacked = tuple(
333
+ torch.zeros(
334
+ max_loras,
335
+ 1,
336
+ lora_bias_out_size,
337
+ dtype=lora_config.lora_dtype,
338
+ device=self.device,
339
+ ) for _ in range(self.n_slices))
340
+ self.output_slices = (self.lora_b_stacked[0].shape[2], )
341
342
def reset_lora(self, index: int):
317
- self.lora_a_stacked[index] = 0
318
- self.lora_b_stacked[index] = 0
319
- if self.lora_config.bias_enabled:
320
- self.bias_stacked[index] = 0
343
+ for s_index in range(self.n_slices):
344
+ self.lora_a_stacked[s_index][index] = 0
345
+ self.lora_b_stacked[s_index][index] = 0
346
+ if self.lora_config.bias_enabled:
347
+ # Make mypy happy
348
+ self.lora_bias_stacked = cast(Tuple[torch.Tensor, ...],
349
+ self.lora_bias_stacked)
350
+ self.lora_bias_stacked[s_index][index] = 0
351
352
def set_lora(
353
self,
@@ -325,29 +355,56 @@ class ReplicatedLinearWithLoRA(BaseLayerWithLoRA):
355
lora_a: torch.Tensor,
356
lora_b: torch.Tensor,
357
embeddings_tensor: Optional[torch.Tensor],
328
- bias: Optional[torch.Tensor] = None,
358
+ lora_bias: Optional[torch.Tensor] = None,
359
):
330
- self.reset_lora(index)
360
+ # Except for QKVParallelLinearWithLora and
361
+ # MergedColumnParallelLinearWithLoRA, all other linear LoRA layers
362
+ # store weights in a tuple of size 1. These two layers will
363
+ # override this function.
364
+ assert (len(self.lora_a_stacked) == len(self.lora_b_stacked) ==
365
+ self.n_slices == 1)
366
332
- self.lora_a_stacked[index,
333
- 0, :lora_a.shape[1], :lora_a.shape[0]].copy_(
334
- lora_a.T, non_blocking=True)
335
- self.lora_b_stacked[index,
336
- 0, :lora_b.shape[1], :lora_b.shape[0]].copy_(
337
- lora_b.T, non_blocking=True)
338
- if bias is not None:
339
- self.bias_stacked[index,
340
- 0, :bias.shape[0]].copy_(bias.T,
341
- non_blocking=True)
342
-
343
- def apply(self, x: torch.Tensor,
344
- bias: Optional[torch.Tensor]) -> torch.Tensor:
367
+ self.reset_lora(index)
368
+ if self.tp_size > 1:
369
+ lora_a = self.slice_lora_a(lora_a)
370
+ lora_b = self.slice_lora_b(lora_b)
371
+ if lora_bias is not None:
372
+ lora_bias = self.slice_bias(lora_bias)
373
+
374
+ self.lora_a_stacked[0][index,
375
+ 0, :lora_a.shape[1], :lora_a.shape[0]].copy_(
376
+ lora_a.T, non_blocking=True)
377
+ self.lora_b_stacked[0][index,
378
+ 0, :lora_b.shape[1], :lora_b.shape[0]].copy_(
379
+ lora_b.T, non_blocking=True)
380
+ if lora_bias is not None:
381
+
382
+ self.lora_bias_stacked = cast(Tuple[torch.Tensor, ...],
383
+ self.lora_bias_stacked)
384
+ assert len(self.lora_bias_stacked)
385
+ self.lora_bias_stacked[0][index, 0, :lora_bias.shape[0]].copy_(
386
+ lora_bias.T, non_blocking=True)
387
+
388
+ def apply(self,
389
+ x: torch.Tensor,
390
+ bias: Optional[torch.Tensor] = None) -> torch.Tensor:
391
output = self.base_layer.quant_method.apply(self.base_layer, x, bias)
346
- self.punica_wrapper.add_lora(output, x, self.lora_a_stacked,
347
- self.lora_b_stacked, self.bias_stacked,
348
- 1.0)
392
+ self.punica_wrapper.add_lora_linear(output, x, self.lora_a_stacked,
393
+ self.lora_b_stacked,
394
+ self.lora_bias_stacked, 1.0,
395
+ self.output_slices)
396
return output
397
398
+
399
+class ReplicatedLinearWithLoRA(BaseLinearLayerWithLoRA):
400
+
401
+ def __init__(self, base_layer: ReplicatedLinear) -> None:
402
+ super().__init__(base_layer, )
403
+ # To ensure interface compatibility, set to 1 always.
404
+ self.tp_size = 1
405
+ self.output_size = self.base_layer.output_size
406
+ self.n_slices = 1
407
+
408
def forward(self, input_):
409
"""Forward of ReplicatedLinearWithLoRA
410
@@ -380,73 +437,26 @@ class ReplicatedLinearWithLoRA(BaseLayerWithLoRA):
437
return type(source_layer) is ReplicatedLinear
438
439
383
-class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
440
+class ColumnParallelLinearWithLoRA(BaseLinearLayerWithLoRA):
441
"""
442
LoRA on top of ColumnParallelLinear layer.
386
-
443
LoRA B is sliced for tensor parallelism.
444
+ There are two types for the `base_layer`:
445
+ 1. ColumnParallelLinear, e.g.`dense_h_to_4h` in `FalconForCausalLM`.
446
+ 2. MergedColumnParallelLinear, e.g.`gate_up_proj` in `Phi3ForCausalLM`.
447
"""
448
449
def __init__(self, base_layer: ColumnParallelLinear) -> None:
391
- super().__init__()
450
+ super().__init__(base_layer)
451
# The base_layer type is ColumnParallelLinear or
452
# MergedColumnParallelLinear, their weight sharding logic is
453
# inconsistent when TP is greater than 1.
454
self.is_merged_col_linear = type(
455
base_layer) is MergedColumnParallelLinear
397
-
398
- self.base_layer = base_layer
456
self.tp_size = get_tensor_model_parallel_world_size()
400
- self.input_size = self.base_layer.input_size
457
self.output_size = self.base_layer.output_size_per_partition
402
- self.device = _get_lora_device(self.base_layer)
403
-
404
- def create_lora_weights(
405
- self,
406
- max_loras: int,
407
- lora_config: LoRAConfig,
408
- model_config: Optional[PretrainedConfig] = None,
409
- ) -> None:
410
- self.lora_config = lora_config
411
- self.tp_size = get_tensor_model_parallel_world_size()
412
- lora_a_output_size_per_partition = (
413
- lora_config.max_lora_rank if not lora_config.fully_sharded_loras
414
- else divide(lora_config.max_lora_rank, self.tp_size))
415
- self.lora_a_stacked = torch.zeros(
416
- max_loras,
417
- 1,
418
- lora_a_output_size_per_partition,
419
- self.input_size,
420
- dtype=lora_config.lora_dtype,
421
- device=self.device,
422
- )
423
- self.lora_b_stacked = torch.zeros(
424
- max_loras,
425
- 1,
426
- self.output_size,
427
- lora_config.max_lora_rank,
428
- dtype=lora_config.lora_dtype,
429
- device=self.device,
430
- )
431
-
432
- if lora_config.bias_enabled:
433
- self.bias_stacked = torch.zeros(
434
- max_loras,
435
- 1,
436
- self.output_size,
437
- dtype=lora_config.lora_dtype,
438
- device=self.device,
439
- )
440
- else:
441
- self.bias_stacked = None
442
-
443
- self.output_dim = self.lora_b_stacked.shape[2]
444
-
445
- def reset_lora(self, index: int):
446
- self.lora_a_stacked[index] = 0
447
- self.lora_b_stacked[index] = 0
448
- if self.lora_config.bias_enabled:
449
- self.bias_stacked[index] = 0
458
+ # There is only one LoRA layer
459
+ self.n_slices = 1
460
461
def slice_lora_a(self, lora_a: torch.Tensor) -> torch.Tensor:
462
return lora_a
@@ -485,40 +495,6 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
495
bias = bias[start_idx:end_idx]
496
return bias
497
488
- def set_lora(
489
- self,
490
- index: int,
491
- lora_a: torch.Tensor,
492
- lora_b: torch.Tensor,
493
- embeddings_tensor: Optional[torch.Tensor],
494
- bias: Optional[torch.Tensor] = None,
495
- ):
496
- self.reset_lora(index)
497
-
498
- if self.tp_size > 1:
499
- lora_a = self.slice_lora_a(lora_a)
500
- lora_b = self.slice_lora_b(lora_b)
501
- bias = self.slice_bias(bias)
502
-
503
- self.lora_a_stacked[index,
504
- 0, :lora_a.shape[1], :lora_a.shape[0]].copy_(
505
- lora_a.T, non_blocking=True)
506
- self.lora_b_stacked[index,
507
- 0, :lora_b.shape[1], :lora_b.shape[0]].copy_(
508
- lora_b.T, non_blocking=True)
509
- if bias is not None:
510
- self.bias_stacked[index,
511
- 0, :bias.shape[0]].copy_(bias.T,
512
- non_blocking=True)
513
-
514
- def apply(self, x: torch.Tensor,
515
- bias: Optional[torch.Tensor]) -> torch.Tensor:
516
- output = self.base_layer.quant_method.apply(self.base_layer, x, bias)
517
- self.punica_wrapper.add_lora(output, x, self.lora_a_stacked,
518
- self.lora_b_stacked, self.bias_stacked,
519
- 1.0)
520
- return output
521
-
498
def forward(self, input_):
499
"""Forward of ColumnParallelLinear
500
@@ -568,6 +544,8 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
544
545
def __init__(self, base_layer: MergedColumnParallelLinear) -> None:
546
super().__init__(base_layer)
547
+ # There are two LoRA layers
548
+ self.n_slices = len(self.base_layer.output_sizes)
549
550
def create_lora_weights(
551
self,
@@ -575,9 +553,13 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
553
lora_config: LoRAConfig,
554
model_config: Optional[PretrainedConfig] = None,
555
) -> None:
556
+ """
557
+ The main reason for overriding this function is to enhance code
558
+ maintainability.
559
+ """
560
self.lora_config = lora_config
579
- n_slices = 2
580
- if not (len(self.base_layer.output_sizes) == n_slices
561
+
562
+ if not (len(self.base_layer.output_sizes) == self.n_slices == 2
563
and self.base_layer.output_sizes[0]
564
== self.base_layer.output_sizes[1]):
565
raise ValueError(
@@ -598,7 +580,7 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
580
self.input_size,
581
dtype=lora_config.lora_dtype,
582
device=self.device,
601
- ) for _ in range(n_slices))
583
+ ) for _ in range(self.n_slices))
584
self.lora_b_stacked = tuple(
585
torch.zeros(
586
max_loras,
@@ -607,30 +589,19 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
589
lora_config.max_lora_rank,
590
dtype=lora_config.lora_dtype,
591
device=self.device,
610
- ) for _ in range(n_slices))
592
+ ) for _ in range(self.n_slices))
593
if lora_config.bias_enabled:
612
- self.bias_stacked = tuple(
594
+ self.lora_bias_stacked = tuple(
595
torch.zeros(
596
max_loras,
597
1,
598
self.output_size // 2,
599
dtype=lora_config.lora_dtype,
600
device=self.device,
619
- ) for _ in range(n_slices))
620
- else:
621
- self.bias_stacked = None
601
+ ) for _ in range(self.n_slices))
602
self.output_dim = self.lora_b_stacked[0].shape[2]
603
self.output_slices = (self.output_dim, self.output_dim)
604
625
- def reset_lora(self, index: int):
626
- self.lora_a_stacked[0][index] = 0
627
- self.lora_a_stacked[1][index] = 0
628
- self.lora_b_stacked[0][index] = 0
629
- self.lora_b_stacked[1][index] = 0
630
- if self.lora_config.bias_enabled:
631
- self.bias_stacked[0][index] = 0
632
- self.bias_stacked[1][index] = 0
633
-
605
def slice_lora_a(
606
self, lora_a: List[Union[torch.Tensor, None]]
607
) -> List[Union[torch.Tensor, None]]:
@@ -668,15 +639,15 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
639
lora_a: torch.Tensor,
640
lora_b: torch.Tensor,
641
embeddings_tensor: Optional[torch.Tensor],
671
- bias: Optional[torch.Tensor] = None,
642
+ lora_bias: Optional[torch.Tensor] = None,
643
):
644
self.reset_lora(index)
645
646
if self.tp_size > 1:
647
lora_a = self.slice_lora_a(lora_a)
648
lora_b = self.slice_lora_b(lora_b)
678
- if bias is not None:
679
- bias = self.slice_bias(bias)
649
+ if lora_bias is not None:
650
+ lora_bias = self.slice_bias(lora_bias)
651
652
if lora_a[0] is not None:
653
self.lora_a_stacked[0][
@@ -685,10 +656,11 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
656
self.lora_b_stacked[0][
657
index, 0, :lora_b[0].shape[1], :lora_b[0].shape[0]].copy_(
658
lora_b[0].T, non_blocking=True)
688
- if bias is not None and bias[0] is not None:
689
- self.bias_stacked[0][index,
690
- 0, :bias[0].shape[0]].copy_(bias[0].T,
691
- non_blocking=True)
659
+ if lora_bias is not None and lora_bias[0] is not None:
660
+ self.lora_bias_stacked = cast(Tuple[torch.Tensor, ...],
661
+ self.lora_bias_stacked)
662
+ self.lora_bias_stacked[0][index, 0, :lora_bias[0].shape[0]].copy_(
663
+ lora_bias[0].T, non_blocking=True)
664
if lora_a[1] is not None:
665
self.lora_a_stacked[1][
666
index, 0, :lora_a[1].shape[1], :lora_a[1].shape[0]].copy_(
@@ -696,18 +668,11 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
668
self.lora_b_stacked[1][
669
index, 0, :lora_b[1].shape[1], :lora_b[1].shape[0]].copy_(
670
lora_b[1].T, non_blocking=True)
699
- if bias is not None and bias[1] is not None:
700
- self.bias_stacked[1][index,
701
- 0, :bias[1].shape[0]].copy_(bias[1].T,
702
- non_blocking=True)
703
-
704
- def apply(self, x: torch.Tensor,
705
- bias: Optional[torch.Tensor]) -> torch.Tensor:
706
- output = self.base_layer.quant_method.apply(self.base_layer, x, bias)
707
- self.punica_wrapper.add_lora_packed_nslice(
708
- output, x, self.lora_a_stacked, self.lora_b_stacked,
709
- self.bias_stacked, 1.0, (self.output_dim, self.output_dim))
710
- return output
671
+ if lora_bias is not None and lora_bias[1] is not None:
672
+ self.lora_bias_stacked = cast(Tuple[torch.Tensor, ...],
673
+ self.lora_bias_stacked)
674
+ self.lora_bias_stacked[1][index, 0, :lora_bias[1].shape[0]].copy_(
675
+ lora_bias[1].T, non_blocking=True)
676
677
@classmethod
678
@_not_fully_sharded_can_replace
@@ -737,7 +702,6 @@ class QKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
702
703
def __init__(self, base_layer: QKVParallelLinear) -> None:
704
super().__init__(base_layer)
740
- self.tp_size = get_tensor_model_parallel_world_size()
705
self.q_proj_total_size = (self.base_layer.total_num_heads *
706
self.base_layer.head_size)
707
self.q_proj_shard_size = (self.base_layer.num_heads *
@@ -746,6 +710,8 @@ class QKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
710
self.base_layer.head_size)
711
self.kv_proj_total_size = (self.base_layer.total_num_kv_heads *
712
self.base_layer.head_size)
713
+ # There is only one LoRA layer
714
+ self.n_slices = 1
715
716
def slice_lora_b(self, lora_b: torch.Tensor) -> torch.Tensor:
717
tp_rank = get_tensor_model_parallel_rank()
@@ -780,32 +746,6 @@ class QKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
746
bias = torch.cat([bias_q, bias_k, bias_v], dim=1)
747
return bias
748
783
- def set_lora(
784
- self,
785
- index: int,
786
- lora_a: torch.Tensor,
787
- lora_b: torch.Tensor,
788
- embeddings_tensor: Optional[torch.Tensor],
789
- bias: Optional[torch.Tensor] = None,
790
- ):
791
- self.reset_lora(index)
792
- if self.tp_size > 1:
793
- lora_a = self.slice_lora_a(lora_a)
794
- lora_b = self.slice_lora_b(lora_b)
795
- if bias is not None:
796
- bias = self.slice_bias(bias)
797
-
798
- self.lora_a_stacked[index,
799
- 0, :lora_a.shape[1], :lora_a.shape[0]].copy_(
800
- lora_a.T, non_blocking=True)
801
- self.lora_b_stacked[index,
802
- 0, :lora_b.shape[1], :lora_b.shape[0]].copy_(
803
- lora_b.T, non_blocking=True)
804
- if bias is not None:
805
- self.bias_stacked[index,
806
- 0, :bias.shape[0]].copy_(bias.T,
807
- non_blocking=True)
808
-
749
@classmethod
750
@_not_fully_sharded_can_replace
751
def can_replace_layer(cls, source_layer: nn.Module,
@@ -828,6 +768,10 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
768
769
def __init__(self, base_layer: QKVParallelLinear) -> None:
770
super().__init__(base_layer)
771
+ # There are three LoRA layer.
772
+ self.n_slices = len(self.base_layer.output_sizes)
773
+ self.tp_size = get_tensor_model_parallel_world_size()
774
+ self.tp_rank = get_tensor_model_parallel_rank()
775
776
def create_lora_weights(
777
self,
@@ -835,9 +779,16 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
779
lora_config: LoRAConfig,
780
model_config: Optional[PretrainedConfig] = None,
781
) -> None:
782
+ """
783
+ The main reason for overloading this function is to handle inconsistent
784
+ weight dimensions in qkv lora.
785
+ """
786
self.lora_config = lora_config
839
- self.tp_size = get_tensor_model_parallel_world_size()
840
- self.tp_rank = get_tensor_model_parallel_rank()
787
+
788
+ if not (len(self.base_layer.output_sizes) == self.n_slices == 3):
789
+ raise ValueError(
790
+ "LoRAColumnParallelLinear3Slice requires 3 slices.")
791
+
792
self.q_proj_shard_size = (self.base_layer.num_heads *
793
self.base_layer.head_size)
794
self.kv_proj_shard_size = (self.base_layer.num_kv_heads *
@@ -902,7 +853,7 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
853
),
854
)
855
if lora_config.bias_enabled:
905
- self.bias_stacked = (
856
+ self.lora_bias_stacked = (
857
torch.zeros(
858
max_loras,
859
1,
@@ -925,9 +876,6 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
876
device=self.device,
877
),
878
)
928
- else:
929
- self.bias_stacked = None
930
-
879
self.output_slices = (
880
self.q_proj_shard_size,
881
self.kv_proj_shard_size,
@@ -939,18 +887,6 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
887
self.indices: torch.Tensor
888
self.indices_len: List[int]
889
942
- def reset_lora(self, index: int):
943
- self.lora_a_stacked[0][index] = 0
944
- self.lora_b_stacked[0][index] = 0
945
- self.lora_a_stacked[1][index] = 0
946
- self.lora_b_stacked[1][index] = 0
947
- self.lora_a_stacked[2][index] = 0
948
- self.lora_b_stacked[2][index] = 0
949
- if self.lora_config.bias_enabled:
950
- self.bias_stacked[0][index] = 0
951
- self.bias_stacked[1][index] = 0
952
- self.bias_stacked[2][index] = 0
953
-
890
def slice_lora_a(
891
self, lora_a: List[Union[torch.Tensor, None]]
892
) -> List[Union[torch.Tensor, None]]:
@@ -1000,15 +936,15 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
936
lora_a: torch.Tensor,
937
lora_b: torch.Tensor,
938
embeddings_tensor: Optional[torch.Tensor],
1003
- bias: Optional[torch.Tensor] = None,
939
+ lora_bias: Optional[torch.Tensor] = None,
940
):
941
self.reset_lora(index)
942
943
if self.tp_size > 1:
944
lora_a = self.slice_lora_a(lora_a)
945
lora_b = self.slice_lora_b(lora_b)
1010
- if bias is not None:
1011
- bias = self.slice_bias(bias)
946
+ if lora_bias is not None:
947
+ lora_bias = self.slice_bias(lora_bias)
948
949
if lora_b[0] is not None:
950
lora_b_q = lora_b[0]
@@ -1039,26 +975,24 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
975
index, 0, :lora_a[2].shape[1], :lora_a[2].shape[0]].copy_(
976
lora_a[2].T, non_blocking=True)
977
1042
- if bias is not None:
1043
- if bias[0] is not None:
1044
- self.bias_stacked[0][index, 0, :bias[0].shape[0]].copy_(
1045
- bias[0].T, non_blocking=True)
1046
- if bias[1] is not None:
1047
- self.bias_stacked[1][index, 0, :bias[1].shape[0]].copy_(
1048
- bias[1].T, non_blocking=True)
1049
- if bias[2] is not None:
1050
- self.bias_stacked[2][index, 0, :bias[2].shape[0]].copy_(
1051
- bias[2].T, non_blocking=True)
1052
-
1053
- def apply(self, x: torch.Tensor,
1054
- bias: Optional[torch.Tensor]) -> torch.Tensor:
1055
- output = self.base_layer.quant_method.apply(self.base_layer, x, bias)
1056
- self.punica_wrapper.add_lora_packed_nslice(output, x,
1057
- self.lora_a_stacked,
1058
- self.lora_b_stacked,
1059
- self.bias_stacked, 1.0,
1060
- self.output_slices)
1061
- return output
978
+ if lora_bias is not None:
979
+ self.lora_bias_stacked = cast(Tuple[torch.Tensor, ...],
980
+ self.lora_bias_stacked)
981
+ if lora_bias[0] is not None:
982
+ self.lora_bias_stacked[0][index,
983
+ 0, :lora_bias[0].shape[0]].copy_(
984
+ lora_bias[0].T,
985
+ non_blocking=True)
986
+ if lora_bias[1] is not None:
987
+ self.lora_bias_stacked[1][index,
988
+ 0, :lora_bias[1].shape[0]].copy_(
989
+ lora_bias[1].T,
990
+ non_blocking=True)
991
+ if lora_bias[2] is not None:
992
+ self.lora_bias_stacked[2][index,
993
+ 0, :lora_bias[2].shape[0]].copy_(
994
+ lora_bias[2].T,
995
+ non_blocking=True)
996
997
@classmethod
998
@_not_fully_sharded_can_replace
@@ -1073,76 +1007,25 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
1007
and len(packed_modules_list) == 3)
1008
1009
1076
-class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
1010
+class RowParallelLinearWithLoRA(BaseLinearLayerWithLoRA):
1011
1012
def __init__(self, base_layer: RowParallelLinear) -> None:
1079
- super().__init__()
1080
- self.base_layer = base_layer
1013
+ super().__init__(base_layer)
1014
+
1015
+ self.tp_size = get_tensor_model_parallel_world_size()
1016
+ # reset input_size
1017
self.input_size = self.base_layer.input_size_per_partition
1018
self.output_size = self.base_layer.output_size
1083
- self.device = _get_lora_device(self.base_layer)
1019
1085
- def create_lora_weights(
1086
- self,
1087
- max_loras: int,
1088
- lora_config: LoRAConfig,
1089
- model_config: Optional[PretrainedConfig] = None,
1090
- ) -> None:
1091
- self.lora_config = lora_config
1020
self.tp_rank = get_tensor_model_parallel_rank()
1093
- self.lora_a_stacked = torch.zeros(
1094
- (
1095
- max_loras,
1096
- 1,
1097
- lora_config.max_lora_rank,
1098
- self.input_size,
1099
- ),
1100
- dtype=lora_config.lora_dtype,
1101
- device=self.device,
1102
- )
1103
- tp_size = get_tensor_model_parallel_world_size()
1104
- lora_b_output_size_per_partition = (
1105
- self.output_size if not lora_config.fully_sharded_loras else
1106
- divide(self.output_size, tp_size))
1107
-
1108
- self.lora_b_stacked = torch.zeros(
1109
- (
1110
- max_loras,
1111
- 1,
1112
- lora_b_output_size_per_partition,
1113
- lora_config.max_lora_rank,
1114
- ),
1115
- dtype=lora_config.lora_dtype,
1116
- device=self.device,
1117
- )
1118
-
1119
- if lora_config.bias_enabled:
1120
- self.bias_stacked = torch.zeros(
1121
- (
1122
- max_loras,
1123
- 1,
1124
- self.output_size,
1125
- ),
1126
- dtype=lora_config.lora_dtype,
1127
- device=self.device,
1128
- )
1129
- else:
1130
- self.bias_stacked = None
1131
- # Lazily initialized
1132
- self.indices: torch.Tensor
1133
- self.indices_len: List[int]
1134
-
1135
- def reset_lora(self, index: int):
1136
- self.lora_a_stacked[index] = 0
1137
- self.lora_b_stacked[index] = 0
1138
- if self.lora_config.bias_enabled:
1139
- self.bias_stacked[index] = 0
1021
+ # There is only one LoRA layer.
1022
+ self.n_slices = 1
1023
1024
def slice_lora_a(self, lora_a: torch.Tensor) -> torch.Tensor:
1142
- tensor_model_parallel_rank = get_tensor_model_parallel_rank()
1025
+
1026
shard_size = self.input_size
1144
- start_idx = tensor_model_parallel_rank * shard_size
1145
- end_idx = (tensor_model_parallel_rank + 1) * shard_size
1027
+ start_idx = self.tp_rank * shard_size
1028
+ end_idx = (self.tp_rank + 1) * shard_size
1029
lora_a = lora_a[start_idx:end_idx, :]
1030
return lora_a
1031
@@ -1152,40 +1035,6 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
1035
def slice_bias(self, bias: torch.Tensor) -> torch.Tensor:
1036
return bias
1037
1155
- def set_lora(
1156
- self,
1157
- index: int,
1158
- lora_a: torch.Tensor,
1159
- lora_b: torch.Tensor,
1160
- embeddings_tensor: Optional[torch.Tensor],
1161
- bias: Optional[torch.Tensor] = None,
1162
- ):
1163
- self.reset_lora(index)
1164
-
1165
- if self.base_layer.tp_size > 1:
1166
- lora_a = self.slice_lora_a(lora_a)
1167
- lora_b = self.slice_lora_b(lora_b)
1168
- if bias is not None:
1169
- bias = self.slice_bias(bias)
1170
-
1171
- self.lora_a_stacked[index,
1172
- 0, :lora_a.shape[1], :lora_a.shape[0]].copy_(
1173
- lora_a.T, non_blocking=True)
1174
- self.lora_b_stacked[index,
1175
- 0, :lora_b.shape[1], :lora_b.shape[0]].copy_(
1176
- lora_b.T, non_blocking=True)
1177
- if bias is not None:
1178
- self.bias_stacked[index,
1179
- 0, :bias.shape[0]].copy_(bias.T,
1180
- non_blocking=True)
1181
-
1182
- def apply(self, x: torch.Tensor) -> torch.Tensor:
1183
- output = self.base_layer.quant_method.apply(self.base_layer, x)
1184
- self.punica_wrapper.add_lora(output, x, self.lora_a_stacked,
1185
- self.lora_b_stacked, self.bias_stacked,
1186
- 1.0)
1187
- return output
1188
-
1038
def forward(self, input_):
1039
"""Forward of RowParallelLinear
1040
@@ -1203,10 +1052,9 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
1052
input_parallel = input_
1053
else:
1054
# TODO: simplify code below
1206
- tp_rank = get_tensor_model_parallel_rank()
1055
splitted_input = split_tensor_along_last_dim(
1056
input_, num_partitions=self.base_layer.tp_size)
1209
- input_parallel = splitted_input[tp_rank].contiguous()
1057
+ input_parallel = splitted_input[self.tp_rank].contiguous()
1058
1059
# Matrix multiply.
1060
output_parallel = self.apply(input_parallel)
vllm/lora/models.py
+4
-4
@@ -555,17 +555,17 @@ class LoRAModelManager(AdapterModelManager):
555
input_dim,
556
output_dim,
557
rank,
558
- module.lora_a_stacked.dtype,
558
+ module.lora_a_stacked[0].dtype,
559
"cpu",
560
embeddings_tensor_dim=embeddings_tensor_dim,
561
bias_enabled=bias_enabled)
562
else:
563
lora = LoRALayerWeights.create_dummy_lora_weights(
564
module_name,
565
- module.lora_a_stacked.shape[-1],
566
- module.lora_b_stacked.shape[-2],
565
+ module.lora_a_stacked[0].shape[-1],
566
+ module.lora_b_stacked[0].shape[-2],
567
rank,
568
- module.lora_a_stacked.dtype,
568
+ module.lora_a_stacked[0].dtype,
569
"cpu",
570
bias_enabled=bias_enabled,
571
)
vllm/lora/punica.py
+193
-172
@@ -362,7 +362,7 @@ class PunicaWrapper:
362
long_lora_len = self.indices_len[4]
363
return self._long_lora_indices[:long_lora_len]
364
365
- def shrink_prefill(
365
+ def _shrink_prefill(
366
self,
367
y: torch.Tensor,
368
x: torch.Tensor,
@@ -380,7 +380,7 @@ class PunicaWrapper:
380
scale,
381
)
382
383
- def shrink_decode(
383
+ def _shrink_decode(
384
self,
385
y: torch.Tensor,
386
x: torch.Tensor,
@@ -389,7 +389,7 @@ class PunicaWrapper:
389
):
390
bgmv_shrink(x, w_t_all, y, self.token_lora_indices, scale)
391
392
- def expand_prefill(
392
+ def _expand_prefill(
393
self,
394
y: torch.Tensor,
395
x: torch.Tensor,
@@ -407,7 +407,7 @@ class PunicaWrapper:
407
add_input,
408
)
409
410
- def expand_decode(
410
+ def _expand_decode(
411
self,
412
y: torch.Tensor,
413
x: torch.Tensor,
@@ -416,7 +416,7 @@ class PunicaWrapper:
416
):
417
bgmv_expand(x, w_t_all, y, self.token_lora_indices, add_input)
418
419
- def expand_slice_prefill(
419
+ def _expand_slice_prefill(
420
self,
421
y: torch.Tensor,
422
x: torch.Tensor,
@@ -438,7 +438,7 @@ class PunicaWrapper:
438
add_input,
439
)
440
441
- def expand_slice_decode(
441
+ def _expand_slice_decode(
442
self,
443
y: torch.Tensor,
444
x: torch.Tensor,
@@ -450,41 +450,35 @@ class PunicaWrapper:
450
bgmv_expand_slice(x, w_t_all, y, self.token_lora_indices, y_offset,
451
y_slice_size, add_input)
452
453
- def apply_bias(
454
- self,
455
- indices: torch.Tensor,
456
- output: torch.Tensor,
457
- bias_stacked: torch.Tensor,
458
- ):
459
- """Applies bias to output
460
-
461
- Input shapes:
462
- bias_stacked: (num_loras, output_dim)
463
- indices: (batch_size)
464
- output: (batch_size, output_dim)
453
+ def _apply_expand(self,
454
+ y: torch.Tensor,
455
+ x: torch.Tensor,
456
+ w_t_all: torch.Tensor,
457
+ y_offset: Optional[int],
458
+ y_slice_size: Optional[int],
459
+ add_input: bool = True):
460
+ """
461
+ Perform the ` y[:,y_offset:y_offset+y_slice_size]+=x@w_t_all`
462
+ computation, which is suitable for the
463
+ GEMM of lora'b.
464
"""
466
- org_output = output
467
- output = output.view(-1, output.shape[-1])
468
- indices = indices.view(-1)
469
-
470
- bias_stacked = bias_stacked.view(-1, bias_stacked.shape[-1])
471
- bias_stacked = bias_stacked[indices]
472
- bias_stacked[indices == -1] = 0
473
- output += bias_stacked
465
475
- return output.view_as(org_output)
466
+ expand_slice_fun: Callable = (self._expand_slice_prefill
467
+ if self.is_prefill else
468
+ self._expand_slice_decode)
469
+ expand_slice_fun(y, x, w_t_all, y_offset, y_slice_size, add_input)
470
477
- def apply_bias_packed_nslice(
471
+ def _apply_bias(
472
self,
473
indices: torch.Tensor,
474
output: torch.Tensor,
475
output_slices: Tuple[int, ...],
482
- bias_stacked: Tuple[Optional[torch.Tensor], ...],
476
+ lora_bias_stacked: Tuple[Optional[torch.Tensor], ...],
477
):
478
"""Applies bias to output
479
480
Input shapes:
487
- bias_stacked: 3 element tuple of (num_loras, output_dim)
481
+ lora_bias_stacked: 3 element tuple of (num_loras, output_dim)
482
indices: (batch_size)
483
output: (batch_size, q_slice_size + 2*kv_slice_size)
484
output_slices: n-1 element tuple of (slice_size...),
@@ -496,7 +490,7 @@ class PunicaWrapper:
490
491
offset_left = 0
492
for slice_idx, slice in enumerate(output_slices):
499
- bias = bias_stacked[slice_idx]
493
+ bias = lora_bias_stacked[slice_idx]
494
if bias is not None:
495
bias = bias.view(-1, bias.shape[-1])
496
bias = bias[indices]
@@ -506,7 +500,7 @@ class PunicaWrapper:
500
501
return output.view_as(org_output)
502
509
- def add_shrink(
503
+ def _apply_shrink(
504
self,
505
y: torch.Tensor,
506
x: torch.Tensor,
@@ -517,188 +511,215 @@ class PunicaWrapper:
511
Perform the ` y+=x@w_t_all` computation, which is suitable for the
512
GEMM of lora'a.
513
When `is_prefill is` true, it indicates that it is currently the
520
- prefill stage, and the `shrink_prefill` function should be called.
521
- Otherwise, it is the decode stage, and the shrink_decode function
514
+ prefill stage, and the `_shrink_prefill` function should be called.
515
+ Otherwise, it is the decode stage, and the _shrink_decode function
516
should be called.
517
"""
524
- shrink_fun: Callable = (self.shrink_prefill
525
- if self.is_prefill else self.shrink_decode)
518
+ y_org = y
519
+ y = y.view(-1, y.shape[-1])
520
+ shrink_fun: Callable = (self._shrink_prefill
521
+ if self.is_prefill else self._shrink_decode)
522
shrink_fun(y, x, w_t_all, scale)
523
+ y = y.view_as(y_org)
524
528
- def add_expand(
525
+ def add_shrink(
526
self,
530
- y: torch.Tensor,
527
+ y: Union[Tuple[torch.Tensor, ...], torch.Tensor],
528
x: torch.Tensor,
532
- w_t_all: torch.Tensor,
533
- bias_all: Optional[torch.Tensor],
534
- add_input: bool = True,
529
+ lora_a_stacked: Tuple[torch.Tensor, ...],
530
+ scale: float,
531
):
532
"""
537
- Perform the ` y+=x@w_t_all+bias` computation, which is suitable for the
538
- GEMM of lora'b.
539
- When `is_prefill` is true, it indicates that it is currently the
540
- prefill stage, and the `expand_prefill` function should be called.
541
- Otherwise, it is the decode stage, and the expand_decode function
533
+ Performs GEMM for multiple slices of lora_a.
534
+ When `is_prefill is` true, it indicates that it is currently the
535
+ prefill stage, and the `_shrink_prefill` function should be called.
536
+ Otherwise, it is the decode stage, and the _shrink_decode function
537
should be called.
543
- """
544
- if bias_all is not None:
545
- y = self.apply_bias(self.token_lora_indices, y, bias_all)
546
-
547
- expand_fun: Callable = (self.expand_prefill
548
- if self.is_prefill else self.expand_decode)
549
- expand_fun(y, x, w_t_all, add_input)
550
-
551
- def add_expand_slice(self,
552
- y: torch.Tensor,
553
- x: torch.Tensor,
554
- w_t_all: torch.Tensor,
555
- bias_all: Optional[torch.Tensor],
556
- y_offset: Optional[int],
557
- y_slice_size: Optional[int],
558
- add_input: bool = True):
559
- """
560
- Similar to `add_expand`
561
- """
562
- if bias_all is not None:
563
- y = self.apply_bias(self.token_lora_indices, y, bias_all)
538
+
539
+ Semantics:
540
+ for i in range(len(lora_a_stacked)):
541
+ y[i] += (x @ lora_a_stacked[i]) * scale
542
+
543
+ Args:
544
+ y (Union[Tuple[torch.Tensor, ...], torch.Tensor]): Output tensors
545
+ x (torch.Tensor): Input tensor
546
+ lora_a_stacked (Tuple[torch.Tensor, ...]): lora_a's weights
547
+ scale (float): Scaling factor for the operation
548
+ """
549
565
- expand_slice_fun: Callable = (self.expand_slice_prefill
566
- if self.is_prefill else
567
- self.expand_slice_decode)
568
- expand_slice_fun(y, x, w_t_all, y_offset, y_slice_size, add_input)
550
+ x = x.view(-1, x.shape[-1])
551
+ # TODO fuse these kernels
552
+ for slice_idx in range(len(lora_a_stacked)):
553
+ self._apply_shrink(y[slice_idx], x, lora_a_stacked[slice_idx],
554
+ scale)
555
570
- def add_expand_packed_nslice(self, y: torch.Tensor, x: torch.Tensor,
571
- lora_b_stacked: Tuple[torch.Tensor, ...],
572
- bias_stacked: Optional[Tuple[torch.Tensor,
573
- ...]],
574
- scale: float,
575
- output_slices: Tuple[int, ...]) -> None:
576
- """
577
- Similar to `add_expand`
556
+ def add_expand(
557
+ self,
558
+ y: torch.Tensor,
559
+ x: Union[Tuple[torch.Tensor, ...], torch.Tensor],
560
+ lora_b_stacked: Tuple[torch.Tensor, ...],
561
+ lora_bias_stacked: Optional[Tuple[torch.Tensor, ...]],
562
+ output_slices: Tuple[int, ...],
563
+ offset_start: int = 0,
564
+ add_input=True,
565
+ ) -> None:
566
"""
567
+ Performs GEMM and bias addition for multiple slices of lora_b.
568
+
569
+ Semantics:
570
+ for i in range(len(lora_b_stacked)):
571
+ slice = output_slices[i]
572
+ y[:, offset:offset+slice] += x[i] @ lora_b_stacked[i] +
573
+ lora_bias_stacked[i]
574
+ offset += slice
575
+
576
+ Args:
577
+ y (torch.Tensor): Output tensor.
578
+ x (Union[Tuple[torch.Tensor, ...], torch.Tensor]): Input tensors
579
+ lora_b_stacked (Tuple[torch.Tensor, ...]): lora_b's weight
580
+ lora_bias_stacked (Optional[Tuple[torch.Tensor, ...]]):
581
+ bias's weight
582
+ output_slices (Tuple[int, ...]): Every slice's size
583
+ add_input (bool): Defaults to True.
584
+ """
585
y_org = y
586
y = y.view(-1, y.shape[-1])
581
- offset_left = 0
582
- if bias_stacked is not None:
583
- self.apply_bias_packed_nslice(self.token_lora_indices, y,
584
- output_slices, bias_stacked)
587
+ offset_left = offset_start
588
+ if lora_bias_stacked is not None:
589
+ self._apply_bias(self.token_lora_indices, y, output_slices,
590
+ lora_bias_stacked)
591
for slice_idx in range(len(lora_b_stacked)):
586
- self.add_expand_slice(y,
587
- x[slice_idx],
588
- lora_b_stacked[slice_idx],
589
- None,
590
- offset_left,
591
- output_slices[slice_idx],
592
- add_input=True)
592
+ self._apply_expand(
593
+ y,
594
+ x[slice_idx],
595
+ lora_b_stacked[slice_idx],
596
+ offset_left,
597
+ output_slices[slice_idx],
598
+ add_input=add_input,
599
+ )
600
offset_left += output_slices[slice_idx]
594
-
601
y = y.view_as(y_org)
602
597
- def add_lora(self,
598
- y: torch.Tensor,
599
- x: torch.Tensor,
600
- wa_t_all: torch.Tensor,
601
- wb_t_all: torch.Tensor,
602
- bias_all: Optional[torch.Tensor],
603
- scale: float,
604
- y_offset: Optional[int] = None,
605
- y_slice_size: Optional[int] = None,
606
- *,
607
- buffer: Optional[torch.Tensor] = None) -> None:
603
+ def add_lora_embedding(
604
+ self,
605
+ y: torch.Tensor,
606
+ x: torch.Tensor,
607
+ lora_b_stacked: torch.Tensor,
608
+ add_input: bool = True,
609
+ ):
610
+ """
611
+ Applies lora specifically for VocabParallelEmbeddingWithLoRA.
612
+
613
+ Semantics:
614
+ y += x @ lora_b_stacked
615
+
616
+ Args:
617
+ y (torch.Tensor): Output tensor.
618
+ x (torch.Tensor): Input tensor.
619
+ lora_b_stacked (torch.Tensor): lora_b's weights.
620
+ add_input (bool): Default to True.
621
+
622
+ """
623
+
624
+ # Embedding layer only need expand op
625
+ expand_fun: Callable = (self._expand_prefill
626
+ if self.is_prefill else self._expand_decode)
627
+ expand_fun(y, x, lora_b_stacked, add_input)
628
+
629
+ def add_lora_linear(
630
+ self,
631
+ y: torch.Tensor,
632
+ x: torch.Tensor,
633
+ lora_a_stacked: Tuple[torch.Tensor, ...],
634
+ lora_b_stacked: Tuple[torch.Tensor, ...],
635
+ lora_bias_stacked: Optional[Tuple[torch.Tensor, ...]],
636
+ scale: float,
637
+ output_slices: Tuple[int, ...],
638
+ *,
639
+ buffer: Optional[Tuple[torch.Tensor, ...]] = None) -> None:
640
"""
641
+ Applicable to linear-related lora.
642
+
643
Semantics:
610
- y[i] += (
611
- x[i].unsqueeze(0)
612
- @ wa_t_all[indices[i], layer_idx, :, :].transpose(-1, -2)
613
- @ wb_t_all[indices[i], layer_idx, :, :].transpose(-1, -2)
614
- * scale
615
- ).squeeze(0)+bias[i]
644
+ for i in range(len(lora_a_stacked)):
645
+ y[i] += (
646
+ x[i].unsqueeze(0)
647
+ @ lora_a_stacked[indices[i], layer_idx, :, :]
648
+ @ lora_b_stacked[indices[i], layer_idx, :, :]
649
+ * scale
650
+ ).squeeze(0)+lora_bias_stacked[i]
651
+
652
Args:
617
- y (torch.Tensor): Output tensor. Will be changed in-place.
653
+ y (torch.Tensor): Output tensor. Will be changed in-place.
654
x (torch.Tensor): Input tensor
619
- wa_t_all (torch.Tensor): lora_a's weight
620
- wb_t_all (torch.Tensor): lora_b's weight
621
- bias_all: (torch.Tensor): lora's bias
655
+ lora_a_stacked (Tuple[torch.Tensor, ...]): lora_a's weight.
656
+ lora_b_stacked (Tuple[torch.Tensor, ...]): lora_b's weight.
657
+ lora_bias_stacked (Optional[Tuple[torch.Tensor, ...]]): lora's bias.
658
scale (float): Scaling factor.
623
- y_offset (Optional[int], optional): Offset to apply to the starting
624
- column of y.
625
- y_slice_size (Optional[int], optional): Size of the y column slice.
626
- buffer (Optional[torch.Tensor], optional): Defaults to None.
659
+ output_slices (Tuple[int, ...]): Every slice's size.
660
+ buffer (Optional[Tuple[torch.Tensor, ...]]): Defaults to None.
661
"""
628
- y_org = y
629
- y = y.view(-1, y.shape[-1])
630
- x = x.view(-1, x.shape[-1])
631
- r = wb_t_all.size(-1)
662
+
663
+ assert len(lora_a_stacked) == len(lora_b_stacked) == len(output_slices)
664
+ if lora_bias_stacked is not None:
665
+ assert len(lora_bias_stacked) == len(output_slices)
666
+ y = self._apply_bias(self.token_lora_indices, y, output_slices,
667
+ lora_bias_stacked)
668
+
669
if buffer is None:
670
+ r = lora_b_stacked[0].size(-1)
671
# We set the buffer to be float32 by default ,refer to:
672
# https://github.com/triton-lang/triton/issues/1387
635
- buffer = torch.zeros((x.size(0), r),
636
- dtype=torch.float32,
637
- device=x.device)
638
- if bias_all is not None:
639
- y = self.apply_bias(self.token_lora_indices, y, bias_all)
640
- self.add_shrink(buffer, x, wa_t_all, scale)
641
- if y_offset is None and y_slice_size is None:
642
- self.add_expand(y, buffer, wb_t_all, bias_all=None, add_input=True)
643
- else:
644
- self.add_expand_slice(y,
645
- buffer,
646
- wb_t_all,
647
- None,
648
- y_offset,
649
- y_slice_size,
650
- add_input=True)
651
- y = y.view_as(y_org)
652
-
653
- def add_lora_packed_nslice(self, y: torch.Tensor, x: torch.Tensor,
654
- lora_a_stacked: Tuple[torch.Tensor, ...],
655
- lora_b_stacked: Tuple[torch.Tensor, ...],
656
- bias_all: Tuple[Optional[torch.Tensor],
657
- ...], scale: float,
658
- output_slices: Tuple[int, ...]) -> None:
659
- """
660
- Applies lora to each input. Similar to add_lora, This method is
661
- used for layers that are composed of multiple sublayers
662
- (slices) packed together.
663
- """
664
- y_org = y
665
- x = x.view(-1, x.shape[-1])
666
- y = y.view(-1, y.shape[-1])
667
- offset_left = 0
668
- if bias_all is not None:
669
- y = self.apply_bias_packed_nslice(self.token_lora_indices, y,
670
- output_slices, bias_all)
671
- # TODO fuse these kernels
672
- for slice_idx in range(len(output_slices)):
673
- self.add_lora(y, x, lora_a_stacked[slice_idx],
674
- lora_b_stacked[slice_idx], None, scale, offset_left,
675
- output_slices[slice_idx])
676
- offset_left += output_slices[slice_idx]
677
-
678
- y = y.view_as(y_org)
673
+ buffer = tuple(
674
+ torch.zeros(
675
+ (x.size(0), r), dtype=torch.float32, device=x.device)
676
+ for _ in range(len(output_slices)))
677
+ self.add_shrink(buffer, x, lora_a_stacked, scale)
678
+ self.add_expand(y,
679
+ buffer,
680
+ lora_b_stacked,
681
+ None,
682
+ output_slices,
683
+ add_input=True)
684
685
def add_lora_logits(self,
686
y: torch.Tensor,
687
x: torch.Tensor,
683
- wa_t_all: torch.Tensor,
684
- wb_t_all: torch.Tensor,
688
+ lora_a_stacked: torch.Tensor,
689
+ lora_b_stacked: torch.Tensor,
690
scale,
691
*,
692
buffer: Optional[torch.Tensor] = None) -> None:
693
"""
689
- LogitsProcessorWithLoRA always using bgmv
690
- """
694
+ Applies lora specifically for LogitsProcessorWithLoRA.
695
+
696
+ Semantics:
697
+ buffer = (x @ lora_a_stacked) * scale
698
+ y += buffer @ lora_b_stacked
699
+
700
+ Args:
701
+ y (torch.Tensor): Output tensor.
702
+ x (torch.Tensor): Input tensor.
703
+ lora_a_stacked (torch.Tensor): lora_a's weights.
704
+ lora_b_stacked (torch.Tensor):lora_b's weights.
705
+ scale (float): Scaling factor.
706
+ buffer (Optional[torch.Tensor]):Default to None.
707
+ """
708
y_org = y
709
y = y.view(-1, y.shape[-1])
710
x = x.view(-1, x.shape[-1])
694
- r = wb_t_all.size(-1)
711
+ r = lora_b_stacked.size(-1)
712
if buffer is None:
713
# We set the buffer to be float32 by default ,refer to:
714
# https://github.com/triton-lang/triton/issues/1387
715
buffer = torch.zeros((x.size(0), r),
716
dtype=torch.float32,
717
device=x.device)
701
-
702
- bgmv_shrink(x, wa_t_all, buffer, self.sampler_indices, scale)
703
- bgmv_expand(buffer, wb_t_all, y, self.sampler_indices, add_inputs=True)
718
+ # LogitsProcessorWithLoRA always using bgmv.
719
+ bgmv_shrink(x, lora_a_stacked, buffer, self.sampler_indices, scale)
720
+ bgmv_expand(buffer,
721
+ lora_b_stacked,
722
+ y,
723
+ self.sampler_indices,
724
+ add_inputs=True)
725
y = y.view_as(y_org)