@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 571da8fc43
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Misc][LoRA] Clean up the function interface of Punica (#10917)

Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>

Jee Jee Li committed Dec 5, 2024 at 21:22 UTC 571da8fc431ec36427ee1034a7779b23229b015e
5 files changed +497 -631
tests/lora/test_layers.py
+32 -10
@@ -565,7 +565,9 @@ def test_lm_head_logits_processor(dist_init, num_loras, device, vocab_size,
565 @pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
566 @pytest.mark.parametrize("device", CUDA_DEVICES)
567 @pytest.mark.parametrize("stage", STAGES)
568 -def test_linear_replicated(dist_init, num_loras, device, stage) -> None:
568 +@pytest.mark.parametrize("bias_enabled", [True, False])
569 +def test_linear_replicated(dist_init, num_loras, device, stage,
570 + bias_enabled) -> None:
571
572 torch.cuda.set_device(device)
573 torch.set_default_device(device)
@@ -573,7 +575,8 @@ def test_linear_replicated(dist_init, num_loras, device, stage) -> None:
575 max_loras = 8
576 lora_config = LoRAConfig(max_loras=max_loras,
577 max_lora_rank=8,
576 - lora_dtype=torch.float16)
578 + lora_dtype=torch.float16,
579 + bias_enabled=bias_enabled)
580
581 def create_random_linear_replicated_layer():
582
@@ -585,7 +588,12 @@ def test_linear_replicated(dist_init, num_loras, device, stage) -> None:
588 lora_linear = ReplicatedLinearWithLoRA(linear)
589
590 lora_linear.create_lora_weights(max_loras, lora_config)
588 -
591 + assert (lora_linear.n_slices == len(lora_linear.lora_a_stacked) == len(
592 + lora_linear.lora_b_stacked) == 1)
593 + if bias_enabled:
594 + assert len(lora_linear.lora_bias_stacked) == lora_linear.n_slices
595 + else:
596 + assert lora_linear.lora_bias_stacked is None
597 return linear, lora_linear
598
599 for i in range(10):
@@ -669,8 +677,9 @@ def test_linear_replicated(dist_init, num_loras, device, stage) -> None:
677 @pytest.mark.parametrize("fully_shard", [True, False])
678 @pytest.mark.parametrize("device", CUDA_DEVICES)
679 @pytest.mark.parametrize("stage", STAGES)
680 +@pytest.mark.parametrize("bias_enabled", [True, False])
681 def test_linear_parallel(dist_init, num_loras, orientation, fully_shard,
673 - device, stage) -> None:
682 + device, stage, bias_enabled) -> None:
683
684 torch.cuda.set_device(device)
685 torch.set_default_device(device)
@@ -679,7 +688,8 @@ def test_linear_parallel(dist_init, num_loras, orientation, fully_shard,
688 lora_config = LoRAConfig(max_loras=max_loras,
689 max_lora_rank=8,
690 fully_sharded_loras=fully_shard,
682 - lora_dtype=torch.float16)
691 + lora_dtype=torch.float16,
692 + bias_enabled=bias_enabled)
693
694 def create_random_linear_parallel_layer():
695 if orientation == "row":
@@ -700,7 +710,12 @@ def test_linear_parallel(dist_init, num_loras, orientation, fully_shard,
710 if not fully_shard else
711 ColumnParallelLinearWithShardedLoRA(linear))
712 lora_linear.create_lora_weights(max_loras, lora_config)
703 -
713 + assert (lora_linear.n_slices == len(lora_linear.lora_a_stacked) == len(
714 + lora_linear.lora_b_stacked) == 1)
715 + if bias_enabled:
716 + assert len(lora_linear.lora_bias_stacked) == lora_linear.n_slices
717 + else:
718 + assert lora_linear.lora_bias_stacked is None
719 return linear, lora_linear
720
721 for i in range(10):
@@ -784,8 +799,9 @@ def test_linear_parallel(dist_init, num_loras, orientation, fully_shard,
799 @pytest.mark.parametrize("fully_shard", [True, False])
800 @pytest.mark.parametrize("device", CUDA_DEVICES)
801 @pytest.mark.parametrize("stage", STAGES)
802 +@pytest.mark.parametrize("bias_enabled", [True, False])
803 def test_column_parallel_packed(dist_init, num_loras, repeats, fully_shard,
788 - device, stage) -> None:
804 + device, stage, bias_enabled) -> None:
805
806 torch.cuda.set_device(device)
807 torch.set_default_device(device)
@@ -794,7 +810,8 @@ def test_column_parallel_packed(dist_init, num_loras, repeats, fully_shard,
810 lora_config = LoRAConfig(max_loras=max_loras,
811 max_lora_rank=8,
812 fully_sharded_loras=fully_shard,
797 - lora_dtype=torch.float16)
813 + lora_dtype=torch.float16,
814 + bias_enabled=bias_enabled)
815
816 def create_column_parallel_packed_layer():
817 if repeats == 2:
@@ -832,10 +849,16 @@ def test_column_parallel_packed(dist_init, num_loras, repeats, fully_shard,
849 num_key_value_heads = 32
850 num_attention_heads = 32
851
852 + n_slices = repeats
853 lora_linear.create_lora_weights(max_loras,
854 lora_config,
855 model_config=FakeConfig())
838 -
856 + assert (lora_linear.n_slices == len(lora_linear.lora_a_stacked) == len(
857 + lora_linear.lora_b_stacked) == n_slices)
858 + if bias_enabled:
859 + assert len(lora_linear.lora_bias_stacked) == lora_linear.n_slices
860 + else:
861 + assert lora_linear.lora_bias_stacked is None
862 return linear, lora_linear
863
864 for i in range(10):
@@ -911,7 +934,6 @@ def test_column_parallel_packed(dist_init, num_loras, repeats, fully_shard,
934 512,
935 lora_config.lora_extra_vocab_size,
936 )
914 - # lora_linear.set_mapping(*mapping_info)
937
938 lora_result = lora_linear(torch.cat(inputs))[0]
939 expected_result = linear(torch.cat(inputs))[0]
vllm/lora/fully_sharded_layers.py
+75 -100
@@ -1,5 +1,5 @@
1 # pylint: disable=unused-argument
2 -from typing import TYPE_CHECKING, List, Optional, Union
2 +from typing import TYPE_CHECKING, List, Optional, Tuple, Union, cast
3
4 import torch
5 import torch.nn as nn
@@ -32,6 +32,44 @@ def _fully_sharded_can_replace(can_replace):
32 return dec
33
34
35 +def _mcp_apply(x, bias, layer: ColumnParallelLinearWithLoRA):
36 + """
37 + For `ColumnParallelLinearWithLoRA` or classes that inherit from
38 + `ColumnParallelLinearWithLoRA`, they share the same `apply` logic.
39 + """
40 + assert (layer.n_slices == len(layer.lora_a_stacked) == len(
41 + layer.lora_b_stacked) == len(layer.output_slices))
42 + if layer.lora_bias_stacked is not None:
43 + assert layer.n_slices == len(layer.lora_bias_stacked)
44 +
45 + output = layer.base_layer.quant_method.apply(layer.base_layer, x, bias)
46 +
47 + x = x.view(-1, x.shape[-1])
48 + output, out_orig_shape = output.view(-1, output.shape[-1]), output.shape
49 +
50 + # Since communication is needed, the buffer is directly initialized as a
51 + # tensor rather than a tuple of tensor.
52 + buffers = torch.zeros(
53 + (layer.n_slices, x.shape[0], layer.lora_a_stacked[0].shape[2]),
54 + dtype=torch.float32,
55 + device=x.device,
56 + )
57 +
58 + layer.punica_wrapper.add_shrink(buffers, x, layer.lora_a_stacked, 1.0)
59 + buffers = tensor_model_parallel_all_gather(buffers)
60 + layer.punica_wrapper.add_expand(output,
61 + buffers,
62 + layer.lora_b_stacked,
63 + layer.lora_bias_stacked,
64 + layer.output_slices,
65 + offset_start=0,
66 + add_input=True)
67 +
68 + output = output.view(*out_orig_shape)
69 + # now have column partitioned and packed output
70 + return output
71 +
72 +
73 # these layers are based on the tensor parallelism strategy given in
74 # Y. Sheng et al., S-LoRA: Serving Thousands of Concurrent LoRA Adapters. 2023,
75 # https://arxiv.org/abs/2311.03285.
@@ -51,34 +89,15 @@ class ColumnParallelLinearWithShardedLoRA(ColumnParallelLinearWithLoRA):
89 # gather operation.
90 def slice_lora_a(self, lora_a: torch.Tensor) -> torch.Tensor:
91 tp_rank = get_tensor_model_parallel_rank()
54 - shard_size = self.lora_a_stacked.shape[2]
92 + shard_size = self.lora_a_stacked[0].shape[2]
93 start_idx = tp_rank * shard_size
94 lora_a = lora_a[:, start_idx:start_idx + shard_size]
95 return lora_a
96
59 - def apply(self, x: torch.Tensor,
60 - bias: Optional[torch.Tensor]) -> torch.Tensor:
61 - output = self.base_layer.quant_method.apply(self.base_layer, x, bias)
62 -
63 - x = x.view(-1, x.shape[-1])
64 - output, out_orig_shape = output.view(-1,
65 - output.shape[-1]), output.shape
66 - buffer = torch.zeros(
67 - (x.shape[0], self.lora_a_stacked.shape[2]),
68 - dtype=torch.float32,
69 - device=x.device,
70 - )
71 - self.punica_wrapper.add_shrink(buffer, x, self.lora_a_stacked, 1.0)
72 - buffer = tensor_model_parallel_all_gather(buffer)
73 - self.punica_wrapper.add_expand(output,
74 - buffer,
75 - self.lora_b_stacked,
76 - self.bias_stacked,
77 - add_input=True)
78 - # now have column partitioned output
79 -
80 - output = output.view(*out_orig_shape)
81 - return output
97 + def apply(self,
98 + x: torch.Tensor,
99 + bias: Optional[torch.Tensor] = None) -> torch.Tensor:
100 + return _mcp_apply(x, bias, self)
101
102 @classmethod
103 @_fully_sharded_can_replace
@@ -99,46 +118,6 @@ class ColumnParallelLinearWithShardedLoRA(ColumnParallelLinearWithLoRA):
118 )
119
120
102 -def _mcp_apply(x, bias, layer: QKVParallelLinearWithLora):
103 - """
104 - MergedColumnParallelLinearWithShardedLoRA and
105 - MergedQKVParallelLinearWithShardedLora share the same
106 - LoRa weight application method.
107 -
108 - The main difference is the step by shard_size for lora_b which can
109 - vary for MergedQKVParallelLinearWithShardedLora but is constant for
110 - MergedColumnParallelLinearWithShardedLoRA.
111 - """
112 - # expecting 2 for column parallel and 3 for qkv
113 - n = len(layer.lora_a_stacked)
114 - output = layer.base_layer.quant_method.apply(layer.base_layer, x, bias)
115 -
116 - x = x.view(-1, x.shape[-1])
117 - output, out_orig_shape = output.view(-1, output.shape[-1]), output.shape
118 - buffers = torch.zeros(
119 - (n, x.shape[0], layer.lora_a_stacked[0].shape[2]),
120 - dtype=torch.float32,
121 - device=x.device,
122 - )
123 - for idx in range(n):
124 - layer.punica_wrapper.add_shrink(buffers[idx], x,
125 - layer.lora_a_stacked[idx], 1.0)
126 -
127 - buffers = tensor_model_parallel_all_gather(buffers)
128 - layer.punica_wrapper.add_expand_packed_nslice(
129 - output,
130 - buffers,
131 - layer.lora_b_stacked,
132 - layer.bias_stacked,
133 - 1.0,
134 - layer.output_slices,
135 - )
136 -
137 - output = output.view(*out_orig_shape)
138 - # now have column partitioned and packed output
139 - return output
140 -
141 -
121 class MergedColumnParallelLinearWithShardedLoRA(
122 MergedColumnParallelLinearWithLoRA):
123 """
@@ -162,8 +141,9 @@ class MergedColumnParallelLinearWithShardedLoRA(
141 ]
142 return lora_a
143
165 - def apply(self, x: torch.Tensor,
166 - bias: Optional[torch.Tensor]) -> torch.Tensor:
144 + def apply(self,
145 + x: torch.Tensor,
146 + bias: Optional[torch.Tensor] = None) -> torch.Tensor:
147 return _mcp_apply(x, bias, self)
148
149 @classmethod
@@ -195,31 +175,15 @@ class QKVParallelLinearWithShardedLora(QKVParallelLinearWithLora):
175
176 def slice_lora_a(self, lora_a: torch.Tensor) -> torch.Tensor:
177 tp_rank = get_tensor_model_parallel_rank()
198 - shard_size = self.lora_a_stacked.shape[2]
178 + shard_size = self.lora_a_stacked[0].shape[2]
179 start_idx = tp_rank * shard_size
180 lora_a = lora_a[:, start_idx:start_idx + shard_size]
181 return lora_a
182
203 - def apply(self, x: torch.Tensor,
204 - bias: Optional[torch.Tensor]) -> torch.Tensor:
205 - output = self.base_layer.quant_method.apply(self.base_layer, x, bias)
206 -
207 - x = x.view(-1, x.shape[-1])
208 - output, out_orig_shape = output.view(-1,
209 - output.shape[-1]), output.shape
210 - buffer = torch.zeros((x.shape[0], self.lora_a_stacked.shape[2]),
211 - dtype=torch.float32,
212 - device=x.device)
213 - self.punica_wrapper.add_shrink(buffer, x, self.lora_a_stacked, 1.0)
214 - buffer = tensor_model_parallel_all_gather(buffer)
215 - self.punica_wrapper.add_expand(output,
216 - buffer,
217 - self.lora_b_stacked,
218 - self.bias_stacked,
219 - add_input=True)
220 - # now have column partitioned output
221 - output = output.view(*out_orig_shape)
222 - return output
183 + def apply(self,
184 + x: torch.Tensor,
185 + bias: Optional[torch.Tensor] = None) -> torch.Tensor:
186 + return _mcp_apply(x, bias, self)
187
188 @classmethod
189 @_fully_sharded_can_replace
@@ -260,8 +224,9 @@ class MergedQKVParallelLinearWithShardedLora(MergedQKVParallelLinearWithLora):
224 ]
225 return lora_a
226
263 - def apply(self, x: torch.Tensor,
264 - bias: Optional[torch.Tensor]) -> torch.Tensor:
227 + def apply(self,
228 + x: torch.Tensor,
229 + bias: Optional[torch.Tensor] = None) -> torch.Tensor:
230 return _mcp_apply(x, bias, self)
231
232 @classmethod
@@ -294,7 +259,7 @@ class RowParallelLinearWithShardedLoRA(RowParallelLinearWithLoRA):
259 """
260
261 def slice_lora_b(self, lora_b: torch.Tensor) -> torch.Tensor:
297 - shard_size = self.lora_b_stacked.shape[2]
262 + shard_size = self.lora_b_stacked[0].shape[2]
263 start_idx = self.tp_rank * shard_size
264 end_idx = (self.tp_rank + 1) * shard_size
265 lora_b = lora_b[:, start_idx:end_idx]
@@ -303,20 +268,24 @@ class RowParallelLinearWithShardedLoRA(RowParallelLinearWithLoRA):
268 def slice_bias(self, bias: torch.Tensor) -> torch.Tensor:
269 if bias is None:
270 return bias
306 - shard_size = self.bias_stacked.shape[2]
271 + self.lora_bias_stacked = cast(Tuple[torch.Tensor, ...],
272 + self.lora_bias_stacked)
273 + shard_size = self.lora_bias_stacked[0].shape[2]
274 start_idx = self.tp_rank * shard_size
275 end_idx = (self.tp_rank + 1) * shard_size
276 bias = bias[start_idx:end_idx]
277 return bias
278
312 - def apply(self, x: torch.Tensor) -> torch.Tensor:
279 + def apply(self,
280 + x: torch.Tensor,
281 + bias: Optional[torch.Tensor] = None) -> torch.Tensor:
282 output = self.base_layer.quant_method.apply(self.base_layer, x)
283
284 x = x.view(-1, x.shape[-1])
285 output, out_orig_shape = output.view(-1,
286 output.shape[-1]), output.shape
287 buffer = torch.zeros(
319 - (x.shape[0], self.lora_a_stacked.shape[2]),
288 + (self.n_slices, x.shape[0], self.lora_a_stacked[0].shape[2]),
289 dtype=torch.float32,
290 device=x.device,
291 )
@@ -330,12 +299,18 @@ class RowParallelLinearWithShardedLoRA(RowParallelLinearWithLoRA):
299 # remains is a standard all_reduce. User should be aware though that
300 # the output is not the same as a normal row_parallel, it should be
301 # reduced before being used
333 - shard_size = self.lora_b_stacked.shape[2]
334 - start_idx = self.tp_rank * shard_size
335 - self.punica_wrapper.add_expand_slice(output, buffer,
336 - self.lora_b_stacked,
337 - self.bias_stacked, start_idx,
338 - shard_size)
302 + # NOTE offset are based on the rank.
303 + shard_size = self.lora_b_stacked[0].shape[2]
304 + offset_start = self.tp_rank * shard_size
305 + self.punica_wrapper.add_expand(
306 + output,
307 + buffer,
308 + self.lora_b_stacked,
309 + self.lora_bias_stacked,
310 + self.output_slices,
311 + offset_start=offset_start,
312 + add_input=True,
313 + )
314 output = output.view(*out_orig_shape)
315 return output
316
vllm/lora/layers.py
+193 -345
@@ -1,7 +1,7 @@
1 # pylint: disable=unused-argument
2 import math
3 from dataclasses import dataclass
4 -from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union
4 +from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union, cast
5
6 import torch
7 import torch.nn as nn
@@ -18,11 +18,14 @@ from vllm.distributed import (get_tensor_model_parallel_rank,
18 tensor_model_parallel_gather)
19 from vllm.distributed.utils import divide
20 from vllm.lora.punica import PunicaWrapper
21 +# yapf: disable
22 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
23 + LinearBase,
24 MergedColumnParallelLinear,
25 QKVParallelLinear,
26 ReplicatedLinear,
27 RowParallelLinear)
28 +# yapf: enable
29 from vllm.model_executor.layers.logits_processor import LogitsProcessor
30 from vllm.model_executor.layers.rotary_embedding import (
31 LinearScalingRotaryEmbedding, RotaryEmbedding)
@@ -249,13 +252,10 @@ class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA):
252 full_lora_a_embeddings.shape[1],
253 -1,
254 )
252 -
253 - # Embedding layer only need expand op
254 - self.punica_wrapper.add_expand(full_output,
255 - full_lora_a_embeddings,
256 - self.lora_b_stacked,
257 - bias_all=None,
258 - add_input=True)
255 + self.punica_wrapper.add_lora_embedding(full_output,
256 + full_lora_a_embeddings,
257 + self.lora_b_stacked,
258 + add_input=True)
259 return full_output.view_as(full_output_org)
260
261 @classmethod
@@ -269,14 +269,19 @@ class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA):
269 return type(source_layer) is VocabParallelEmbedding
270
271
272 -class ReplicatedLinearWithLoRA(BaseLayerWithLoRA):
272 +class BaseLinearLayerWithLoRA(BaseLayerWithLoRA):
273
274 - def __init__(self, base_layer: ReplicatedLinear) -> None:
274 + def __init__(self, base_layer: LinearBase):
275 super().__init__()
276 self.base_layer = base_layer
277 self.input_size = self.base_layer.input_size
278 - self.output_size = self.base_layer.output_size
278 self.device = _get_lora_device(self.base_layer)
279 + self.lora_bias_stacked: Optional[Tuple[torch.Tensor, ...]] = None
280 +
281 + self.output_slices: Tuple[int, ...]
282 + self.tp_size: int
283 + self.output_size: int
284 + self.n_slices: int
285
286 def create_lora_weights(
287 self,
@@ -285,39 +290,64 @@ class ReplicatedLinearWithLoRA(BaseLayerWithLoRA):
290 model_config: Optional[PretrainedConfig] = None,
291 ) -> None:
292 self.lora_config = lora_config
288 - lora_a_output_size = lora_config.max_lora_rank
289 - self.lora_a_stacked = torch.zeros(
290 - max_loras,
291 - 1,
292 - lora_a_output_size,
293 - self.input_size,
294 - dtype=lora_config.lora_dtype,
295 - device=self.device,
296 - )
297 - self.lora_b_stacked = torch.zeros(
298 - max_loras,
299 - 1,
300 - self.output_size,
301 - lora_config.max_lora_rank,
302 - dtype=lora_config.lora_dtype,
303 - device=self.device,
304 - )
305 - if lora_config.bias_enabled:
306 - self.bias_stacked = torch.zeros(
293 + #
294 + if isinstance(self.base_layer, ReplicatedLinear):
295 + lora_a_out_size = lora_config.max_lora_rank
296 + lora_b_out_size = self.output_size
297 +
298 + elif isinstance(self.base_layer, ColumnParallelLinear):
299 + lora_a_out_size = (lora_config.max_lora_rank if
300 + not lora_config.fully_sharded_loras else divide(
301 + lora_config.max_lora_rank, self.tp_size))
302 + lora_b_out_size = self.output_size
303 +
304 + elif isinstance(self.base_layer, RowParallelLinear):
305 + lora_a_out_size = lora_config.max_lora_rank
306 + lora_b_out_size = (self.output_size if
307 + not lora_config.fully_sharded_loras else divide(
308 + self.output_size, self.tp_size))
309 + else:
310 + raise NotImplementedError
311 +
312 + self.lora_a_stacked = tuple(
313 + torch.zeros(
314 max_loras,
315 1,
309 - self.output_size,
316 + lora_a_out_size,
317 + self.input_size,
318 dtype=lora_config.lora_dtype,
319 device=self.device,
312 - )
313 - else:
314 - self.bias_stacked = None
320 + ) for _ in range(self.n_slices))
321 + self.lora_b_stacked = tuple(
322 + torch.zeros(
323 + max_loras,
324 + 1,
325 + lora_b_out_size,
326 + lora_config.max_lora_rank,
327 + dtype=lora_config.lora_dtype,
328 + device=self.device,
329 + ) for _ in range(self.n_slices))
330 + if lora_config.bias_enabled:
331 + lora_bias_out_size = lora_b_out_size
332 + self.lora_bias_stacked = tuple(
333 + torch.zeros(
334 + max_loras,
335 + 1,
336 + lora_bias_out_size,
337 + dtype=lora_config.lora_dtype,
338 + device=self.device,
339 + ) for _ in range(self.n_slices))
340 + self.output_slices = (self.lora_b_stacked[0].shape[2], )
341
342 def reset_lora(self, index: int):
317 - self.lora_a_stacked[index] = 0
318 - self.lora_b_stacked[index] = 0
319 - if self.lora_config.bias_enabled:
320 - self.bias_stacked[index] = 0
343 + for s_index in range(self.n_slices):
344 + self.lora_a_stacked[s_index][index] = 0
345 + self.lora_b_stacked[s_index][index] = 0
346 + if self.lora_config.bias_enabled:
347 + # Make mypy happy
348 + self.lora_bias_stacked = cast(Tuple[torch.Tensor, ...],
349 + self.lora_bias_stacked)
350 + self.lora_bias_stacked[s_index][index] = 0
351
352 def set_lora(
353 self,
@@ -325,29 +355,56 @@ class ReplicatedLinearWithLoRA(BaseLayerWithLoRA):
355 lora_a: torch.Tensor,
356 lora_b: torch.Tensor,
357 embeddings_tensor: Optional[torch.Tensor],
328 - bias: Optional[torch.Tensor] = None,
358 + lora_bias: Optional[torch.Tensor] = None,
359 ):
330 - self.reset_lora(index)
360 + # Except for QKVParallelLinearWithLora and
361 + # MergedColumnParallelLinearWithLoRA, all other linear LoRA layers
362 + # store weights in a tuple of size 1. These two layers will
363 + # override this function.
364 + assert (len(self.lora_a_stacked) == len(self.lora_b_stacked) ==
365 + self.n_slices == 1)
366
332 - self.lora_a_stacked[index,
333 - 0, :lora_a.shape[1], :lora_a.shape[0]].copy_(
334 - lora_a.T, non_blocking=True)
335 - self.lora_b_stacked[index,
336 - 0, :lora_b.shape[1], :lora_b.shape[0]].copy_(
337 - lora_b.T, non_blocking=True)
338 - if bias is not None:
339 - self.bias_stacked[index,
340 - 0, :bias.shape[0]].copy_(bias.T,
341 - non_blocking=True)
342 -
343 - def apply(self, x: torch.Tensor,
344 - bias: Optional[torch.Tensor]) -> torch.Tensor:
367 + self.reset_lora(index)
368 + if self.tp_size > 1:
369 + lora_a = self.slice_lora_a(lora_a)
370 + lora_b = self.slice_lora_b(lora_b)
371 + if lora_bias is not None:
372 + lora_bias = self.slice_bias(lora_bias)
373 +
374 + self.lora_a_stacked[0][index,
375 + 0, :lora_a.shape[1], :lora_a.shape[0]].copy_(
376 + lora_a.T, non_blocking=True)
377 + self.lora_b_stacked[0][index,
378 + 0, :lora_b.shape[1], :lora_b.shape[0]].copy_(
379 + lora_b.T, non_blocking=True)
380 + if lora_bias is not None:
381 +
382 + self.lora_bias_stacked = cast(Tuple[torch.Tensor, ...],
383 + self.lora_bias_stacked)
384 + assert len(self.lora_bias_stacked)
385 + self.lora_bias_stacked[0][index, 0, :lora_bias.shape[0]].copy_(
386 + lora_bias.T, non_blocking=True)
387 +
388 + def apply(self,
389 + x: torch.Tensor,
390 + bias: Optional[torch.Tensor] = None) -> torch.Tensor:
391 output = self.base_layer.quant_method.apply(self.base_layer, x, bias)
346 - self.punica_wrapper.add_lora(output, x, self.lora_a_stacked,
347 - self.lora_b_stacked, self.bias_stacked,
348 - 1.0)
392 + self.punica_wrapper.add_lora_linear(output, x, self.lora_a_stacked,
393 + self.lora_b_stacked,
394 + self.lora_bias_stacked, 1.0,
395 + self.output_slices)
396 return output
397
398 +
399 +class ReplicatedLinearWithLoRA(BaseLinearLayerWithLoRA):
400 +
401 + def __init__(self, base_layer: ReplicatedLinear) -> None:
402 + super().__init__(base_layer, )
403 + # To ensure interface compatibility, set to 1 always.
404 + self.tp_size = 1
405 + self.output_size = self.base_layer.output_size
406 + self.n_slices = 1
407 +
408 def forward(self, input_):
409 """Forward of ReplicatedLinearWithLoRA
410
@@ -380,73 +437,26 @@ class ReplicatedLinearWithLoRA(BaseLayerWithLoRA):
437 return type(source_layer) is ReplicatedLinear
438
439
383 -class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
440 +class ColumnParallelLinearWithLoRA(BaseLinearLayerWithLoRA):
441 """
442 LoRA on top of ColumnParallelLinear layer.
386 -
443 LoRA B is sliced for tensor parallelism.
444 + There are two types for the `base_layer`:
445 + 1. ColumnParallelLinear, e.g.`dense_h_to_4h` in `FalconForCausalLM`.
446 + 2. MergedColumnParallelLinear, e.g.`gate_up_proj` in `Phi3ForCausalLM`.
447 """
448
449 def __init__(self, base_layer: ColumnParallelLinear) -> None:
391 - super().__init__()
450 + super().__init__(base_layer)
451 # The base_layer type is ColumnParallelLinear or
452 # MergedColumnParallelLinear, their weight sharding logic is
453 # inconsistent when TP is greater than 1.
454 self.is_merged_col_linear = type(
455 base_layer) is MergedColumnParallelLinear
397 -
398 - self.base_layer = base_layer
456 self.tp_size = get_tensor_model_parallel_world_size()
400 - self.input_size = self.base_layer.input_size
457 self.output_size = self.base_layer.output_size_per_partition
402 - self.device = _get_lora_device(self.base_layer)
403 -
404 - def create_lora_weights(
405 - self,
406 - max_loras: int,
407 - lora_config: LoRAConfig,
408 - model_config: Optional[PretrainedConfig] = None,
409 - ) -> None:
410 - self.lora_config = lora_config
411 - self.tp_size = get_tensor_model_parallel_world_size()
412 - lora_a_output_size_per_partition = (
413 - lora_config.max_lora_rank if not lora_config.fully_sharded_loras
414 - else divide(lora_config.max_lora_rank, self.tp_size))
415 - self.lora_a_stacked = torch.zeros(
416 - max_loras,
417 - 1,
418 - lora_a_output_size_per_partition,
419 - self.input_size,
420 - dtype=lora_config.lora_dtype,
421 - device=self.device,
422 - )
423 - self.lora_b_stacked = torch.zeros(
424 - max_loras,
425 - 1,
426 - self.output_size,
427 - lora_config.max_lora_rank,
428 - dtype=lora_config.lora_dtype,
429 - device=self.device,
430 - )
431 -
432 - if lora_config.bias_enabled:
433 - self.bias_stacked = torch.zeros(
434 - max_loras,
435 - 1,
436 - self.output_size,
437 - dtype=lora_config.lora_dtype,
438 - device=self.device,
439 - )
440 - else:
441 - self.bias_stacked = None
442 -
443 - self.output_dim = self.lora_b_stacked.shape[2]
444 -
445 - def reset_lora(self, index: int):
446 - self.lora_a_stacked[index] = 0
447 - self.lora_b_stacked[index] = 0
448 - if self.lora_config.bias_enabled:
449 - self.bias_stacked[index] = 0
458 + # There is only one LoRA layer
459 + self.n_slices = 1
460
461 def slice_lora_a(self, lora_a: torch.Tensor) -> torch.Tensor:
462 return lora_a
@@ -485,40 +495,6 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
495 bias = bias[start_idx:end_idx]
496 return bias
497
488 - def set_lora(
489 - self,
490 - index: int,
491 - lora_a: torch.Tensor,
492 - lora_b: torch.Tensor,
493 - embeddings_tensor: Optional[torch.Tensor],
494 - bias: Optional[torch.Tensor] = None,
495 - ):
496 - self.reset_lora(index)
497 -
498 - if self.tp_size > 1:
499 - lora_a = self.slice_lora_a(lora_a)
500 - lora_b = self.slice_lora_b(lora_b)
501 - bias = self.slice_bias(bias)
502 -
503 - self.lora_a_stacked[index,
504 - 0, :lora_a.shape[1], :lora_a.shape[0]].copy_(
505 - lora_a.T, non_blocking=True)
506 - self.lora_b_stacked[index,
507 - 0, :lora_b.shape[1], :lora_b.shape[0]].copy_(
508 - lora_b.T, non_blocking=True)
509 - if bias is not None:
510 - self.bias_stacked[index,
511 - 0, :bias.shape[0]].copy_(bias.T,
512 - non_blocking=True)
513 -
514 - def apply(self, x: torch.Tensor,
515 - bias: Optional[torch.Tensor]) -> torch.Tensor:
516 - output = self.base_layer.quant_method.apply(self.base_layer, x, bias)
517 - self.punica_wrapper.add_lora(output, x, self.lora_a_stacked,
518 - self.lora_b_stacked, self.bias_stacked,
519 - 1.0)
520 - return output
521 -
498 def forward(self, input_):
499 """Forward of ColumnParallelLinear
500
@@ -568,6 +544,8 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
544
545 def __init__(self, base_layer: MergedColumnParallelLinear) -> None:
546 super().__init__(base_layer)
547 + # There are two LoRA layers
548 + self.n_slices = len(self.base_layer.output_sizes)
549
550 def create_lora_weights(
551 self,
@@ -575,9 +553,13 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
553 lora_config: LoRAConfig,
554 model_config: Optional[PretrainedConfig] = None,
555 ) -> None:
556 + """
557 + The main reason for overriding this function is to enhance code
558 + maintainability.
559 + """
560 self.lora_config = lora_config
579 - n_slices = 2
580 - if not (len(self.base_layer.output_sizes) == n_slices
561 +
562 + if not (len(self.base_layer.output_sizes) == self.n_slices == 2
563 and self.base_layer.output_sizes[0]
564 == self.base_layer.output_sizes[1]):
565 raise ValueError(
@@ -598,7 +580,7 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
580 self.input_size,
581 dtype=lora_config.lora_dtype,
582 device=self.device,
601 - ) for _ in range(n_slices))
583 + ) for _ in range(self.n_slices))
584 self.lora_b_stacked = tuple(
585 torch.zeros(
586 max_loras,
@@ -607,30 +589,19 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
589 lora_config.max_lora_rank,
590 dtype=lora_config.lora_dtype,
591 device=self.device,
610 - ) for _ in range(n_slices))
592 + ) for _ in range(self.n_slices))
593 if lora_config.bias_enabled:
612 - self.bias_stacked = tuple(
594 + self.lora_bias_stacked = tuple(
595 torch.zeros(
596 max_loras,
597 1,
598 self.output_size // 2,
599 dtype=lora_config.lora_dtype,
600 device=self.device,
619 - ) for _ in range(n_slices))
620 - else:
621 - self.bias_stacked = None
601 + ) for _ in range(self.n_slices))
602 self.output_dim = self.lora_b_stacked[0].shape[2]
603 self.output_slices = (self.output_dim, self.output_dim)
604
625 - def reset_lora(self, index: int):
626 - self.lora_a_stacked[0][index] = 0
627 - self.lora_a_stacked[1][index] = 0
628 - self.lora_b_stacked[0][index] = 0
629 - self.lora_b_stacked[1][index] = 0
630 - if self.lora_config.bias_enabled:
631 - self.bias_stacked[0][index] = 0
632 - self.bias_stacked[1][index] = 0
633 -
605 def slice_lora_a(
606 self, lora_a: List[Union[torch.Tensor, None]]
607 ) -> List[Union[torch.Tensor, None]]:
@@ -668,15 +639,15 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
639 lora_a: torch.Tensor,
640 lora_b: torch.Tensor,
641 embeddings_tensor: Optional[torch.Tensor],
671 - bias: Optional[torch.Tensor] = None,
642 + lora_bias: Optional[torch.Tensor] = None,
643 ):
644 self.reset_lora(index)
645
646 if self.tp_size > 1:
647 lora_a = self.slice_lora_a(lora_a)
648 lora_b = self.slice_lora_b(lora_b)
678 - if bias is not None:
679 - bias = self.slice_bias(bias)
649 + if lora_bias is not None:
650 + lora_bias = self.slice_bias(lora_bias)
651
652 if lora_a[0] is not None:
653 self.lora_a_stacked[0][
@@ -685,10 +656,11 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
656 self.lora_b_stacked[0][
657 index, 0, :lora_b[0].shape[1], :lora_b[0].shape[0]].copy_(
658 lora_b[0].T, non_blocking=True)
688 - if bias is not None and bias[0] is not None:
689 - self.bias_stacked[0][index,
690 - 0, :bias[0].shape[0]].copy_(bias[0].T,
691 - non_blocking=True)
659 + if lora_bias is not None and lora_bias[0] is not None:
660 + self.lora_bias_stacked = cast(Tuple[torch.Tensor, ...],
661 + self.lora_bias_stacked)
662 + self.lora_bias_stacked[0][index, 0, :lora_bias[0].shape[0]].copy_(
663 + lora_bias[0].T, non_blocking=True)
664 if lora_a[1] is not None:
665 self.lora_a_stacked[1][
666 index, 0, :lora_a[1].shape[1], :lora_a[1].shape[0]].copy_(
@@ -696,18 +668,11 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
668 self.lora_b_stacked[1][
669 index, 0, :lora_b[1].shape[1], :lora_b[1].shape[0]].copy_(
670 lora_b[1].T, non_blocking=True)
699 - if bias is not None and bias[1] is not None:
700 - self.bias_stacked[1][index,
701 - 0, :bias[1].shape[0]].copy_(bias[1].T,
702 - non_blocking=True)
703 -
704 - def apply(self, x: torch.Tensor,
705 - bias: Optional[torch.Tensor]) -> torch.Tensor:
706 - output = self.base_layer.quant_method.apply(self.base_layer, x, bias)
707 - self.punica_wrapper.add_lora_packed_nslice(
708 - output, x, self.lora_a_stacked, self.lora_b_stacked,
709 - self.bias_stacked, 1.0, (self.output_dim, self.output_dim))
710 - return output
671 + if lora_bias is not None and lora_bias[1] is not None:
672 + self.lora_bias_stacked = cast(Tuple[torch.Tensor, ...],
673 + self.lora_bias_stacked)
674 + self.lora_bias_stacked[1][index, 0, :lora_bias[1].shape[0]].copy_(
675 + lora_bias[1].T, non_blocking=True)
676
677 @classmethod
678 @_not_fully_sharded_can_replace
@@ -737,7 +702,6 @@ class QKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
702
703 def __init__(self, base_layer: QKVParallelLinear) -> None:
704 super().__init__(base_layer)
740 - self.tp_size = get_tensor_model_parallel_world_size()
705 self.q_proj_total_size = (self.base_layer.total_num_heads *
706 self.base_layer.head_size)
707 self.q_proj_shard_size = (self.base_layer.num_heads *
@@ -746,6 +710,8 @@ class QKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
710 self.base_layer.head_size)
711 self.kv_proj_total_size = (self.base_layer.total_num_kv_heads *
712 self.base_layer.head_size)
713 + # There is only one LoRA layer
714 + self.n_slices = 1
715
716 def slice_lora_b(self, lora_b: torch.Tensor) -> torch.Tensor:
717 tp_rank = get_tensor_model_parallel_rank()
@@ -780,32 +746,6 @@ class QKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
746 bias = torch.cat([bias_q, bias_k, bias_v], dim=1)
747 return bias
748
783 - def set_lora(
784 - self,
785 - index: int,
786 - lora_a: torch.Tensor,
787 - lora_b: torch.Tensor,
788 - embeddings_tensor: Optional[torch.Tensor],
789 - bias: Optional[torch.Tensor] = None,
790 - ):
791 - self.reset_lora(index)
792 - if self.tp_size > 1:
793 - lora_a = self.slice_lora_a(lora_a)
794 - lora_b = self.slice_lora_b(lora_b)
795 - if bias is not None:
796 - bias = self.slice_bias(bias)
797 -
798 - self.lora_a_stacked[index,
799 - 0, :lora_a.shape[1], :lora_a.shape[0]].copy_(
800 - lora_a.T, non_blocking=True)
801 - self.lora_b_stacked[index,
802 - 0, :lora_b.shape[1], :lora_b.shape[0]].copy_(
803 - lora_b.T, non_blocking=True)
804 - if bias is not None:
805 - self.bias_stacked[index,
806 - 0, :bias.shape[0]].copy_(bias.T,
807 - non_blocking=True)
808 -
749 @classmethod
750 @_not_fully_sharded_can_replace
751 def can_replace_layer(cls, source_layer: nn.Module,
@@ -828,6 +768,10 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
768
769 def __init__(self, base_layer: QKVParallelLinear) -> None:
770 super().__init__(base_layer)
771 + # There are three LoRA layer.
772 + self.n_slices = len(self.base_layer.output_sizes)
773 + self.tp_size = get_tensor_model_parallel_world_size()
774 + self.tp_rank = get_tensor_model_parallel_rank()
775
776 def create_lora_weights(
777 self,
@@ -835,9 +779,16 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
779 lora_config: LoRAConfig,
780 model_config: Optional[PretrainedConfig] = None,
781 ) -> None:
782 + """
783 + The main reason for overloading this function is to handle inconsistent
784 + weight dimensions in qkv lora.
785 + """
786 self.lora_config = lora_config
839 - self.tp_size = get_tensor_model_parallel_world_size()
840 - self.tp_rank = get_tensor_model_parallel_rank()
787 +
788 + if not (len(self.base_layer.output_sizes) == self.n_slices == 3):
789 + raise ValueError(
790 + "LoRAColumnParallelLinear3Slice requires 3 slices.")
791 +
792 self.q_proj_shard_size = (self.base_layer.num_heads *
793 self.base_layer.head_size)
794 self.kv_proj_shard_size = (self.base_layer.num_kv_heads *
@@ -902,7 +853,7 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
853 ),
854 )
855 if lora_config.bias_enabled:
905 - self.bias_stacked = (
856 + self.lora_bias_stacked = (
857 torch.zeros(
858 max_loras,
859 1,
@@ -925,9 +876,6 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
876 device=self.device,
877 ),
878 )
928 - else:
929 - self.bias_stacked = None
930 -
879 self.output_slices = (
880 self.q_proj_shard_size,
881 self.kv_proj_shard_size,
@@ -939,18 +887,6 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
887 self.indices: torch.Tensor
888 self.indices_len: List[int]
889
942 - def reset_lora(self, index: int):
943 - self.lora_a_stacked[0][index] = 0
944 - self.lora_b_stacked[0][index] = 0
945 - self.lora_a_stacked[1][index] = 0
946 - self.lora_b_stacked[1][index] = 0
947 - self.lora_a_stacked[2][index] = 0
948 - self.lora_b_stacked[2][index] = 0
949 - if self.lora_config.bias_enabled:
950 - self.bias_stacked[0][index] = 0
951 - self.bias_stacked[1][index] = 0
952 - self.bias_stacked[2][index] = 0
953 -
890 def slice_lora_a(
891 self, lora_a: List[Union[torch.Tensor, None]]
892 ) -> List[Union[torch.Tensor, None]]:
@@ -1000,15 +936,15 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
936 lora_a: torch.Tensor,
937 lora_b: torch.Tensor,
938 embeddings_tensor: Optional[torch.Tensor],
1003 - bias: Optional[torch.Tensor] = None,
939 + lora_bias: Optional[torch.Tensor] = None,
940 ):
941 self.reset_lora(index)
942
943 if self.tp_size > 1:
944 lora_a = self.slice_lora_a(lora_a)
945 lora_b = self.slice_lora_b(lora_b)
1010 - if bias is not None:
1011 - bias = self.slice_bias(bias)
946 + if lora_bias is not None:
947 + lora_bias = self.slice_bias(lora_bias)
948
949 if lora_b[0] is not None:
950 lora_b_q = lora_b[0]
@@ -1039,26 +975,24 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
975 index, 0, :lora_a[2].shape[1], :lora_a[2].shape[0]].copy_(
976 lora_a[2].T, non_blocking=True)
977
1042 - if bias is not None:
1043 - if bias[0] is not None:
1044 - self.bias_stacked[0][index, 0, :bias[0].shape[0]].copy_(
1045 - bias[0].T, non_blocking=True)
1046 - if bias[1] is not None:
1047 - self.bias_stacked[1][index, 0, :bias[1].shape[0]].copy_(
1048 - bias[1].T, non_blocking=True)
1049 - if bias[2] is not None:
1050 - self.bias_stacked[2][index, 0, :bias[2].shape[0]].copy_(
1051 - bias[2].T, non_blocking=True)
1052 -
1053 - def apply(self, x: torch.Tensor,
1054 - bias: Optional[torch.Tensor]) -> torch.Tensor:
1055 - output = self.base_layer.quant_method.apply(self.base_layer, x, bias)
1056 - self.punica_wrapper.add_lora_packed_nslice(output, x,
1057 - self.lora_a_stacked,
1058 - self.lora_b_stacked,
1059 - self.bias_stacked, 1.0,
1060 - self.output_slices)
1061 - return output
978 + if lora_bias is not None:
979 + self.lora_bias_stacked = cast(Tuple[torch.Tensor, ...],
980 + self.lora_bias_stacked)
981 + if lora_bias[0] is not None:
982 + self.lora_bias_stacked[0][index,
983 + 0, :lora_bias[0].shape[0]].copy_(
984 + lora_bias[0].T,
985 + non_blocking=True)
986 + if lora_bias[1] is not None:
987 + self.lora_bias_stacked[1][index,
988 + 0, :lora_bias[1].shape[0]].copy_(
989 + lora_bias[1].T,
990 + non_blocking=True)
991 + if lora_bias[2] is not None:
992 + self.lora_bias_stacked[2][index,
993 + 0, :lora_bias[2].shape[0]].copy_(
994 + lora_bias[2].T,
995 + non_blocking=True)
996
997 @classmethod
998 @_not_fully_sharded_can_replace
@@ -1073,76 +1007,25 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
1007 and len(packed_modules_list) == 3)
1008
1009
1076 -class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
1010 +class RowParallelLinearWithLoRA(BaseLinearLayerWithLoRA):
1011
1012 def __init__(self, base_layer: RowParallelLinear) -> None:
1079 - super().__init__()
1080 - self.base_layer = base_layer
1013 + super().__init__(base_layer)
1014 +
1015 + self.tp_size = get_tensor_model_parallel_world_size()
1016 + # reset input_size
1017 self.input_size = self.base_layer.input_size_per_partition
1018 self.output_size = self.base_layer.output_size
1083 - self.device = _get_lora_device(self.base_layer)
1019
1085 - def create_lora_weights(
1086 - self,
1087 - max_loras: int,
1088 - lora_config: LoRAConfig,
1089 - model_config: Optional[PretrainedConfig] = None,
1090 - ) -> None:
1091 - self.lora_config = lora_config
1020 self.tp_rank = get_tensor_model_parallel_rank()
1093 - self.lora_a_stacked = torch.zeros(
1094 - (
1095 - max_loras,
1096 - 1,
1097 - lora_config.max_lora_rank,
1098 - self.input_size,
1099 - ),
1100 - dtype=lora_config.lora_dtype,
1101 - device=self.device,
1102 - )
1103 - tp_size = get_tensor_model_parallel_world_size()
1104 - lora_b_output_size_per_partition = (
1105 - self.output_size if not lora_config.fully_sharded_loras else
1106 - divide(self.output_size, tp_size))
1107 -
1108 - self.lora_b_stacked = torch.zeros(
1109 - (
1110 - max_loras,
1111 - 1,
1112 - lora_b_output_size_per_partition,
1113 - lora_config.max_lora_rank,
1114 - ),
1115 - dtype=lora_config.lora_dtype,
1116 - device=self.device,
1117 - )
1118 -
1119 - if lora_config.bias_enabled:
1120 - self.bias_stacked = torch.zeros(
1121 - (
1122 - max_loras,
1123 - 1,
1124 - self.output_size,
1125 - ),
1126 - dtype=lora_config.lora_dtype,
1127 - device=self.device,
1128 - )
1129 - else:
1130 - self.bias_stacked = None
1131 - # Lazily initialized
1132 - self.indices: torch.Tensor
1133 - self.indices_len: List[int]
1134 -
1135 - def reset_lora(self, index: int):
1136 - self.lora_a_stacked[index] = 0
1137 - self.lora_b_stacked[index] = 0
1138 - if self.lora_config.bias_enabled:
1139 - self.bias_stacked[index] = 0
1021 + # There is only one LoRA layer.
1022 + self.n_slices = 1
1023
1024 def slice_lora_a(self, lora_a: torch.Tensor) -> torch.Tensor:
1142 - tensor_model_parallel_rank = get_tensor_model_parallel_rank()
1025 +
1026 shard_size = self.input_size
1144 - start_idx = tensor_model_parallel_rank * shard_size
1145 - end_idx = (tensor_model_parallel_rank + 1) * shard_size
1027 + start_idx = self.tp_rank * shard_size
1028 + end_idx = (self.tp_rank + 1) * shard_size
1029 lora_a = lora_a[start_idx:end_idx, :]
1030 return lora_a
1031
@@ -1152,40 +1035,6 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
1035 def slice_bias(self, bias: torch.Tensor) -> torch.Tensor:
1036 return bias
1037
1155 - def set_lora(
1156 - self,
1157 - index: int,
1158 - lora_a: torch.Tensor,
1159 - lora_b: torch.Tensor,
1160 - embeddings_tensor: Optional[torch.Tensor],
1161 - bias: Optional[torch.Tensor] = None,
1162 - ):
1163 - self.reset_lora(index)
1164 -
1165 - if self.base_layer.tp_size > 1:
1166 - lora_a = self.slice_lora_a(lora_a)
1167 - lora_b = self.slice_lora_b(lora_b)
1168 - if bias is not None:
1169 - bias = self.slice_bias(bias)
1170 -
1171 - self.lora_a_stacked[index,
1172 - 0, :lora_a.shape[1], :lora_a.shape[0]].copy_(
1173 - lora_a.T, non_blocking=True)
1174 - self.lora_b_stacked[index,
1175 - 0, :lora_b.shape[1], :lora_b.shape[0]].copy_(
1176 - lora_b.T, non_blocking=True)
1177 - if bias is not None:
1178 - self.bias_stacked[index,
1179 - 0, :bias.shape[0]].copy_(bias.T,
1180 - non_blocking=True)
1181 -
1182 - def apply(self, x: torch.Tensor) -> torch.Tensor:
1183 - output = self.base_layer.quant_method.apply(self.base_layer, x)
1184 - self.punica_wrapper.add_lora(output, x, self.lora_a_stacked,
1185 - self.lora_b_stacked, self.bias_stacked,
1186 - 1.0)
1187 - return output
1188 -
1038 def forward(self, input_):
1039 """Forward of RowParallelLinear
1040
@@ -1203,10 +1052,9 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
1052 input_parallel = input_
1053 else:
1054 # TODO: simplify code below
1206 - tp_rank = get_tensor_model_parallel_rank()
1055 splitted_input = split_tensor_along_last_dim(
1056 input_, num_partitions=self.base_layer.tp_size)
1209 - input_parallel = splitted_input[tp_rank].contiguous()
1057 + input_parallel = splitted_input[self.tp_rank].contiguous()
1058
1059 # Matrix multiply.
1060 output_parallel = self.apply(input_parallel)
vllm/lora/models.py
+4 -4
@@ -555,17 +555,17 @@ class LoRAModelManager(AdapterModelManager):
555 input_dim,
556 output_dim,
557 rank,
558 - module.lora_a_stacked.dtype,
558 + module.lora_a_stacked[0].dtype,
559 "cpu",
560 embeddings_tensor_dim=embeddings_tensor_dim,
561 bias_enabled=bias_enabled)
562 else:
563 lora = LoRALayerWeights.create_dummy_lora_weights(
564 module_name,
565 - module.lora_a_stacked.shape[-1],
566 - module.lora_b_stacked.shape[-2],
565 + module.lora_a_stacked[0].shape[-1],
566 + module.lora_b_stacked[0].shape[-2],
567 rank,
568 - module.lora_a_stacked.dtype,
568 + module.lora_a_stacked[0].dtype,
569 "cpu",
570 bias_enabled=bias_enabled,
571 )
vllm/lora/punica.py
+193 -172
@@ -362,7 +362,7 @@ class PunicaWrapper:
362 long_lora_len = self.indices_len[4]
363 return self._long_lora_indices[:long_lora_len]
364
365 - def shrink_prefill(
365 + def _shrink_prefill(
366 self,
367 y: torch.Tensor,
368 x: torch.Tensor,
@@ -380,7 +380,7 @@ class PunicaWrapper:
380 scale,
381 )
382
383 - def shrink_decode(
383 + def _shrink_decode(
384 self,
385 y: torch.Tensor,
386 x: torch.Tensor,
@@ -389,7 +389,7 @@ class PunicaWrapper:
389 ):
390 bgmv_shrink(x, w_t_all, y, self.token_lora_indices, scale)
391
392 - def expand_prefill(
392 + def _expand_prefill(
393 self,
394 y: torch.Tensor,
395 x: torch.Tensor,
@@ -407,7 +407,7 @@ class PunicaWrapper:
407 add_input,
408 )
409
410 - def expand_decode(
410 + def _expand_decode(
411 self,
412 y: torch.Tensor,
413 x: torch.Tensor,
@@ -416,7 +416,7 @@ class PunicaWrapper:
416 ):
417 bgmv_expand(x, w_t_all, y, self.token_lora_indices, add_input)
418
419 - def expand_slice_prefill(
419 + def _expand_slice_prefill(
420 self,
421 y: torch.Tensor,
422 x: torch.Tensor,
@@ -438,7 +438,7 @@ class PunicaWrapper:
438 add_input,
439 )
440
441 - def expand_slice_decode(
441 + def _expand_slice_decode(
442 self,
443 y: torch.Tensor,
444 x: torch.Tensor,
@@ -450,41 +450,35 @@ class PunicaWrapper:
450 bgmv_expand_slice(x, w_t_all, y, self.token_lora_indices, y_offset,
451 y_slice_size, add_input)
452
453 - def apply_bias(
454 - self,
455 - indices: torch.Tensor,
456 - output: torch.Tensor,
457 - bias_stacked: torch.Tensor,
458 - ):
459 - """Applies bias to output
460 -
461 - Input shapes:
462 - bias_stacked: (num_loras, output_dim)
463 - indices: (batch_size)
464 - output: (batch_size, output_dim)
453 + def _apply_expand(self,
454 + y: torch.Tensor,
455 + x: torch.Tensor,
456 + w_t_all: torch.Tensor,
457 + y_offset: Optional[int],
458 + y_slice_size: Optional[int],
459 + add_input: bool = True):
460 + """
461 + Perform the ` y[:,y_offset:y_offset+y_slice_size]+=x@w_t_all`
462 + computation, which is suitable for the
463 + GEMM of lora'b.
464 """
466 - org_output = output
467 - output = output.view(-1, output.shape[-1])
468 - indices = indices.view(-1)
469 -
470 - bias_stacked = bias_stacked.view(-1, bias_stacked.shape[-1])
471 - bias_stacked = bias_stacked[indices]
472 - bias_stacked[indices == -1] = 0
473 - output += bias_stacked
465
475 - return output.view_as(org_output)
466 + expand_slice_fun: Callable = (self._expand_slice_prefill
467 + if self.is_prefill else
468 + self._expand_slice_decode)
469 + expand_slice_fun(y, x, w_t_all, y_offset, y_slice_size, add_input)
470
477 - def apply_bias_packed_nslice(
471 + def _apply_bias(
472 self,
473 indices: torch.Tensor,
474 output: torch.Tensor,
475 output_slices: Tuple[int, ...],
482 - bias_stacked: Tuple[Optional[torch.Tensor], ...],
476 + lora_bias_stacked: Tuple[Optional[torch.Tensor], ...],
477 ):
478 """Applies bias to output
479
480 Input shapes:
487 - bias_stacked: 3 element tuple of (num_loras, output_dim)
481 + lora_bias_stacked: 3 element tuple of (num_loras, output_dim)
482 indices: (batch_size)
483 output: (batch_size, q_slice_size + 2*kv_slice_size)
484 output_slices: n-1 element tuple of (slice_size...),
@@ -496,7 +490,7 @@ class PunicaWrapper:
490
491 offset_left = 0
492 for slice_idx, slice in enumerate(output_slices):
499 - bias = bias_stacked[slice_idx]
493 + bias = lora_bias_stacked[slice_idx]
494 if bias is not None:
495 bias = bias.view(-1, bias.shape[-1])
496 bias = bias[indices]
@@ -506,7 +500,7 @@ class PunicaWrapper:
500
501 return output.view_as(org_output)
502
509 - def add_shrink(
503 + def _apply_shrink(
504 self,
505 y: torch.Tensor,
506 x: torch.Tensor,
@@ -517,188 +511,215 @@ class PunicaWrapper:
511 Perform the ` y+=x@w_t_all` computation, which is suitable for the
512 GEMM of lora'a.
513 When `is_prefill is` true, it indicates that it is currently the
520 - prefill stage, and the `shrink_prefill` function should be called.
521 - Otherwise, it is the decode stage, and the shrink_decode function
514 + prefill stage, and the `_shrink_prefill` function should be called.
515 + Otherwise, it is the decode stage, and the _shrink_decode function
516 should be called.
517 """
524 - shrink_fun: Callable = (self.shrink_prefill
525 - if self.is_prefill else self.shrink_decode)
518 + y_org = y
519 + y = y.view(-1, y.shape[-1])
520 + shrink_fun: Callable = (self._shrink_prefill
521 + if self.is_prefill else self._shrink_decode)
522 shrink_fun(y, x, w_t_all, scale)
523 + y = y.view_as(y_org)
524
528 - def add_expand(
525 + def add_shrink(
526 self,
530 - y: torch.Tensor,
527 + y: Union[Tuple[torch.Tensor, ...], torch.Tensor],
528 x: torch.Tensor,
532 - w_t_all: torch.Tensor,
533 - bias_all: Optional[torch.Tensor],
534 - add_input: bool = True,
529 + lora_a_stacked: Tuple[torch.Tensor, ...],
530 + scale: float,
531 ):
532 """
537 - Perform the ` y+=x@w_t_all+bias` computation, which is suitable for the
538 - GEMM of lora'b.
539 - When `is_prefill` is true, it indicates that it is currently the
540 - prefill stage, and the `expand_prefill` function should be called.
541 - Otherwise, it is the decode stage, and the expand_decode function
533 + Performs GEMM for multiple slices of lora_a.
534 + When `is_prefill is` true, it indicates that it is currently the
535 + prefill stage, and the `_shrink_prefill` function should be called.
536 + Otherwise, it is the decode stage, and the _shrink_decode function
537 should be called.
543 - """
544 - if bias_all is not None:
545 - y = self.apply_bias(self.token_lora_indices, y, bias_all)
546 -
547 - expand_fun: Callable = (self.expand_prefill
548 - if self.is_prefill else self.expand_decode)
549 - expand_fun(y, x, w_t_all, add_input)
550 -
551 - def add_expand_slice(self,
552 - y: torch.Tensor,
553 - x: torch.Tensor,
554 - w_t_all: torch.Tensor,
555 - bias_all: Optional[torch.Tensor],
556 - y_offset: Optional[int],
557 - y_slice_size: Optional[int],
558 - add_input: bool = True):
559 - """
560 - Similar to `add_expand`
561 - """
562 - if bias_all is not None:
563 - y = self.apply_bias(self.token_lora_indices, y, bias_all)
538 +
539 + Semantics:
540 + for i in range(len(lora_a_stacked)):
541 + y[i] += (x @ lora_a_stacked[i]) * scale
542 +
543 + Args:
544 + y (Union[Tuple[torch.Tensor, ...], torch.Tensor]): Output tensors
545 + x (torch.Tensor): Input tensor
546 + lora_a_stacked (Tuple[torch.Tensor, ...]): lora_a's weights
547 + scale (float): Scaling factor for the operation
548 + """
549
565 - expand_slice_fun: Callable = (self.expand_slice_prefill
566 - if self.is_prefill else
567 - self.expand_slice_decode)
568 - expand_slice_fun(y, x, w_t_all, y_offset, y_slice_size, add_input)
550 + x = x.view(-1, x.shape[-1])
551 + # TODO fuse these kernels
552 + for slice_idx in range(len(lora_a_stacked)):
553 + self._apply_shrink(y[slice_idx], x, lora_a_stacked[slice_idx],
554 + scale)
555
570 - def add_expand_packed_nslice(self, y: torch.Tensor, x: torch.Tensor,
571 - lora_b_stacked: Tuple[torch.Tensor, ...],
572 - bias_stacked: Optional[Tuple[torch.Tensor,
573 - ...]],
574 - scale: float,
575 - output_slices: Tuple[int, ...]) -> None:
576 - """
577 - Similar to `add_expand`
556 + def add_expand(
557 + self,
558 + y: torch.Tensor,
559 + x: Union[Tuple[torch.Tensor, ...], torch.Tensor],
560 + lora_b_stacked: Tuple[torch.Tensor, ...],
561 + lora_bias_stacked: Optional[Tuple[torch.Tensor, ...]],
562 + output_slices: Tuple[int, ...],
563 + offset_start: int = 0,
564 + add_input=True,
565 + ) -> None:
566 """
567 + Performs GEMM and bias addition for multiple slices of lora_b.
568 +
569 + Semantics:
570 + for i in range(len(lora_b_stacked)):
571 + slice = output_slices[i]
572 + y[:, offset:offset+slice] += x[i] @ lora_b_stacked[i] +
573 + lora_bias_stacked[i]
574 + offset += slice
575 +
576 + Args:
577 + y (torch.Tensor): Output tensor.
578 + x (Union[Tuple[torch.Tensor, ...], torch.Tensor]): Input tensors
579 + lora_b_stacked (Tuple[torch.Tensor, ...]): lora_b's weight
580 + lora_bias_stacked (Optional[Tuple[torch.Tensor, ...]]):
581 + bias's weight
582 + output_slices (Tuple[int, ...]): Every slice's size
583 + add_input (bool): Defaults to True.
584 + """
585 y_org = y
586 y = y.view(-1, y.shape[-1])
581 - offset_left = 0
582 - if bias_stacked is not None:
583 - self.apply_bias_packed_nslice(self.token_lora_indices, y,
584 - output_slices, bias_stacked)
587 + offset_left = offset_start
588 + if lora_bias_stacked is not None:
589 + self._apply_bias(self.token_lora_indices, y, output_slices,
590 + lora_bias_stacked)
591 for slice_idx in range(len(lora_b_stacked)):
586 - self.add_expand_slice(y,
587 - x[slice_idx],
588 - lora_b_stacked[slice_idx],
589 - None,
590 - offset_left,
591 - output_slices[slice_idx],
592 - add_input=True)
592 + self._apply_expand(
593 + y,
594 + x[slice_idx],
595 + lora_b_stacked[slice_idx],
596 + offset_left,
597 + output_slices[slice_idx],
598 + add_input=add_input,
599 + )
600 offset_left += output_slices[slice_idx]
594 -
601 y = y.view_as(y_org)
602
597 - def add_lora(self,
598 - y: torch.Tensor,
599 - x: torch.Tensor,
600 - wa_t_all: torch.Tensor,
601 - wb_t_all: torch.Tensor,
602 - bias_all: Optional[torch.Tensor],
603 - scale: float,
604 - y_offset: Optional[int] = None,
605 - y_slice_size: Optional[int] = None,
606 - *,
607 - buffer: Optional[torch.Tensor] = None) -> None:
603 + def add_lora_embedding(
604 + self,
605 + y: torch.Tensor,
606 + x: torch.Tensor,
607 + lora_b_stacked: torch.Tensor,
608 + add_input: bool = True,
609 + ):
610 + """
611 + Applies lora specifically for VocabParallelEmbeddingWithLoRA.
612 +
613 + Semantics:
614 + y += x @ lora_b_stacked
615 +
616 + Args:
617 + y (torch.Tensor): Output tensor.
618 + x (torch.Tensor): Input tensor.
619 + lora_b_stacked (torch.Tensor): lora_b's weights.
620 + add_input (bool): Default to True.
621 +
622 + """
623 +
624 + # Embedding layer only need expand op
625 + expand_fun: Callable = (self._expand_prefill
626 + if self.is_prefill else self._expand_decode)
627 + expand_fun(y, x, lora_b_stacked, add_input)
628 +
629 + def add_lora_linear(
630 + self,
631 + y: torch.Tensor,
632 + x: torch.Tensor,
633 + lora_a_stacked: Tuple[torch.Tensor, ...],
634 + lora_b_stacked: Tuple[torch.Tensor, ...],
635 + lora_bias_stacked: Optional[Tuple[torch.Tensor, ...]],
636 + scale: float,
637 + output_slices: Tuple[int, ...],
638 + *,
639 + buffer: Optional[Tuple[torch.Tensor, ...]] = None) -> None:
640 """
641 + Applicable to linear-related lora.
642 +
643 Semantics:
610 - y[i] += (
611 - x[i].unsqueeze(0)
612 - @ wa_t_all[indices[i], layer_idx, :, :].transpose(-1, -2)
613 - @ wb_t_all[indices[i], layer_idx, :, :].transpose(-1, -2)
614 - * scale
615 - ).squeeze(0)+bias[i]
644 + for i in range(len(lora_a_stacked)):
645 + y[i] += (
646 + x[i].unsqueeze(0)
647 + @ lora_a_stacked[indices[i], layer_idx, :, :]
648 + @ lora_b_stacked[indices[i], layer_idx, :, :]
649 + * scale
650 + ).squeeze(0)+lora_bias_stacked[i]
651 +
652 Args:
617 - y (torch.Tensor): Output tensor. Will be changed in-place.
653 + y (torch.Tensor): Output tensor. Will be changed in-place.
654 x (torch.Tensor): Input tensor
619 - wa_t_all (torch.Tensor): lora_a's weight
620 - wb_t_all (torch.Tensor): lora_b's weight
621 - bias_all: (torch.Tensor): lora's bias
655 + lora_a_stacked (Tuple[torch.Tensor, ...]): lora_a's weight.
656 + lora_b_stacked (Tuple[torch.Tensor, ...]): lora_b's weight.
657 + lora_bias_stacked (Optional[Tuple[torch.Tensor, ...]]): lora's bias.
658 scale (float): Scaling factor.
623 - y_offset (Optional[int], optional): Offset to apply to the starting
624 - column of y.
625 - y_slice_size (Optional[int], optional): Size of the y column slice.
626 - buffer (Optional[torch.Tensor], optional): Defaults to None.
659 + output_slices (Tuple[int, ...]): Every slice's size.
660 + buffer (Optional[Tuple[torch.Tensor, ...]]): Defaults to None.
661 """
628 - y_org = y
629 - y = y.view(-1, y.shape[-1])
630 - x = x.view(-1, x.shape[-1])
631 - r = wb_t_all.size(-1)
662 +
663 + assert len(lora_a_stacked) == len(lora_b_stacked) == len(output_slices)
664 + if lora_bias_stacked is not None:
665 + assert len(lora_bias_stacked) == len(output_slices)
666 + y = self._apply_bias(self.token_lora_indices, y, output_slices,
667 + lora_bias_stacked)
668 +
669 if buffer is None:
670 + r = lora_b_stacked[0].size(-1)
671 # We set the buffer to be float32 by default ,refer to:
672 # https://github.com/triton-lang/triton/issues/1387
635 - buffer = torch.zeros((x.size(0), r),
636 - dtype=torch.float32,
637 - device=x.device)
638 - if bias_all is not None:
639 - y = self.apply_bias(self.token_lora_indices, y, bias_all)
640 - self.add_shrink(buffer, x, wa_t_all, scale)
641 - if y_offset is None and y_slice_size is None:
642 - self.add_expand(y, buffer, wb_t_all, bias_all=None, add_input=True)
643 - else:
644 - self.add_expand_slice(y,
645 - buffer,
646 - wb_t_all,
647 - None,
648 - y_offset,
649 - y_slice_size,
650 - add_input=True)
651 - y = y.view_as(y_org)
652 -
653 - def add_lora_packed_nslice(self, y: torch.Tensor, x: torch.Tensor,
654 - lora_a_stacked: Tuple[torch.Tensor, ...],
655 - lora_b_stacked: Tuple[torch.Tensor, ...],
656 - bias_all: Tuple[Optional[torch.Tensor],
657 - ...], scale: float,
658 - output_slices: Tuple[int, ...]) -> None:
659 - """
660 - Applies lora to each input. Similar to add_lora, This method is
661 - used for layers that are composed of multiple sublayers
662 - (slices) packed together.
663 - """
664 - y_org = y
665 - x = x.view(-1, x.shape[-1])
666 - y = y.view(-1, y.shape[-1])
667 - offset_left = 0
668 - if bias_all is not None:
669 - y = self.apply_bias_packed_nslice(self.token_lora_indices, y,
670 - output_slices, bias_all)
671 - # TODO fuse these kernels
672 - for slice_idx in range(len(output_slices)):
673 - self.add_lora(y, x, lora_a_stacked[slice_idx],
674 - lora_b_stacked[slice_idx], None, scale, offset_left,
675 - output_slices[slice_idx])
676 - offset_left += output_slices[slice_idx]
677 -
678 - y = y.view_as(y_org)
673 + buffer = tuple(
674 + torch.zeros(
675 + (x.size(0), r), dtype=torch.float32, device=x.device)
676 + for _ in range(len(output_slices)))
677 + self.add_shrink(buffer, x, lora_a_stacked, scale)
678 + self.add_expand(y,
679 + buffer,
680 + lora_b_stacked,
681 + None,
682 + output_slices,
683 + add_input=True)
684
685 def add_lora_logits(self,
686 y: torch.Tensor,
687 x: torch.Tensor,
683 - wa_t_all: torch.Tensor,
684 - wb_t_all: torch.Tensor,
688 + lora_a_stacked: torch.Tensor,
689 + lora_b_stacked: torch.Tensor,
690 scale,
691 *,
692 buffer: Optional[torch.Tensor] = None) -> None:
693 """
689 - LogitsProcessorWithLoRA always using bgmv
690 - """
694 + Applies lora specifically for LogitsProcessorWithLoRA.
695 +
696 + Semantics:
697 + buffer = (x @ lora_a_stacked) * scale
698 + y += buffer @ lora_b_stacked
699 +
700 + Args:
701 + y (torch.Tensor): Output tensor.
702 + x (torch.Tensor): Input tensor.
703 + lora_a_stacked (torch.Tensor): lora_a's weights.
704 + lora_b_stacked (torch.Tensor):lora_b's weights.
705 + scale (float): Scaling factor.
706 + buffer (Optional[torch.Tensor]):Default to None.
707 + """
708 y_org = y
709 y = y.view(-1, y.shape[-1])
710 x = x.view(-1, x.shape[-1])
694 - r = wb_t_all.size(-1)
711 + r = lora_b_stacked.size(-1)
712 if buffer is None:
713 # We set the buffer to be float32 by default ,refer to:
714 # https://github.com/triton-lang/triton/issues/1387
715 buffer = torch.zeros((x.size(0), r),
716 dtype=torch.float32,
717 device=x.device)
701 -
702 - bgmv_shrink(x, wa_t_all, buffer, self.sampler_indices, scale)
703 - bgmv_expand(buffer, wb_t_all, y, self.sampler_indices, add_inputs=True)
718 + # LogitsProcessorWithLoRA always using bgmv.
719 + bgmv_shrink(x, lora_a_stacked, buffer, self.sampler_indices, scale)
720 + bgmv_expand(buffer,
721 + lora_b_stacked,
722 + y,
723 + self.sampler_indices,
724 + add_inputs=True)
725 y = y.view_as(y_org)