A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Kernel] Full Tensor Parallelism for LoRA Layers (#3524)
Co-authored-by: Antoni Baum <antoni.baum@protonmail.com>
Austin Veselka committed
Apr 27, 2024 at 02:03 UTC
eefeb16464af5f3a61e3052d1a4128480bff7f47
19 files changed
+686
-111
csrc/punica/bgmv/bgmv_bf16_bf16_bf16.cu
+1
@@ -2,3 +2,4 @@
2
#include "bgmv_impl.cuh"
3
4
FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_bfloat16, nv_bfloat16, nv_bfloat16)
5
+FOR_INST_BGMV_WIDE_NARROW(INST_BGMV_ONESIDE, nv_bfloat16, nv_bfloat16, nv_bfloat16)
csrc/punica/bgmv/bgmv_bf16_fp32_bf16.cu
+1
@@ -2,3 +2,4 @@
2
#include "bgmv_impl.cuh"
3
4
FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_bfloat16, float, nv_bfloat16)
5
+FOR_INST_BGMV_WIDE_NARROW(INST_BGMV_ONESIDE, nv_bfloat16, float, nv_bfloat16)
csrc/punica/bgmv/bgmv_config.h
+78
@@ -74,6 +74,74 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
74
// Keep above in sync with vllm/lora/layers::LogitsProcessorWithLoRA
75
// and vllm/tests/lora/test_punica.py
76
77
+// Used for defining kernels going from the variety of
78
+// dim in to the narrow dim out
79
+ // Using it for the fully sharded column
80
+ // parallel LoRA A which splits the rank dim
81
+#define FOR_INST_BGMV_NARROW(f, in_T, out_T, W_T, narrow) \
82
+ f(in_T, out_T, W_T, 128, narrow) \
83
+ f(in_T, out_T, W_T, 256, narrow) \
84
+ f(in_T, out_T, W_T, 512, narrow) \
85
+ f(in_T, out_T, W_T, 640, narrow) \
86
+ f(in_T, out_T, W_T, 768, narrow) \
87
+ f(in_T, out_T, W_T, 1024, narrow) \
88
+ f(in_T, out_T, W_T, 1152, narrow) \
89
+ f(in_T, out_T, W_T, 1280, narrow) \
90
+ f(in_T, out_T, W_T, 1536, narrow) \
91
+ f(in_T, out_T, W_T, 1728, narrow) \
92
+ f(in_T, out_T, W_T, 1792, narrow) \
93
+ f(in_T, out_T, W_T, 2048, narrow) \
94
+ f(in_T, out_T, W_T, 2304, narrow) \
95
+ f(in_T, out_T, W_T, 2560, narrow) \
96
+ f(in_T, out_T, W_T, 2752, narrow) \
97
+ f(in_T, out_T, W_T, 2816, narrow) \
98
+ f(in_T, out_T, W_T, 3072, narrow) \
99
+ f(in_T, out_T, W_T, 3456, narrow) \
100
+ f(in_T, out_T, W_T, 3584, narrow) \
101
+ f(in_T, out_T, W_T, 4096, narrow) \
102
+ f(in_T, out_T, W_T, 4608, narrow) \
103
+ f(in_T, out_T, W_T, 5120, narrow) \
104
+ f(in_T, out_T, W_T, 5504, narrow) \
105
+ f(in_T, out_T, W_T, 5632, narrow) \
106
+ f(in_T, out_T, W_T, 6144, narrow) \
107
+ f(in_T, out_T, W_T, 6848, narrow) \
108
+ f(in_T, out_T, W_T, 6912, narrow) \
109
+ f(in_T, out_T, W_T, 7168, narrow) \
110
+ f(in_T, out_T, W_T, 8192, narrow) \
111
+ f(in_T, out_T, W_T, 9216, narrow) \
112
+ f(in_T, out_T, W_T, 10240, narrow) \
113
+ f(in_T, out_T, W_T, 11008, narrow) \
114
+ f(in_T, out_T, W_T, 12288, narrow) \
115
+ f(in_T, out_T, W_T, 13696, narrow) \
116
+ f(in_T, out_T, W_T, 13824, narrow) \
117
+ f(in_T, out_T, W_T, 14336, narrow) \
118
+ f(in_T, out_T, W_T, 15360, narrow) \
119
+ f(in_T, out_T, W_T, 16384, narrow) \
120
+ f(in_T, out_T, W_T, 20480, narrow) \
121
+ f(in_T, out_T, W_T, 22016, narrow) \
122
+ f(in_T, out_T, W_T, 24576, narrow) \
123
+ f(in_T, out_T, W_T, 27392, narrow) \
124
+ f(in_T, out_T, W_T, 28672, narrow) \
125
+ f(in_T, out_T, W_T, 32000, narrow) \
126
+ f(in_T, out_T, W_T, 32256, narrow) \
127
+ f(in_T, out_T, W_T, 32512, narrow) \
128
+ f(in_T, out_T, W_T, 32768, narrow) \
129
+ f(in_T, out_T, W_T, 33024, narrow) \
130
+ f(in_T, out_T, W_T, 36864, narrow) \
131
+ f(in_T, out_T, W_T, 43264, narrow) \
132
+ f(in_T, out_T, W_T, 49152, narrow) \
133
+ f(in_T, out_T, W_T, 64000, narrow) \
134
+ f(in_T, out_T, W_T, 64256, narrow) \
135
+ f(in_T, out_T, W_T, 64512, narrow) \
136
+ f(in_T, out_T, W_T, 102400, narrow) \
137
+ f(in_T, out_T, W_T, 102656, narrow) \
138
+ f(in_T, out_T, W_T, 102912, narrow) \
139
+ f(in_T, out_T, W_T, 128000, narrow) \
140
+ f(in_T, out_T, W_T, 128256, narrow) \
141
+ f(in_T, out_T, W_T, 128512, narrow) \
142
+// Keep above in sync with vllm/lora/layers::SamplerWithLoRA
143
+
144
+
145
// Keep this in sync with vllm/config::LoRAConfig
146
#define FOR_BGMV_WIDE_NARROW(f, in_T, out_T, W_T) \
147
FOR_BGMV_WIDE(f, in_T, out_T, W_T, 8) \
@@ -81,4 +149,14 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
149
FOR_BGMV_WIDE(f, in_T, out_T, W_T, 32) \
150
FOR_BGMV_WIDE(f, in_T, out_T, W_T, 64)
151
152
+
153
+#define FOR_INST_BGMV_WIDE_NARROW(f, in_T, out_T, W_T) \
154
+ FOR_INST_BGMV_NARROW(f, in_T, out_T, W_T, 1) \
155
+ FOR_INST_BGMV_NARROW(f, in_T, out_T, W_T, 2) \
156
+ FOR_INST_BGMV_NARROW(f, in_T, out_T, W_T, 4) \
157
+ f(in_T, out_T, W_T, 8, 64) \
158
+ f(in_T, out_T, W_T, 16, 64) \
159
+ f(in_T, out_T, W_T, 32, 64) \
160
+ f(in_T, out_T, W_T, 64, 64)
161
+
162
// clang-format on
csrc/punica/bgmv/bgmv_fp16_fp16_fp16.cu
+1
@@ -2,3 +2,4 @@
2
#include "bgmv_impl.cuh"
3
4
FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_half, nv_half, nv_half)
5
+FOR_INST_BGMV_WIDE_NARROW(INST_BGMV_ONESIDE, nv_half, nv_half, nv_half)
csrc/punica/bgmv/bgmv_fp16_fp32_fp16.cu
+1
@@ -2,3 +2,4 @@
2
#include "bgmv_impl.cuh"
3
4
FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_half, float, nv_half)
5
+FOR_INST_BGMV_WIDE_NARROW(INST_BGMV_ONESIDE, nv_half, float, nv_half)
csrc/punica/bgmv/bgmv_fp32_bf16_bf16.cu
+1
@@ -2,3 +2,4 @@
2
#include "bgmv_impl.cuh"
3
4
FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, float, nv_bfloat16, nv_bfloat16)
5
+FOR_INST_BGMV_WIDE_NARROW(INST_BGMV_ONESIDE, float, nv_bfloat16, nv_bfloat16)
csrc/punica/bgmv/bgmv_fp32_fp16_fp16.cu
+1
@@ -2,3 +2,4 @@
2
#include "bgmv_impl.cuh"
3
4
FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, float, nv_half, nv_half)
5
+FOR_INST_BGMV_WIDE_NARROW(INST_BGMV_ONESIDE, float, nv_half, nv_half)
csrc/punica/bgmv/bgmv_impl.cuh
+4
-1
@@ -199,7 +199,7 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
199
constexpr int tz = 4;
200
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
201
202
- if constexpr (feat_in < feat_out) {
202
+ if constexpr (feat_in <= feat_out) {
203
static_assert(feat_in % vec_size == 0);
204
constexpr int tx = feat_in / vec_size;
205
@@ -289,6 +289,9 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
289
int64_t y_offset, int64_t full_y_size, int64_t batch_size, \
290
int64_t num_layers, int64_t layer_idx, float scale);
291
292
+#define INST_BGMV_ONESIDE(in_T, out_T, W_T, feat_in, feat_out) \
293
+ INST_BGMV(feat_in, feat_out, in_T, out_T, W_T)
294
+
295
#define INST_BGMV_TWOSIDE(in_T, out_T, W_T, narrow, wide) \
296
INST_BGMV(narrow, wide, in_T, out_T, W_T) \
297
INST_BGMV(wide, narrow, in_T, out_T, W_T)
csrc/punica/bgmv/generator.py
+1
@@ -10,6 +10,7 @@ TEMPLATE = """
10
#include "bgmv_impl.cuh"
11
12
FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, {input_dtype}, {output_dtype}, {weight_dtype})
13
+FOR_INST_BGMV_WIDE_NARROW(INST_BGMV_ONESIDE, {input_dtype}, {output_dtype}, {weight_dtype})
14
""".lstrip() # noqa: E501
15
16
for input_dtype in DTYPES:
csrc/punica/punica_ops.cc
+1
-1
@@ -79,12 +79,12 @@ inline bool launch_bgmv_kernel(out_T *Y, const in_T *X, const W_T *W,
79
CASE_ONESIDE(in_T, out_T, W_T, wide, narrow)
80
81
FOR_BGMV_WIDE_NARROW(CASE, _, _, _)
82
+ FOR_INST_BGMV_WIDE_NARROW(CASE_ONESIDE, _, _, _)
83
#undef CASE
84
#undef CASE_ONESIDE
85
default:
86
return false;
87
}
87
-
88
return true;
89
}
90
tests/lora/test_layers.py
+23
-6
@@ -8,6 +8,10 @@ import torch
8
import torch.nn.functional as F
9
10
from vllm.config import LoRAConfig
11
+from vllm.lora.fully_sharded_layers import (
12
+ ColumnParallelLinearWithShardedLoRA,
13
+ MergedColumnParallelLinearWithShardedLoRA,
14
+ MergedQKVParallelLinearWithShardedLora, RowParallelLinearWithShardedLoRA)
15
# yapf conflicts with isort for this block
16
# yapf: disable
17
from vllm.lora.layers import (BaseLayerWithLoRA, ColumnParallelLinearWithLoRA,
@@ -524,13 +528,16 @@ def test_lm_head_logits_processor(dist_init, num_loras, device,
528
@torch.inference_mode()
529
@pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
530
@pytest.mark.parametrize("orientation", ["row", "column"])
531
+@pytest.mark.parametrize("fully_shard", [True, False])
532
@pytest.mark.parametrize("device", CUDA_DEVICES)
528
-def test_linear_parallel(dist_init, num_loras, orientation, device) -> None:
533
+def test_linear_parallel(dist_init, num_loras, orientation, fully_shard,
534
+ device) -> None:
535
536
torch.set_default_device(device)
537
max_loras = 8
538
lora_config = LoRAConfig(max_loras=max_loras,
539
max_lora_rank=8,
540
+ fully_sharded_loras=fully_shard,
541
lora_dtype=torch.float16)
542
543
def create_random_linear_parallel_layer():
@@ -540,14 +547,17 @@ def test_linear_parallel(dist_init, num_loras, orientation, device) -> None:
547
bias=False,
548
params_dtype=torch.float16)
549
linear.weight.data = torch.rand_like(linear.weight.data)
543
- lora_linear = RowParallelLinearWithLoRA(linear)
550
+ lora_linear = (RowParallelLinearWithLoRA(linear) if not fully_shard
551
+ else RowParallelLinearWithShardedLoRA(linear))
552
else:
553
linear = ColumnParallelLinear(4096,
554
4096,
555
bias=False,
556
params_dtype=torch.float16)
557
linear.weight.data = torch.rand_like(linear.weight.data)
550
- lora_linear = ColumnParallelLinearWithLoRA(linear)
558
+ lora_linear = (ColumnParallelLinearWithLoRA(linear)
559
+ if not fully_shard else
560
+ ColumnParallelLinearWithShardedLoRA(linear))
561
lora_linear.create_lora_weights(max_loras, lora_config)
562
563
return linear, lora_linear
@@ -629,13 +639,16 @@ def test_linear_parallel(dist_init, num_loras, orientation, device) -> None:
639
@torch.inference_mode()
640
@pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
641
@pytest.mark.parametrize("repeats", [1, 2, 3])
642
+@pytest.mark.parametrize("fully_shard", [True, False])
643
@pytest.mark.parametrize("device", CUDA_DEVICES)
633
-def test_column_parallel_packed(dist_init, num_loras, repeats, device) -> None:
644
+def test_column_parallel_packed(dist_init, num_loras, repeats, fully_shard,
645
+ device) -> None:
646
647
torch.set_default_device(device)
648
max_loras = 8
649
lora_config = LoRAConfig(max_loras=max_loras,
650
max_lora_rank=8,
651
+ fully_sharded_loras=fully_shard,
652
lora_dtype=torch.float16)
653
654
def create_column_parallel_packed_layer():
@@ -644,7 +657,9 @@ def test_column_parallel_packed(dist_init, num_loras, repeats, device) -> None:
657
bias=False,
658
params_dtype=torch.float16)
659
linear.weight.data = torch.rand_like(linear.weight.data)
647
- lora_linear = MergedColumnParallelLinearWithLoRA(linear)
660
+ lora_linear = (MergedColumnParallelLinearWithLoRA(linear)
661
+ if not fully_shard else
662
+ MergedColumnParallelLinearWithShardedLoRA(linear))
663
elif repeats == 3:
664
linear = QKVParallelLinear(4096,
665
64,
@@ -652,7 +667,9 @@ def test_column_parallel_packed(dist_init, num_loras, repeats, device) -> None:
667
bias=False,
668
params_dtype=torch.float16)
669
linear.weight.data = torch.rand_like(linear.weight.data)
655
- lora_linear = MergedQKVParallelLinearWithLora(linear)
670
+ lora_linear = (MergedQKVParallelLinearWithLora(linear)
671
+ if not fully_shard else
672
+ MergedQKVParallelLinearWithShardedLora(linear))
673
else:
674
linear = QKVParallelLinear(4096,
675
64,
tests/lora/test_punica.py
+49
-2
@@ -34,11 +34,14 @@ def _lora_ref_impl(
34
for i, lora_idx in zip(range(bs), indicies.cpu().tolist()):
35
xi = x[i].unsqueeze(0).to(torch.float32)
36
wa = wa_T_all[lora_idx, layer_idx].transpose(-1, -2).to(torch.float32)
37
- wb = wb_T_all[lora_idx, layer_idx].transpose(-1, -2).to(torch.float32)
37
+ if wb_T_all is not None:
38
+ wb = wb_T_all[lora_idx, layer_idx].transpose(-1,
39
+ -2).to(torch.float32)
40
41
tmp = xi @ wa
42
y_stage_1[i] = tmp.squeeze(0)
41
- y_final[i] += (tmp @ wb).squeeze(0) * s
43
+ y_final[i] += ((tmp @ wb).squeeze(0) *
44
+ s if wb_T_all is not None else y_stage_1[i])
45
return y_final, y_stage_1
46
47
@@ -91,12 +94,56 @@ H1 = H2 = [
94
128000,
95
128256,
96
]
97
+H2 = [64] + H2
98
+R = [1, 2, 4]
99
SEED = [0xabcdabcd987]
100
CUDA_DEVICES = [
101
f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
102
]
103
104
105
+@pytest.mark.parametrize("dtype_str", ["float16", "bfloat16"])
106
+@pytest.mark.parametrize("h1", H1)
107
+@pytest.mark.parametrize("r", R)
108
+@pytest.mark.parametrize("seed", SEED)
109
+@torch.inference_mode()
110
+def test_lora_a_extra_shapes(dtype_str, h1, r, seed):
111
+ torch.manual_seed(seed)
112
+ num_loras = 4
113
+ num_layers = 1
114
+ bs = 32
115
+ dtype = getattr(torch, dtype_str)
116
+ device = torch.device("cuda")
117
+
118
+ wa_T_all = torch.randn(num_loras,
119
+ num_layers,
120
+ r,
121
+ h1,
122
+ dtype=dtype,
123
+ device=device)
124
+ indices = torch.randint(num_loras, (bs, ), dtype=torch.long, device=device)
125
+
126
+ for layer_idx in range(num_layers):
127
+ x = torch.randn(bs, h1, dtype=dtype, device=device)
128
+ y = torch.randn(bs, r, dtype=dtype, device=device)
129
+
130
+ y_ref = y.clone()
131
+ _lora_ref_impl(
132
+ y_ref,
133
+ x,
134
+ wa_T_all,
135
+ None,
136
+ indices,
137
+ layer_idx,
138
+ 1.0,
139
+ )
140
+
141
+ y_our = y.clone()
142
+ punica.bgmv(y_our, x, wa_T_all, indices, layer_idx, 1.0)
143
+
144
+ assert_close(y_ref, y_our)
145
+
146
+
147
@pytest.mark.parametrize("dtype_str", ["float16", "bfloat16"])
148
@pytest.mark.parametrize("h1", H1)
149
@pytest.mark.parametrize("h2", H2)
vllm/config.py
+1
@@ -862,6 +862,7 @@ class SpeculativeConfig:
862
class LoRAConfig:
863
max_lora_rank: int
864
max_loras: int
865
+ fully_sharded_loras: bool = False
866
max_cpu_loras: Optional[int] = None
867
lora_dtype: Optional[torch.dtype] = None
868
lora_extra_vocab_size: int = 256
vllm/engine/arg_utils.py
+10
@@ -52,6 +52,7 @@ class EngineArgs:
52
enable_lora: bool = False
53
max_loras: int = 1
54
max_lora_rank: int = 16
55
+ fully_sharded_loras: bool = False
56
lora_extra_vocab_size: int = 256
57
lora_dtype = 'auto'
58
max_cpu_loras: Optional[int] = None
@@ -376,6 +377,14 @@ class EngineArgs:
377
help=('Maximum number of LoRAs to store in CPU memory. '
378
'Must be >= than max_num_seqs. '
379
'Defaults to max_num_seqs.'))
380
+ parser.add_argument(
381
+ '--fully-sharded-loras',
382
+ action='store_true',
383
+ help=('By default, only half of the LoRA computation is '
384
+ 'sharded with tensor parallelism. '
385
+ 'Enabling this will use the fully sharded layers. '
386
+ 'At high sequence length, max rank or '
387
+ 'tensor parallel size, this is likely faster.'))
388
parser.add_argument("--device",
389
type=str,
390
default=EngineArgs.device,
@@ -509,6 +518,7 @@ class EngineArgs:
518
lora_config = LoRAConfig(
519
max_lora_rank=self.max_lora_rank,
520
max_loras=self.max_loras,
521
+ fully_sharded_loras=self.fully_sharded_loras,
522
lora_extra_vocab_size=self.lora_extra_vocab_size,
523
lora_dtype=self.lora_dtype,
524
max_cpu_loras=self.max_cpu_loras if self.max_cpu_loras
vllm/lora/fully_sharded_layers.py
new
+262
@@ -0,0 +1,262 @@
1
+# pylint: disable=unused-argument
2
+from typing import TYPE_CHECKING, List, Optional
3
+
4
+import torch
5
+import torch.nn as nn
6
+from transformers import PretrainedConfig
7
+
8
+from vllm.config import LoRAConfig
9
+from vllm.distributed.communication_op import (
10
+ tensor_model_parallel_all_gather, tensor_model_parallel_all_reduce)
11
+from vllm.distributed.parallel_state import get_tensor_model_parallel_rank
12
+from vllm.lora.layers import (ColumnParallelLinearWithLoRA,
13
+ MergedColumnParallelLinearWithLoRA,
14
+ MergedQKVParallelLinearWithLora,
15
+ RowParallelLinearWithLoRA)
16
+from vllm.lora.punica import bgmv, dispatch_bgmv_low_level
17
+
18
+if TYPE_CHECKING:
19
+ pass
20
+
21
+
22
+def _fully_sharded_can_replace(can_replace):
23
+ """
24
+ decorator which adds the condition of fully sharded loras
25
+ intended to wrap can_replace_layer()
26
+ """
27
+
28
+ def dec(*args, **kwargs):
29
+ return (can_replace(*args, **kwargs)
30
+ and kwargs['lora_config'].fully_sharded_loras)
31
+
32
+ return dec
33
+
34
+
35
+# these layers are based on the tensor parallelism strategy given in
36
+# Y. Sheng et al., S-LoRA: Serving Thousands of Concurrent LoRA Adapters. 2023,
37
+# https://arxiv.org/abs/2311.03285.
38
+
39
+
40
+class ColumnParallelLinearWithShardedLoRA(ColumnParallelLinearWithLoRA):
41
+ """
42
+ Differs from ColumnParallelLinearWithLoRA by slicing LoRA A also.
43
+
44
+ Based on S-LoRA, slicing happens along the rank dim.
45
+ """
46
+
47
+ def slice_lora_a(self, lora_a: torch.Tensor) -> torch.Tensor:
48
+ tp_rank = get_tensor_model_parallel_rank()
49
+ shard_size = self.lora_a_stacked.shape[2]
50
+ start_idx = tp_rank * shard_size
51
+ lora_a = lora_a[:, start_idx:start_idx + shard_size]
52
+ return lora_a
53
+
54
+ def apply_weights(self, x: torch.Tensor,
55
+ bias: Optional[torch.Tensor]) -> torch.Tensor:
56
+ output = self.base_layer.linear_method.apply_weights(
57
+ self.base_layer, x, bias)
58
+
59
+ x = x.view(-1, x.shape[-1])
60
+ output, out_orig_shape = output.view(-1,
61
+ output.shape[-1]), output.shape
62
+ buffer = torch.zeros((x.shape[0], self.lora_a_stacked.shape[2]),
63
+ dtype=torch.float32,
64
+ device=x.device)
65
+
66
+ bgmv(buffer, x, self.lora_a_stacked,
67
+ self.indices[:self.indices_len[0]], 0, 1.0)
68
+ buffer = tensor_model_parallel_all_gather(buffer)
69
+ bgmv(output, buffer, self.lora_b_stacked,
70
+ self.indices[:self.indices_len[0]], 0, 1.0)
71
+ # now have column partitioned output
72
+
73
+ output = output.view(*out_orig_shape)
74
+ return output
75
+
76
+ @classmethod
77
+ @_fully_sharded_can_replace
78
+ def can_replace_layer(cls, source_layer: nn.Module,
79
+ lora_config: LoRAConfig, packed_modules_list: List,
80
+ model_config: Optional[PretrainedConfig]) -> bool:
81
+ # specifying kwargs so they can be easily accessed in decorator
82
+ return super().can_replace_layer(
83
+ source_layer=source_layer,
84
+ lora_config=lora_config,
85
+ packed_modules_list=packed_modules_list,
86
+ model_config=model_config,
87
+ decorate=False,
88
+ )
89
+
90
+
91
+def _mcp_apply_weights(x, bias, layer):
92
+ """
93
+ MergedColumnParallelLinearWithShardedLoRA and
94
+ QKVParallelLinearWithShardedLora share the same
95
+ LoRa weight application method.
96
+
97
+ The main difference is the step by shard_size for lora_b which can
98
+ vary for QKVParallelLinearWithShardedLora but is constant for
99
+ MergedColumnParallelLinearWithShardedLoRA.
100
+ """
101
+ # expecting 2 for column parallel and 3 for qkv
102
+ n = len(layer.lora_a_stacked)
103
+ output = layer.base_layer.linear_method.apply_weights(
104
+ layer.base_layer, x, bias)
105
+
106
+ x = x.view(-1, x.shape[-1])
107
+ output, out_orig_shape = output.view(-1, output.shape[-1]), output.shape
108
+ buffers = torch.zeros((n, x.shape[0], layer.lora_a_stacked[0].shape[2]),
109
+ dtype=torch.float32,
110
+ device=x.device)
111
+ for idx in range(n):
112
+ bgmv(buffers[idx], x, layer.lora_a_stacked[idx],
113
+ layer.indices[:layer.indices_len[0]], 0, 1.0)
114
+
115
+ buffers = tensor_model_parallel_all_gather(buffers)
116
+ left_offset = 0
117
+ for idx in range(n):
118
+ shard_size = layer.lora_b_stacked[idx].shape[2]
119
+ dispatch_bgmv_low_level(output, buffers[idx],
120
+ layer.lora_b_stacked[idx],
121
+ layer.indices[:layer.indices_len[0]], 0, 1.0,
122
+ left_offset, shard_size)
123
+ left_offset += shard_size
124
+
125
+ output = output.view(*out_orig_shape)
126
+ # now have column partitioned and packed output
127
+ return output
128
+
129
+
130
+class MergedColumnParallelLinearWithShardedLoRA(
131
+ MergedColumnParallelLinearWithLoRA):
132
+ """
133
+ Differs from MergedColumnParallelLinearWithLoRA by slicing the
134
+ LoRA A's also.
135
+
136
+ Based on S-LoRA, slicing happens along the rank dim.
137
+ """
138
+
139
+ def slice_lora_a(self, lora_a: List[torch.Tensor]) -> List[torch.Tensor]:
140
+ output_shard_size = self.lora_a_stacked[0].shape[2]
141
+ output_start_idx = self.tp_rank * output_shard_size
142
+ lora_a = [
143
+ lora_a[i][:, output_start_idx:output_start_idx + output_shard_size]
144
+ for i in range(2)
145
+ ]
146
+ return lora_a
147
+
148
+ def apply_weights(self, x: torch.Tensor,
149
+ bias: Optional[torch.Tensor]) -> torch.Tensor:
150
+ return _mcp_apply_weights(x, bias, self)
151
+
152
+ @classmethod
153
+ @_fully_sharded_can_replace
154
+ def can_replace_layer(cls, source_layer: nn.Module,
155
+ lora_config: LoRAConfig, packed_modules_list: List,
156
+ model_config: Optional[PretrainedConfig]) -> bool:
157
+ # specifying kwargs so they can be easily accessed in decorator
158
+ return super().can_replace_layer(
159
+ source_layer=source_layer,
160
+ lora_config=lora_config,
161
+ packed_modules_list=packed_modules_list,
162
+ model_config=model_config,
163
+ decorate=False,
164
+ )
165
+
166
+
167
+class MergedQKVParallelLinearWithShardedLora(MergedQKVParallelLinearWithLora):
168
+ """
169
+ Differs from QKVParallelLinearWithLora by slicing the
170
+ LoRA A's also.
171
+
172
+ Based on S-LoRA, slicing happens along the rank dim.
173
+ """
174
+
175
+ def slice_lora_a(self, lora_a: List[torch.Tensor]) -> List[torch.Tensor]:
176
+ shard_size = [self.lora_a_stacked[i].shape[2] for i in range(3)]
177
+ start_idx = [self.tp_rank * shard_size[i] for i in range(3)]
178
+ lora_a = [
179
+ lora_a[i][:, start_idx[i]:start_idx[i] +
180
+ shard_size[i]] if lora_a[i] is not None else None
181
+ for i in range(3)
182
+ ]
183
+ return lora_a
184
+
185
+ def apply_weights(self, x: torch.Tensor,
186
+ bias: Optional[torch.Tensor]) -> torch.Tensor:
187
+ return _mcp_apply_weights(x, bias, self)
188
+
189
+ @classmethod
190
+ @_fully_sharded_can_replace
191
+ def can_replace_layer(cls, source_layer: nn.Module,
192
+ lora_config: LoRAConfig, packed_modules_list: List,
193
+ model_config: Optional[PretrainedConfig]) -> bool:
194
+ # specifying kwargs so they can be easily accessed in decorator
195
+ return super().can_replace_layer(
196
+ source_layer=source_layer,
197
+ lora_config=lora_config,
198
+ packed_modules_list=packed_modules_list,
199
+ model_config=model_config,
200
+ decorate=False,
201
+ )
202
+
203
+
204
+class RowParallelLinearWithShardedLoRA(RowParallelLinearWithLoRA):
205
+ """
206
+ Differs from RowParallelLinearWithLoRA by slicing the
207
+ LoRA B's also.
208
+
209
+ Based on S-LoRA, slicing happens along the output dim.
210
+ This yields a combined partial sum from the row parallel base
211
+ layer and column partitioned output from the LoRA.
212
+ """
213
+
214
+ def slice_lora_b(self, lora_b: torch.Tensor) -> torch.Tensor:
215
+ shard_size = self.lora_b_stacked.shape[2]
216
+ start_idx = self.tp_rank * shard_size
217
+ end_idx = (self.tp_rank + 1) * shard_size
218
+ lora_b = lora_b[:, start_idx:end_idx]
219
+ return lora_b
220
+
221
+ def apply_weights(self, x: torch.Tensor) -> torch.Tensor:
222
+ output = self.base_layer.linear_method.apply_weights(
223
+ self.base_layer, x)
224
+
225
+ x = x.view(-1, x.shape[-1])
226
+ output, out_orig_shape = output.view(-1,
227
+ output.shape[-1]), output.shape
228
+ buffer = torch.zeros((x.shape[0], self.lora_a_stacked.shape[2]),
229
+ dtype=torch.float32,
230
+ device=x.device)
231
+ bgmv(buffer, x, self.lora_a_stacked,
232
+ self.indices[:self.indices_len[0]], 0, 1.0)
233
+ buffer = tensor_model_parallel_all_reduce(buffer)
234
+
235
+ # following S-LoRA, allows the fusing of all_gather and all_reduce
236
+ # by adding the column partitioned lora output to a slice of output
237
+ # tensor, which is a partial sum due to row parallel. All that
238
+ # remains is a standard all_reduce. User should be aware though that
239
+ # the output is not the same as a normal row_parallel, it should be
240
+ # reduced before being used
241
+ shard_size = self.lora_b_stacked.shape[2]
242
+ start_idx = self.tp_rank * shard_size
243
+ dispatch_bgmv_low_level(output, buffer, self.lora_b_stacked,
244
+ self.indices[:self.indices_len[0]], 0, 1.0,
245
+ start_idx, shard_size)
246
+
247
+ output = output.view(*out_orig_shape)
248
+ return output
249
+
250
+ @classmethod
251
+ @_fully_sharded_can_replace
252
+ def can_replace_layer(cls, source_layer: nn.Module,
253
+ lora_config: LoRAConfig, packed_modules_list: List,
254
+ model_config: Optional[PretrainedConfig]) -> bool:
255
+ # specifying kwargs so they can be easily accessed in decorator
256
+ return super().can_replace_layer(
257
+ source_layer=source_layer,
258
+ lora_config=lora_config,
259
+ packed_modules_list=packed_modules_list,
260
+ model_config=model_config,
261
+ decorate=False,
262
+ )
vllm/lora/layers.py
+146
-97
@@ -1,8 +1,7 @@
1
# pylint: disable=unused-argument
2
-import inspect
2
import math
3
from dataclasses import dataclass
5
-from typing import TYPE_CHECKING, List, Optional, Set, Tuple, Type
4
+from typing import TYPE_CHECKING, List, Optional, Tuple
5
6
import torch
7
import torch.nn as nn
@@ -16,6 +15,7 @@ from vllm.distributed import (get_tensor_model_parallel_rank,
15
tensor_model_parallel_all_gather,
16
tensor_model_parallel_all_reduce,
17
tensor_model_parallel_gather)
18
+from vllm.distributed.utils import divide
19
from vllm.lora.punica import add_lora, add_lora_slice, bgmv
20
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
21
MergedColumnParallelLinear,
@@ -23,7 +23,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
23
RowParallelLinear)
24
from vllm.model_executor.layers.logits_processor import LogitsProcessor
25
from vllm.model_executor.layers.vocab_parallel_embedding import (
26
- ParallelLMHead, VocabParallelEmbedding)
26
+ VocabParallelEmbedding)
27
28
if TYPE_CHECKING:
29
pass
@@ -45,6 +45,21 @@ def _get_lora_device(base_layer: nn.Module) -> torch.device:
45
raise ValueError(f"Unsupported base layer: {base_layer}")
46
47
48
+def _not_fully_sharded_can_replace(can_replace):
49
+ """
50
+ decorator which adds the condition of not using fully sharded loras
51
+ intended to wrap can_replace_layer()
52
+ """
53
+
54
+ def dec(*args, **kwargs):
55
+ decorate = kwargs.pop('decorate') if 'decorate' in kwargs else True
56
+ condition = (not kwargs['lora_config'].fully_sharded_loras
57
+ if decorate else True)
58
+ return can_replace(*args, **kwargs) and condition
59
+
60
+ return dec
61
+
62
+
63
def _apply_lora(
64
x: torch.Tensor,
65
lora_a_stacked: torch.Tensor,
@@ -130,6 +145,14 @@ class LoRAMapping:
145
146
class BaseLayerWithLoRA(nn.Module):
147
148
+ def slice_lora_a(self, lora_a: torch.Tensor) -> torch.Tensor:
149
+ """Slice lora a if splitting for tensor parallelism."""
150
+ ...
151
+
152
+ def slice_lora_b(self, lora_b: torch.Tensor) -> torch.Tensor:
153
+ """Slice lora b if splitting with tensor parallelism."""
154
+ ...
155
+
156
def create_lora_weights(
157
self,
158
max_loras: int,
@@ -317,6 +340,11 @@ class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA):
340
341
342
class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
343
+ """
344
+ LoRA on top of ColumnParallelLinear layer.
345
+
346
+ LoRA B is sliced for tensor parallelism.
347
+ """
348
349
def __init__(self, base_layer: ColumnParallelLinear) -> None:
350
super().__init__()
@@ -331,10 +359,15 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
359
max_loras: int,
360
lora_config: LoRAConfig,
361
model_config: Optional[PretrainedConfig] = None) -> None:
362
+ self.lora_config = lora_config
363
+ self.tp_size = get_tensor_model_parallel_world_size()
364
+ lora_a_output_size_per_partition = (
365
+ lora_config.max_lora_rank if not lora_config.fully_sharded_loras
366
+ else divide(lora_config.max_lora_rank, self.tp_size))
367
self.lora_a_stacked = torch.zeros(
368
max_loras,
369
1,
337
- lora_config.max_lora_rank,
370
+ lora_a_output_size_per_partition,
371
self.input_size,
372
dtype=lora_config.lora_dtype,
373
device=self.device,
@@ -357,6 +390,17 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
390
self.lora_a_stacked[index] = 0
391
self.lora_b_stacked[index] = 0
392
393
+ def slice_lora_a(self, lora_a: torch.Tensor) -> torch.Tensor:
394
+ return lora_a
395
+
396
+ def slice_lora_b(self, lora_b: torch.Tensor) -> torch.Tensor:
397
+ tensor_model_parallel_rank = get_tensor_model_parallel_rank()
398
+ shard_size = self.output_dim
399
+ start_idx = tensor_model_parallel_rank * shard_size
400
+ end_idx = (tensor_model_parallel_rank + 1) * shard_size
401
+ lora_b = lora_b[:, start_idx:end_idx]
402
+ return lora_b
403
+
404
def set_lora(
405
self,
406
index: int,
@@ -365,12 +409,11 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
409
embeddings_tensor: Optional[torch.Tensor],
410
):
411
self.reset_lora(index)
412
+
413
if self.tp_size > 1:
369
- tensor_model_parallel_rank = get_tensor_model_parallel_rank()
370
- shard_size = self.output_dim
371
- start_idx = tensor_model_parallel_rank * shard_size
372
- end_idx = (tensor_model_parallel_rank + 1) * shard_size
373
- lora_b = lora_b[:, start_idx:end_idx]
414
+ lora_a = self.slice_lora_a(lora_a)
415
+ lora_b = self.slice_lora_b(lora_b)
416
+
417
self.lora_a_stacked[index,
418
0, :lora_a.shape[1], :lora_a.shape[0]].copy_(
419
lora_a.T, non_blocking=True)
@@ -426,6 +469,7 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
469
return output, output_bias
470
471
@classmethod
472
+ @_not_fully_sharded_can_replace
473
def can_replace_layer(cls, source_layer: nn.Module,
474
lora_config: LoRAConfig, packed_modules_list: List,
475
model_config: Optional[PretrainedConfig]) -> bool:
@@ -451,6 +495,7 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
495
max_loras: int,
496
lora_config: LoRAConfig,
497
model_config: Optional[PretrainedConfig] = None) -> None:
498
+ self.lora_config = lora_config
499
n_slices = 2
500
if not (len(self.base_layer.output_sizes) == n_slices
501
and self.base_layer.output_sizes[0]
@@ -459,12 +504,17 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
504
"LoRAColumnParallelLinear2Slice requires 2 slices with "
505
"the same size.")
506
self.tp_size = get_tensor_model_parallel_world_size()
507
+ self.tp_rank = get_tensor_model_parallel_rank()
508
+
509
+ lora_a_output_size_per_partition = (
510
+ lora_config.max_lora_rank if not lora_config.fully_sharded_loras
511
+ else divide(lora_config.max_lora_rank, self.tp_size))
512
513
self.lora_a_stacked = tuple(
514
torch.zeros(
515
max_loras,
516
1,
467
- lora_config.max_lora_rank,
517
+ lora_a_output_size_per_partition,
518
self.input_size,
519
dtype=lora_config.lora_dtype,
520
device=self.device,
@@ -489,6 +539,18 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
539
self.lora_b_stacked[0][index] = 0
540
self.lora_b_stacked[1][index] = 0
541
542
+ def slice_lora_a(self, lora_a: List[torch.Tensor]) -> List[torch.Tensor]:
543
+ return lora_a
544
+
545
+ def slice_lora_b(self, lora_b: List[torch.Tensor]) -> List[torch.Tensor]:
546
+ shard_size = self.output_dim
547
+ start_idx = self.tp_rank * shard_size
548
+ end_idx = (self.tp_rank + 1) * shard_size
549
+ lora_b = [
550
+ lora_b[0][:, start_idx:end_idx], lora_b[1][:, start_idx:end_idx]
551
+ ]
552
+ return lora_b
553
+
554
def set_lora(
555
self,
556
index: int,
@@ -499,13 +561,8 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
561
self.reset_lora(index)
562
563
if self.tp_size > 1:
502
- tensor_model_parallel_rank = get_tensor_model_parallel_rank()
503
- shard_size = self.output_dim
504
- start_idx = tensor_model_parallel_rank * shard_size
505
- end_idx = (tensor_model_parallel_rank + 1) * shard_size
506
- lora_b = lora_b[0][:,
507
- start_idx:end_idx], lora_b[1][:,
508
- start_idx:end_idx]
564
+ lora_a = self.slice_lora_a(lora_a)
565
+ lora_b = self.slice_lora_b(lora_b)
566
567
if lora_a[0] is not None:
568
self.lora_a_stacked[0][
@@ -536,6 +593,7 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
593
return output
594
595
@classmethod
596
+ @_not_fully_sharded_can_replace
597
def can_replace_layer(cls, source_layer: nn.Module,
598
lora_config: LoRAConfig, packed_modules_list: List,
599
model_config: Optional[PretrainedConfig]) -> bool:
@@ -627,21 +685,25 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
685
max_loras: int,
686
lora_config: LoRAConfig,
687
model_config: Optional[PretrainedConfig] = None) -> None:
688
+ self.lora_config = lora_config
689
self.tp_size = get_tensor_model_parallel_world_size()
631
- tp_rank = get_tensor_model_parallel_rank()
690
+ self.tp_rank = get_tensor_model_parallel_rank()
691
self.q_proj_shard_size = (self.base_layer.num_heads *
692
self.base_layer.head_size)
693
self.kv_proj_shard_size = (self.base_layer.num_kv_heads *
694
self.base_layer.head_size)
636
- self.q_shard_id = tp_rank
637
- self.kv_shard_id = tp_rank // self.base_layer.num_kv_head_replicas
695
+ self.q_shard_id = self.tp_rank
696
+ self.kv_shard_id = self.tp_rank // self.base_layer.num_kv_head_replicas
697
698
+ lora_a_output_size_per_partition = (
699
+ lora_config.max_lora_rank if not lora_config.fully_sharded_loras
700
+ else divide(lora_config.max_lora_rank, self.tp_size))
701
# q, k, v
702
self.lora_a_stacked = (
703
torch.zeros(
704
max_loras,
705
1,
644
- lora_config.max_lora_rank,
706
+ lora_a_output_size_per_partition,
707
self.input_size,
708
dtype=lora_config.lora_dtype,
709
device=self.device,
@@ -649,7 +711,7 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
711
torch.zeros(
712
max_loras,
713
1,
652
- lora_config.max_lora_rank,
714
+ lora_a_output_size_per_partition,
715
self.input_size,
716
dtype=lora_config.lora_dtype,
717
device=self.device,
@@ -657,7 +719,7 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
719
torch.zeros(
720
max_loras,
721
1,
660
- lora_config.max_lora_rank,
722
+ lora_a_output_size_per_partition,
723
self.input_size,
724
dtype=lora_config.lora_dtype,
725
device=self.device,
@@ -705,6 +767,25 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
767
self.lora_a_stacked[2][index] = 0
768
self.lora_b_stacked[2][index] = 0
769
770
+ def slice_lora_a(self, lora_a: List[torch.Tensor]) -> List[torch.Tensor]:
771
+ return lora_a
772
+
773
+ def slice_lora_b(self, lora_b: List[torch.Tensor]) -> List[torch.Tensor]:
774
+ if lora_b[0] is not None:
775
+ lora_b_q = lora_b[0][:, self.q_proj_shard_size *
776
+ self.q_shard_id:self.q_proj_shard_size *
777
+ (self.q_shard_id + 1)]
778
+ if lora_b[1] is not None:
779
+ lora_b_k = lora_b[1][:, self.kv_proj_shard_size *
780
+ self.kv_shard_id:self.kv_proj_shard_size *
781
+ (self.kv_shard_id + 1)]
782
+ if lora_b[2] is not None:
783
+ lora_b_v = lora_b[2][:, self.kv_proj_shard_size *
784
+ self.kv_shard_id:self.kv_proj_shard_size *
785
+ (self.kv_shard_id + 1)]
786
+ lora_b = [lora_b_q, lora_b_k, lora_b_v]
787
+ return lora_b
788
+
789
def set_lora(
790
self,
791
index: int,
@@ -715,40 +796,24 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
796
self.reset_lora(index)
797
798
if self.tp_size > 1:
718
- if lora_b[0] is not None:
719
- lora_b_q = lora_b[0][:, self.q_proj_shard_size *
720
- self.q_shard_id:self.q_proj_shard_size *
721
- (self.q_shard_id + 1)]
722
- self.lora_b_stacked[0][
723
- index, 0, :lora_b_q.shape[1], :lora_b_q.shape[0]].copy_(
724
- lora_b_q.T, non_blocking=True)
725
- if lora_b[1] is not None:
726
- lora_b_k = lora_b[1][:, self.kv_proj_shard_size *
727
- self.kv_shard_id:self.kv_proj_shard_size *
728
- (self.kv_shard_id + 1)]
729
- self.lora_b_stacked[1][
730
- index, 0, :lora_b_k.shape[1], :lora_b_k.shape[0]].copy_(
731
- lora_b_k.T, non_blocking=True)
732
- if lora_b[2] is not None:
733
- lora_b_v = lora_b[2][:, self.kv_proj_shard_size *
734
- self.kv_shard_id:self.kv_proj_shard_size *
735
- (self.kv_shard_id + 1)]
736
- self.lora_b_stacked[2][
737
- index, 0, :lora_b_v.shape[1], :lora_b_v.shape[0]].copy_(
738
- lora_b_v.T, non_blocking=True)
739
- else:
740
- if lora_b[0] is not None:
741
- self.lora_b_stacked[0][
742
- index, 0, :lora_b[0].shape[1], :lora_b[0].shape[0]].copy_(
743
- lora_b[0].T, non_blocking=True)
744
- if lora_b[1] is not None:
745
- self.lora_b_stacked[1][
746
- index, 0, :lora_b[1].shape[1], :lora_b[1].shape[0]].copy_(
747
- lora_b[1].T, non_blocking=True)
748
- if lora_b[2] is not None:
749
- self.lora_b_stacked[2][
750
- index, 0, :lora_b[2].shape[1], :lora_b[2].shape[0]].copy_(
751
- lora_b[2].T, non_blocking=True)
799
+ lora_a = self.slice_lora_a(lora_a)
800
+ lora_b = self.slice_lora_b(lora_b)
801
+
802
+ if lora_b[0] is not None:
803
+ lora_b_q = lora_b[0]
804
+ self.lora_b_stacked[0][
805
+ index, 0, :lora_b_q.shape[1], :lora_b_q.shape[0]].copy_(
806
+ lora_b_q.T, non_blocking=True)
807
+ if lora_b[1] is not None:
808
+ lora_b_k = lora_b[1]
809
+ self.lora_b_stacked[1][
810
+ index, 0, :lora_b_k.shape[1], :lora_b_k.shape[0]].copy_(
811
+ lora_b_k.T, non_blocking=True)
812
+ if lora_b[2] is not None:
813
+ lora_b_v = lora_b[2]
814
+ self.lora_b_stacked[2][
815
+ index, 0, :lora_b_v.shape[1], :lora_b_v.shape[0]].copy_(
816
+ lora_b_v.T, non_blocking=True)
817
818
if lora_a[0] is not None:
819
self.lora_a_stacked[0][
@@ -777,6 +842,7 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
842
return output
843
844
@classmethod
845
+ @_not_fully_sharded_can_replace
846
def can_replace_layer(cls, source_layer: nn.Module,
847
lora_config: LoRAConfig, packed_modules_list: List,
848
model_config: Optional[PretrainedConfig]) -> bool:
@@ -798,6 +864,8 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
864
max_loras: int,
865
lora_config: LoRAConfig,
866
model_config: Optional[PretrainedConfig] = None) -> None:
867
+ self.lora_config = lora_config
868
+ self.tp_rank = get_tensor_model_parallel_rank()
869
self.lora_a_stacked = torch.zeros(
870
(
871
max_loras,
@@ -808,11 +876,16 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
876
dtype=lora_config.lora_dtype,
877
device=self.device,
878
)
879
+ tp_size = get_tensor_model_parallel_world_size()
880
+ lora_b_output_size_per_partition = (
881
+ self.output_size if not lora_config.fully_sharded_loras else
882
+ divide(self.output_size, tp_size))
883
+
884
self.lora_b_stacked = torch.zeros(
885
(
886
max_loras,
887
1,
815
- self.output_size,
888
+ lora_b_output_size_per_partition,
889
lora_config.max_lora_rank,
890
),
891
dtype=lora_config.lora_dtype,
@@ -826,6 +899,17 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
899
self.lora_a_stacked[index] = 0
900
self.lora_b_stacked[index] = 0
901
902
+ def slice_lora_a(self, lora_a: torch.Tensor) -> torch.Tensor:
903
+ tensor_model_parallel_rank = get_tensor_model_parallel_rank()
904
+ shard_size = self.input_size
905
+ start_idx = tensor_model_parallel_rank * shard_size
906
+ end_idx = (tensor_model_parallel_rank + 1) * shard_size
907
+ lora_a = lora_a[start_idx:end_idx, :]
908
+ return lora_a
909
+
910
+ def slice_lora_b(self, lora_b: torch.Tensor) -> torch.Tensor:
911
+ return lora_b
912
+
913
def set_lora(
914
self,
915
index: int,
@@ -834,12 +918,10 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
918
embeddings_tensor: Optional[torch.Tensor],
919
):
920
self.reset_lora(index)
921
+
922
if self.base_layer.tp_size > 1:
838
- tensor_model_parallel_rank = get_tensor_model_parallel_rank()
839
- shard_size = self.input_size
840
- start_idx = tensor_model_parallel_rank * shard_size
841
- end_idx = (tensor_model_parallel_rank + 1) * shard_size
842
- lora_a = lora_a[start_idx:end_idx, :]
923
+ lora_a = self.slice_lora_a(lora_a)
924
+ lora_b = self.slice_lora_b(lora_b)
925
926
self.lora_a_stacked[index,
927
0, :lora_a.shape[1], :lora_a.shape[0]].copy_(
@@ -915,6 +997,7 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
997
self.base_layer, "weight") else self.base_layer.qweight
998
999
@classmethod
1000
+ @_not_fully_sharded_can_replace
1001
def can_replace_layer(cls, source_layer: nn.Module,
1002
lora_config: LoRAConfig, packed_modules_list: List,
1003
model_config: Optional[PretrainedConfig]) -> bool:
@@ -1096,37 +1179,3 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
1179
model_config: Optional[PretrainedConfig]) -> bool:
1180
# Special handling for the LogitsProcessor.
1181
return False
1099
-
1100
-
1101
-_all_lora_classes: Set[Type[BaseLayerWithLoRA]] = {
1102
- cls
1103
- for cls in globals().values() if inspect.isclass(cls)
1104
- and issubclass(cls, BaseLayerWithLoRA) and cls is not BaseLayerWithLoRA
1105
-}
1106
-
1107
-
1108
-def from_layer(layer: nn.Module,
1109
- max_loras: int,
1110
- lora_config: LoRAConfig,
1111
- packed_modules_list: List,
1112
- model_config: Optional[PretrainedConfig] = None) -> nn.Module:
1113
- for lora_cls in _all_lora_classes:
1114
- if lora_cls.can_replace_layer(layer, lora_config, packed_modules_list,
1115
- model_config):
1116
- ret = lora_cls(layer)
1117
- ret.create_lora_weights(max_loras, lora_config, model_config)
1118
- return ret
1119
- return layer
1120
-
1121
-
1122
-def from_layer_logits_processor(
1123
- layer: LogitsProcessor,
1124
- lm_head: ParallelLMHead,
1125
- max_loras: int,
1126
- lora_config: LoRAConfig,
1127
- model_config: Optional[PretrainedConfig] = None,
1128
-) -> LogitsProcessorWithLoRA:
1129
- ret = LogitsProcessorWithLoRA(layer, lm_head.embedding_dim,
1130
- lm_head.weight.dtype, lm_head.weight.device)
1131
- ret.create_lora_weights(max_loras, lora_config, model_config)
1132
- return ret
vllm/lora/models.py
+3
-3
@@ -11,10 +11,10 @@ from torch import nn
11
12
from vllm.config import LoRAConfig
13
from vllm.logger import init_logger
14
-from vllm.lora.layers import (BaseLayerWithLoRA, LoRAMapping, from_layer,
15
- from_layer_logits_processor)
14
+from vllm.lora.layers import BaseLayerWithLoRA, LoRAMapping
15
from vllm.lora.lora import LoRALayerWeights, PackedLoRALayerWeights
17
-from vllm.lora.utils import parse_fine_tuned_lora_name, replace_submodule
16
+from vllm.lora.utils import (from_layer, from_layer_logits_processor,
17
+ parse_fine_tuned_lora_name, replace_submodule)
18
from vllm.utils import LRUCache, is_pin_memory_available
19
20
logger = init_logger(__name__)
vllm/lora/punica.py
+43
@@ -49,6 +49,49 @@ def bgmv(
49
punica_kernels.dispatch_bgmv(y, x, w_t_all, indicies, layer_idx, scale)
50
51
52
+def dispatch_bgmv_low_level(y: torch.Tensor, x: torch.Tensor,
53
+ w_t_all: torch.Tensor, indicies: torch.LongTensor,
54
+ layer_idx: int, scale: float, y_offset: int,
55
+ y_slice_size: int):
56
+ """
57
+ Same as `bgmv` but you can operate on slices of y.
58
+ Pass whole y, define y_offset and y_slice_size.
59
+
60
+ Semantics:
61
+ y[i] += (
62
+ x[i].unsqueeze(0)
63
+ @ w_t_all[indices[i], layer_idx, :, :].transpose(-1, -2)
64
+ * scale
65
+ ).squeeze(0)
66
+
67
+ Args:
68
+ y: Shape: `[B, H2]`. Output vectors. Will be changed in-place.
69
+ x: Shape: `[B, H1]`. Input vectors.
70
+ w_t_all: Shape: `[None, L, y_slice_size, H1]`. Column partition of
71
+ all of the transposed LoRA matrices.
72
+ indicies: Shape: `[B]`. Indices of the LoRA weights.
73
+ layer_idx: Layer index of LoRA weights.
74
+ scale: Scaling factor.
75
+ y_offset: Offset to apply to the starting column of y.
76
+ y_slice_size: Size of the y column slice.
77
+ """
78
+ try:
79
+ import vllm._punica_C as punica_kernels
80
+ except ImportError as e:
81
+ _raise_import_error(e)
82
+ punica_kernels.dispatch_bgmv_low_level(
83
+ y,
84
+ x,
85
+ w_t_all,
86
+ indicies,
87
+ layer_idx,
88
+ scale,
89
+ x.size(1),
90
+ y_slice_size,
91
+ y_offset,
92
+ )
93
+
94
+
95
def add_lora(y: torch.Tensor,
96
x: torch.Tensor,
97
wa_t_all: torch.Tensor,
vllm/lora/utils.py
+59
-1
@@ -1,11 +1,69 @@
1
-from typing import Tuple
1
+from typing import List, Optional, Set, Tuple, Type
2
3
from torch import nn
4
+from transformers import PretrainedConfig
5
6
+from vllm.config import LoRAConfig
7
from vllm.logger import init_logger
8
+from vllm.lora.fully_sharded_layers import (
9
+ ColumnParallelLinearWithShardedLoRA,
10
+ MergedColumnParallelLinearWithShardedLoRA,
11
+ MergedQKVParallelLinearWithShardedLora, RowParallelLinearWithShardedLoRA)
12
+# being imported for _all_lora_classes below
13
+# yapf conflicts with isort for this block
14
+# yapf: disable
15
+from vllm.lora.layers import (BaseLayerWithLoRA, ColumnParallelLinearWithLoRA,
16
+ LogitsProcessorWithLoRA,
17
+ MergedColumnParallelLinearWithLoRA,
18
+ MergedQKVParallelLinearWithLora,
19
+ QKVParallelLinearWithLora,
20
+ RowParallelLinearWithLoRA,
21
+ VocabParallelEmbeddingWithLoRA)
22
+# yapf: enable
23
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
24
+from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
25
26
logger = init_logger(__name__)
27
28
+_all_lora_classes: Set[Type[BaseLayerWithLoRA]] = {
29
+ VocabParallelEmbeddingWithLoRA, ColumnParallelLinearWithLoRA,
30
+ MergedColumnParallelLinearWithLoRA, QKVParallelLinearWithLora,
31
+ MergedQKVParallelLinearWithLora, RowParallelLinearWithLoRA,
32
+ LogitsProcessorWithLoRA, ColumnParallelLinearWithShardedLoRA,
33
+ MergedColumnParallelLinearWithShardedLoRA,
34
+ MergedQKVParallelLinearWithShardedLora, RowParallelLinearWithShardedLoRA
35
+}
36
+
37
+
38
+def from_layer(layer: nn.Module,
39
+ max_loras: int,
40
+ lora_config: LoRAConfig,
41
+ packed_modules_list: List,
42
+ model_config: Optional[PretrainedConfig] = None) -> nn.Module:
43
+ for lora_cls in _all_lora_classes:
44
+ # specifying kwargs so they can be easily accessed in decorator
45
+ if lora_cls.can_replace_layer(source_layer=layer,
46
+ lora_config=lora_config,
47
+ packed_modules_list=packed_modules_list,
48
+ model_config=model_config):
49
+ ret = lora_cls(layer)
50
+ ret.create_lora_weights(max_loras, lora_config, model_config)
51
+ return ret
52
+ return layer
53
+
54
+
55
+def from_layer_logits_processor(
56
+ layer: LogitsProcessor,
57
+ lm_head: ParallelLMHead,
58
+ max_loras: int,
59
+ lora_config: LoRAConfig,
60
+ model_config: Optional[PretrainedConfig] = None,
61
+) -> LogitsProcessorWithLoRA:
62
+ ret = LogitsProcessorWithLoRA(layer, lm_head.embedding_dim,
63
+ lm_head.weight.dtype, lm_head.weight.device)
64
+ ret.create_lora_weights(max_loras, lora_config, model_config)
65
+ return ret
66
+
67
68
def replace_submodule(model: nn.Module, module_name: str,
69
new_module: nn.Module) -> nn.Module: