@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / eefeb16464
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Kernel] Full Tensor Parallelism for LoRA Layers (#3524)

Co-authored-by: Antoni Baum <antoni.baum@protonmail.com>

Austin Veselka committed Apr 27, 2024 at 02:03 UTC eefeb16464af5f3a61e3052d1a4128480bff7f47
19 files changed +686 -111
csrc/punica/bgmv/bgmv_bf16_bf16_bf16.cu
+1
@@ -2,3 +2,4 @@
2 #include "bgmv_impl.cuh"
3
4 FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_bfloat16, nv_bfloat16, nv_bfloat16)
5 +FOR_INST_BGMV_WIDE_NARROW(INST_BGMV_ONESIDE, nv_bfloat16, nv_bfloat16, nv_bfloat16)
csrc/punica/bgmv/bgmv_bf16_fp32_bf16.cu
+1
@@ -2,3 +2,4 @@
2 #include "bgmv_impl.cuh"
3
4 FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_bfloat16, float, nv_bfloat16)
5 +FOR_INST_BGMV_WIDE_NARROW(INST_BGMV_ONESIDE, nv_bfloat16, float, nv_bfloat16)
csrc/punica/bgmv/bgmv_config.h
+78
@@ -74,6 +74,74 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
74 // Keep above in sync with vllm/lora/layers::LogitsProcessorWithLoRA
75 // and vllm/tests/lora/test_punica.py
76
77 +// Used for defining kernels going from the variety of
78 +// dim in to the narrow dim out
79 + // Using it for the fully sharded column
80 + // parallel LoRA A which splits the rank dim
81 +#define FOR_INST_BGMV_NARROW(f, in_T, out_T, W_T, narrow) \
82 + f(in_T, out_T, W_T, 128, narrow) \
83 + f(in_T, out_T, W_T, 256, narrow) \
84 + f(in_T, out_T, W_T, 512, narrow) \
85 + f(in_T, out_T, W_T, 640, narrow) \
86 + f(in_T, out_T, W_T, 768, narrow) \
87 + f(in_T, out_T, W_T, 1024, narrow) \
88 + f(in_T, out_T, W_T, 1152, narrow) \
89 + f(in_T, out_T, W_T, 1280, narrow) \
90 + f(in_T, out_T, W_T, 1536, narrow) \
91 + f(in_T, out_T, W_T, 1728, narrow) \
92 + f(in_T, out_T, W_T, 1792, narrow) \
93 + f(in_T, out_T, W_T, 2048, narrow) \
94 + f(in_T, out_T, W_T, 2304, narrow) \
95 + f(in_T, out_T, W_T, 2560, narrow) \
96 + f(in_T, out_T, W_T, 2752, narrow) \
97 + f(in_T, out_T, W_T, 2816, narrow) \
98 + f(in_T, out_T, W_T, 3072, narrow) \
99 + f(in_T, out_T, W_T, 3456, narrow) \
100 + f(in_T, out_T, W_T, 3584, narrow) \
101 + f(in_T, out_T, W_T, 4096, narrow) \
102 + f(in_T, out_T, W_T, 4608, narrow) \
103 + f(in_T, out_T, W_T, 5120, narrow) \
104 + f(in_T, out_T, W_T, 5504, narrow) \
105 + f(in_T, out_T, W_T, 5632, narrow) \
106 + f(in_T, out_T, W_T, 6144, narrow) \
107 + f(in_T, out_T, W_T, 6848, narrow) \
108 + f(in_T, out_T, W_T, 6912, narrow) \
109 + f(in_T, out_T, W_T, 7168, narrow) \
110 + f(in_T, out_T, W_T, 8192, narrow) \
111 + f(in_T, out_T, W_T, 9216, narrow) \
112 + f(in_T, out_T, W_T, 10240, narrow) \
113 + f(in_T, out_T, W_T, 11008, narrow) \
114 + f(in_T, out_T, W_T, 12288, narrow) \
115 + f(in_T, out_T, W_T, 13696, narrow) \
116 + f(in_T, out_T, W_T, 13824, narrow) \
117 + f(in_T, out_T, W_T, 14336, narrow) \
118 + f(in_T, out_T, W_T, 15360, narrow) \
119 + f(in_T, out_T, W_T, 16384, narrow) \
120 + f(in_T, out_T, W_T, 20480, narrow) \
121 + f(in_T, out_T, W_T, 22016, narrow) \
122 + f(in_T, out_T, W_T, 24576, narrow) \
123 + f(in_T, out_T, W_T, 27392, narrow) \
124 + f(in_T, out_T, W_T, 28672, narrow) \
125 + f(in_T, out_T, W_T, 32000, narrow) \
126 + f(in_T, out_T, W_T, 32256, narrow) \
127 + f(in_T, out_T, W_T, 32512, narrow) \
128 + f(in_T, out_T, W_T, 32768, narrow) \
129 + f(in_T, out_T, W_T, 33024, narrow) \
130 + f(in_T, out_T, W_T, 36864, narrow) \
131 + f(in_T, out_T, W_T, 43264, narrow) \
132 + f(in_T, out_T, W_T, 49152, narrow) \
133 + f(in_T, out_T, W_T, 64000, narrow) \
134 + f(in_T, out_T, W_T, 64256, narrow) \
135 + f(in_T, out_T, W_T, 64512, narrow) \
136 + f(in_T, out_T, W_T, 102400, narrow) \
137 + f(in_T, out_T, W_T, 102656, narrow) \
138 + f(in_T, out_T, W_T, 102912, narrow) \
139 + f(in_T, out_T, W_T, 128000, narrow) \
140 + f(in_T, out_T, W_T, 128256, narrow) \
141 + f(in_T, out_T, W_T, 128512, narrow) \
142 +// Keep above in sync with vllm/lora/layers::SamplerWithLoRA
143 +
144 +
145 // Keep this in sync with vllm/config::LoRAConfig
146 #define FOR_BGMV_WIDE_NARROW(f, in_T, out_T, W_T) \
147 FOR_BGMV_WIDE(f, in_T, out_T, W_T, 8) \
@@ -81,4 +149,14 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
149 FOR_BGMV_WIDE(f, in_T, out_T, W_T, 32) \
150 FOR_BGMV_WIDE(f, in_T, out_T, W_T, 64)
151
152 +
153 +#define FOR_INST_BGMV_WIDE_NARROW(f, in_T, out_T, W_T) \
154 + FOR_INST_BGMV_NARROW(f, in_T, out_T, W_T, 1) \
155 + FOR_INST_BGMV_NARROW(f, in_T, out_T, W_T, 2) \
156 + FOR_INST_BGMV_NARROW(f, in_T, out_T, W_T, 4) \
157 + f(in_T, out_T, W_T, 8, 64) \
158 + f(in_T, out_T, W_T, 16, 64) \
159 + f(in_T, out_T, W_T, 32, 64) \
160 + f(in_T, out_T, W_T, 64, 64)
161 +
162 // clang-format on
csrc/punica/bgmv/bgmv_fp16_fp16_fp16.cu
+1
@@ -2,3 +2,4 @@
2 #include "bgmv_impl.cuh"
3
4 FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_half, nv_half, nv_half)
5 +FOR_INST_BGMV_WIDE_NARROW(INST_BGMV_ONESIDE, nv_half, nv_half, nv_half)
csrc/punica/bgmv/bgmv_fp16_fp32_fp16.cu
+1
@@ -2,3 +2,4 @@
2 #include "bgmv_impl.cuh"
3
4 FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, nv_half, float, nv_half)
5 +FOR_INST_BGMV_WIDE_NARROW(INST_BGMV_ONESIDE, nv_half, float, nv_half)
csrc/punica/bgmv/bgmv_fp32_bf16_bf16.cu
+1
@@ -2,3 +2,4 @@
2 #include "bgmv_impl.cuh"
3
4 FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, float, nv_bfloat16, nv_bfloat16)
5 +FOR_INST_BGMV_WIDE_NARROW(INST_BGMV_ONESIDE, float, nv_bfloat16, nv_bfloat16)
csrc/punica/bgmv/bgmv_fp32_fp16_fp16.cu
+1
@@ -2,3 +2,4 @@
2 #include "bgmv_impl.cuh"
3
4 FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, float, nv_half, nv_half)
5 +FOR_INST_BGMV_WIDE_NARROW(INST_BGMV_ONESIDE, float, nv_half, nv_half)
csrc/punica/bgmv/bgmv_impl.cuh
+4 -1
@@ -199,7 +199,7 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
199 constexpr int tz = 4;
200 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
201
202 - if constexpr (feat_in < feat_out) {
202 + if constexpr (feat_in <= feat_out) {
203 static_assert(feat_in % vec_size == 0);
204 constexpr int tx = feat_in / vec_size;
205
@@ -289,6 +289,9 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
289 int64_t y_offset, int64_t full_y_size, int64_t batch_size, \
290 int64_t num_layers, int64_t layer_idx, float scale);
291
292 +#define INST_BGMV_ONESIDE(in_T, out_T, W_T, feat_in, feat_out) \
293 + INST_BGMV(feat_in, feat_out, in_T, out_T, W_T)
294 +
295 #define INST_BGMV_TWOSIDE(in_T, out_T, W_T, narrow, wide) \
296 INST_BGMV(narrow, wide, in_T, out_T, W_T) \
297 INST_BGMV(wide, narrow, in_T, out_T, W_T)
csrc/punica/bgmv/generator.py
+1
@@ -10,6 +10,7 @@ TEMPLATE = """
10 #include "bgmv_impl.cuh"
11
12 FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, {input_dtype}, {output_dtype}, {weight_dtype})
13 +FOR_INST_BGMV_WIDE_NARROW(INST_BGMV_ONESIDE, {input_dtype}, {output_dtype}, {weight_dtype})
14 """.lstrip() # noqa: E501
15
16 for input_dtype in DTYPES:
csrc/punica/punica_ops.cc
+1 -1
@@ -79,12 +79,12 @@ inline bool launch_bgmv_kernel(out_T *Y, const in_T *X, const W_T *W,
79 CASE_ONESIDE(in_T, out_T, W_T, wide, narrow)
80
81 FOR_BGMV_WIDE_NARROW(CASE, _, _, _)
82 + FOR_INST_BGMV_WIDE_NARROW(CASE_ONESIDE, _, _, _)
83 #undef CASE
84 #undef CASE_ONESIDE
85 default:
86 return false;
87 }
87 -
88 return true;
89 }
90
tests/lora/test_layers.py
+23 -6
@@ -8,6 +8,10 @@ import torch
8 import torch.nn.functional as F
9
10 from vllm.config import LoRAConfig
11 +from vllm.lora.fully_sharded_layers import (
12 + ColumnParallelLinearWithShardedLoRA,
13 + MergedColumnParallelLinearWithShardedLoRA,
14 + MergedQKVParallelLinearWithShardedLora, RowParallelLinearWithShardedLoRA)
15 # yapf conflicts with isort for this block
16 # yapf: disable
17 from vllm.lora.layers import (BaseLayerWithLoRA, ColumnParallelLinearWithLoRA,
@@ -524,13 +528,16 @@ def test_lm_head_logits_processor(dist_init, num_loras, device,
528 @torch.inference_mode()
529 @pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
530 @pytest.mark.parametrize("orientation", ["row", "column"])
531 +@pytest.mark.parametrize("fully_shard", [True, False])
532 @pytest.mark.parametrize("device", CUDA_DEVICES)
528 -def test_linear_parallel(dist_init, num_loras, orientation, device) -> None:
533 +def test_linear_parallel(dist_init, num_loras, orientation, fully_shard,
534 + device) -> None:
535
536 torch.set_default_device(device)
537 max_loras = 8
538 lora_config = LoRAConfig(max_loras=max_loras,
539 max_lora_rank=8,
540 + fully_sharded_loras=fully_shard,
541 lora_dtype=torch.float16)
542
543 def create_random_linear_parallel_layer():
@@ -540,14 +547,17 @@ def test_linear_parallel(dist_init, num_loras, orientation, device) -> None:
547 bias=False,
548 params_dtype=torch.float16)
549 linear.weight.data = torch.rand_like(linear.weight.data)
543 - lora_linear = RowParallelLinearWithLoRA(linear)
550 + lora_linear = (RowParallelLinearWithLoRA(linear) if not fully_shard
551 + else RowParallelLinearWithShardedLoRA(linear))
552 else:
553 linear = ColumnParallelLinear(4096,
554 4096,
555 bias=False,
556 params_dtype=torch.float16)
557 linear.weight.data = torch.rand_like(linear.weight.data)
550 - lora_linear = ColumnParallelLinearWithLoRA(linear)
558 + lora_linear = (ColumnParallelLinearWithLoRA(linear)
559 + if not fully_shard else
560 + ColumnParallelLinearWithShardedLoRA(linear))
561 lora_linear.create_lora_weights(max_loras, lora_config)
562
563 return linear, lora_linear
@@ -629,13 +639,16 @@ def test_linear_parallel(dist_init, num_loras, orientation, device) -> None:
639 @torch.inference_mode()
640 @pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
641 @pytest.mark.parametrize("repeats", [1, 2, 3])
642 +@pytest.mark.parametrize("fully_shard", [True, False])
643 @pytest.mark.parametrize("device", CUDA_DEVICES)
633 -def test_column_parallel_packed(dist_init, num_loras, repeats, device) -> None:
644 +def test_column_parallel_packed(dist_init, num_loras, repeats, fully_shard,
645 + device) -> None:
646
647 torch.set_default_device(device)
648 max_loras = 8
649 lora_config = LoRAConfig(max_loras=max_loras,
650 max_lora_rank=8,
651 + fully_sharded_loras=fully_shard,
652 lora_dtype=torch.float16)
653
654 def create_column_parallel_packed_layer():
@@ -644,7 +657,9 @@ def test_column_parallel_packed(dist_init, num_loras, repeats, device) -> None:
657 bias=False,
658 params_dtype=torch.float16)
659 linear.weight.data = torch.rand_like(linear.weight.data)
647 - lora_linear = MergedColumnParallelLinearWithLoRA(linear)
660 + lora_linear = (MergedColumnParallelLinearWithLoRA(linear)
661 + if not fully_shard else
662 + MergedColumnParallelLinearWithShardedLoRA(linear))
663 elif repeats == 3:
664 linear = QKVParallelLinear(4096,
665 64,
@@ -652,7 +667,9 @@ def test_column_parallel_packed(dist_init, num_loras, repeats, device) -> None:
667 bias=False,
668 params_dtype=torch.float16)
669 linear.weight.data = torch.rand_like(linear.weight.data)
655 - lora_linear = MergedQKVParallelLinearWithLora(linear)
670 + lora_linear = (MergedQKVParallelLinearWithLora(linear)
671 + if not fully_shard else
672 + MergedQKVParallelLinearWithShardedLora(linear))
673 else:
674 linear = QKVParallelLinear(4096,
675 64,
tests/lora/test_punica.py
+49 -2
@@ -34,11 +34,14 @@ def _lora_ref_impl(
34 for i, lora_idx in zip(range(bs), indicies.cpu().tolist()):
35 xi = x[i].unsqueeze(0).to(torch.float32)
36 wa = wa_T_all[lora_idx, layer_idx].transpose(-1, -2).to(torch.float32)
37 - wb = wb_T_all[lora_idx, layer_idx].transpose(-1, -2).to(torch.float32)
37 + if wb_T_all is not None:
38 + wb = wb_T_all[lora_idx, layer_idx].transpose(-1,
39 + -2).to(torch.float32)
40
41 tmp = xi @ wa
42 y_stage_1[i] = tmp.squeeze(0)
41 - y_final[i] += (tmp @ wb).squeeze(0) * s
43 + y_final[i] += ((tmp @ wb).squeeze(0) *
44 + s if wb_T_all is not None else y_stage_1[i])
45 return y_final, y_stage_1
46
47
@@ -91,12 +94,56 @@ H1 = H2 = [
94 128000,
95 128256,
96 ]
97 +H2 = [64] + H2
98 +R = [1, 2, 4]
99 SEED = [0xabcdabcd987]
100 CUDA_DEVICES = [
101 f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
102 ]
103
104
105 +@pytest.mark.parametrize("dtype_str", ["float16", "bfloat16"])
106 +@pytest.mark.parametrize("h1", H1)
107 +@pytest.mark.parametrize("r", R)
108 +@pytest.mark.parametrize("seed", SEED)
109 +@torch.inference_mode()
110 +def test_lora_a_extra_shapes(dtype_str, h1, r, seed):
111 + torch.manual_seed(seed)
112 + num_loras = 4
113 + num_layers = 1
114 + bs = 32
115 + dtype = getattr(torch, dtype_str)
116 + device = torch.device("cuda")
117 +
118 + wa_T_all = torch.randn(num_loras,
119 + num_layers,
120 + r,
121 + h1,
122 + dtype=dtype,
123 + device=device)
124 + indices = torch.randint(num_loras, (bs, ), dtype=torch.long, device=device)
125 +
126 + for layer_idx in range(num_layers):
127 + x = torch.randn(bs, h1, dtype=dtype, device=device)
128 + y = torch.randn(bs, r, dtype=dtype, device=device)
129 +
130 + y_ref = y.clone()
131 + _lora_ref_impl(
132 + y_ref,
133 + x,
134 + wa_T_all,
135 + None,
136 + indices,
137 + layer_idx,
138 + 1.0,
139 + )
140 +
141 + y_our = y.clone()
142 + punica.bgmv(y_our, x, wa_T_all, indices, layer_idx, 1.0)
143 +
144 + assert_close(y_ref, y_our)
145 +
146 +
147 @pytest.mark.parametrize("dtype_str", ["float16", "bfloat16"])
148 @pytest.mark.parametrize("h1", H1)
149 @pytest.mark.parametrize("h2", H2)
vllm/config.py
+1
@@ -862,6 +862,7 @@ class SpeculativeConfig:
862 class LoRAConfig:
863 max_lora_rank: int
864 max_loras: int
865 + fully_sharded_loras: bool = False
866 max_cpu_loras: Optional[int] = None
867 lora_dtype: Optional[torch.dtype] = None
868 lora_extra_vocab_size: int = 256
vllm/engine/arg_utils.py
+10
@@ -52,6 +52,7 @@ class EngineArgs:
52 enable_lora: bool = False
53 max_loras: int = 1
54 max_lora_rank: int = 16
55 + fully_sharded_loras: bool = False
56 lora_extra_vocab_size: int = 256
57 lora_dtype = 'auto'
58 max_cpu_loras: Optional[int] = None
@@ -376,6 +377,14 @@ class EngineArgs:
377 help=('Maximum number of LoRAs to store in CPU memory. '
378 'Must be >= than max_num_seqs. '
379 'Defaults to max_num_seqs.'))
380 + parser.add_argument(
381 + '--fully-sharded-loras',
382 + action='store_true',
383 + help=('By default, only half of the LoRA computation is '
384 + 'sharded with tensor parallelism. '
385 + 'Enabling this will use the fully sharded layers. '
386 + 'At high sequence length, max rank or '
387 + 'tensor parallel size, this is likely faster.'))
388 parser.add_argument("--device",
389 type=str,
390 default=EngineArgs.device,
@@ -509,6 +518,7 @@ class EngineArgs:
518 lora_config = LoRAConfig(
519 max_lora_rank=self.max_lora_rank,
520 max_loras=self.max_loras,
521 + fully_sharded_loras=self.fully_sharded_loras,
522 lora_extra_vocab_size=self.lora_extra_vocab_size,
523 lora_dtype=self.lora_dtype,
524 max_cpu_loras=self.max_cpu_loras if self.max_cpu_loras
vllm/lora/fully_sharded_layers.py new
+262
@@ -0,0 +1,262 @@
1 +# pylint: disable=unused-argument
2 +from typing import TYPE_CHECKING, List, Optional
3 +
4 +import torch
5 +import torch.nn as nn
6 +from transformers import PretrainedConfig
7 +
8 +from vllm.config import LoRAConfig
9 +from vllm.distributed.communication_op import (
10 + tensor_model_parallel_all_gather, tensor_model_parallel_all_reduce)
11 +from vllm.distributed.parallel_state import get_tensor_model_parallel_rank
12 +from vllm.lora.layers import (ColumnParallelLinearWithLoRA,
13 + MergedColumnParallelLinearWithLoRA,
14 + MergedQKVParallelLinearWithLora,
15 + RowParallelLinearWithLoRA)
16 +from vllm.lora.punica import bgmv, dispatch_bgmv_low_level
17 +
18 +if TYPE_CHECKING:
19 + pass
20 +
21 +
22 +def _fully_sharded_can_replace(can_replace):
23 + """
24 + decorator which adds the condition of fully sharded loras
25 + intended to wrap can_replace_layer()
26 + """
27 +
28 + def dec(*args, **kwargs):
29 + return (can_replace(*args, **kwargs)
30 + and kwargs['lora_config'].fully_sharded_loras)
31 +
32 + return dec
33 +
34 +
35 +# these layers are based on the tensor parallelism strategy given in
36 +# Y. Sheng et al., S-LoRA: Serving Thousands of Concurrent LoRA Adapters. 2023,
37 +# https://arxiv.org/abs/2311.03285.
38 +
39 +
40 +class ColumnParallelLinearWithShardedLoRA(ColumnParallelLinearWithLoRA):
41 + """
42 + Differs from ColumnParallelLinearWithLoRA by slicing LoRA A also.
43 +
44 + Based on S-LoRA, slicing happens along the rank dim.
45 + """
46 +
47 + def slice_lora_a(self, lora_a: torch.Tensor) -> torch.Tensor:
48 + tp_rank = get_tensor_model_parallel_rank()
49 + shard_size = self.lora_a_stacked.shape[2]
50 + start_idx = tp_rank * shard_size
51 + lora_a = lora_a[:, start_idx:start_idx + shard_size]
52 + return lora_a
53 +
54 + def apply_weights(self, x: torch.Tensor,
55 + bias: Optional[torch.Tensor]) -> torch.Tensor:
56 + output = self.base_layer.linear_method.apply_weights(
57 + self.base_layer, x, bias)
58 +
59 + x = x.view(-1, x.shape[-1])
60 + output, out_orig_shape = output.view(-1,
61 + output.shape[-1]), output.shape
62 + buffer = torch.zeros((x.shape[0], self.lora_a_stacked.shape[2]),
63 + dtype=torch.float32,
64 + device=x.device)
65 +
66 + bgmv(buffer, x, self.lora_a_stacked,
67 + self.indices[:self.indices_len[0]], 0, 1.0)
68 + buffer = tensor_model_parallel_all_gather(buffer)
69 + bgmv(output, buffer, self.lora_b_stacked,
70 + self.indices[:self.indices_len[0]], 0, 1.0)
71 + # now have column partitioned output
72 +
73 + output = output.view(*out_orig_shape)
74 + return output
75 +
76 + @classmethod
77 + @_fully_sharded_can_replace
78 + def can_replace_layer(cls, source_layer: nn.Module,
79 + lora_config: LoRAConfig, packed_modules_list: List,
80 + model_config: Optional[PretrainedConfig]) -> bool:
81 + # specifying kwargs so they can be easily accessed in decorator
82 + return super().can_replace_layer(
83 + source_layer=source_layer,
84 + lora_config=lora_config,
85 + packed_modules_list=packed_modules_list,
86 + model_config=model_config,
87 + decorate=False,
88 + )
89 +
90 +
91 +def _mcp_apply_weights(x, bias, layer):
92 + """
93 + MergedColumnParallelLinearWithShardedLoRA and
94 + QKVParallelLinearWithShardedLora share the same
95 + LoRa weight application method.
96 +
97 + The main difference is the step by shard_size for lora_b which can
98 + vary for QKVParallelLinearWithShardedLora but is constant for
99 + MergedColumnParallelLinearWithShardedLoRA.
100 + """
101 + # expecting 2 for column parallel and 3 for qkv
102 + n = len(layer.lora_a_stacked)
103 + output = layer.base_layer.linear_method.apply_weights(
104 + layer.base_layer, x, bias)
105 +
106 + x = x.view(-1, x.shape[-1])
107 + output, out_orig_shape = output.view(-1, output.shape[-1]), output.shape
108 + buffers = torch.zeros((n, x.shape[0], layer.lora_a_stacked[0].shape[2]),
109 + dtype=torch.float32,
110 + device=x.device)
111 + for idx in range(n):
112 + bgmv(buffers[idx], x, layer.lora_a_stacked[idx],
113 + layer.indices[:layer.indices_len[0]], 0, 1.0)
114 +
115 + buffers = tensor_model_parallel_all_gather(buffers)
116 + left_offset = 0
117 + for idx in range(n):
118 + shard_size = layer.lora_b_stacked[idx].shape[2]
119 + dispatch_bgmv_low_level(output, buffers[idx],
120 + layer.lora_b_stacked[idx],
121 + layer.indices[:layer.indices_len[0]], 0, 1.0,
122 + left_offset, shard_size)
123 + left_offset += shard_size
124 +
125 + output = output.view(*out_orig_shape)
126 + # now have column partitioned and packed output
127 + return output
128 +
129 +
130 +class MergedColumnParallelLinearWithShardedLoRA(
131 + MergedColumnParallelLinearWithLoRA):
132 + """
133 + Differs from MergedColumnParallelLinearWithLoRA by slicing the
134 + LoRA A's also.
135 +
136 + Based on S-LoRA, slicing happens along the rank dim.
137 + """
138 +
139 + def slice_lora_a(self, lora_a: List[torch.Tensor]) -> List[torch.Tensor]:
140 + output_shard_size = self.lora_a_stacked[0].shape[2]
141 + output_start_idx = self.tp_rank * output_shard_size
142 + lora_a = [
143 + lora_a[i][:, output_start_idx:output_start_idx + output_shard_size]
144 + for i in range(2)
145 + ]
146 + return lora_a
147 +
148 + def apply_weights(self, x: torch.Tensor,
149 + bias: Optional[torch.Tensor]) -> torch.Tensor:
150 + return _mcp_apply_weights(x, bias, self)
151 +
152 + @classmethod
153 + @_fully_sharded_can_replace
154 + def can_replace_layer(cls, source_layer: nn.Module,
155 + lora_config: LoRAConfig, packed_modules_list: List,
156 + model_config: Optional[PretrainedConfig]) -> bool:
157 + # specifying kwargs so they can be easily accessed in decorator
158 + return super().can_replace_layer(
159 + source_layer=source_layer,
160 + lora_config=lora_config,
161 + packed_modules_list=packed_modules_list,
162 + model_config=model_config,
163 + decorate=False,
164 + )
165 +
166 +
167 +class MergedQKVParallelLinearWithShardedLora(MergedQKVParallelLinearWithLora):
168 + """
169 + Differs from QKVParallelLinearWithLora by slicing the
170 + LoRA A's also.
171 +
172 + Based on S-LoRA, slicing happens along the rank dim.
173 + """
174 +
175 + def slice_lora_a(self, lora_a: List[torch.Tensor]) -> List[torch.Tensor]:
176 + shard_size = [self.lora_a_stacked[i].shape[2] for i in range(3)]
177 + start_idx = [self.tp_rank * shard_size[i] for i in range(3)]
178 + lora_a = [
179 + lora_a[i][:, start_idx[i]:start_idx[i] +
180 + shard_size[i]] if lora_a[i] is not None else None
181 + for i in range(3)
182 + ]
183 + return lora_a
184 +
185 + def apply_weights(self, x: torch.Tensor,
186 + bias: Optional[torch.Tensor]) -> torch.Tensor:
187 + return _mcp_apply_weights(x, bias, self)
188 +
189 + @classmethod
190 + @_fully_sharded_can_replace
191 + def can_replace_layer(cls, source_layer: nn.Module,
192 + lora_config: LoRAConfig, packed_modules_list: List,
193 + model_config: Optional[PretrainedConfig]) -> bool:
194 + # specifying kwargs so they can be easily accessed in decorator
195 + return super().can_replace_layer(
196 + source_layer=source_layer,
197 + lora_config=lora_config,
198 + packed_modules_list=packed_modules_list,
199 + model_config=model_config,
200 + decorate=False,
201 + )
202 +
203 +
204 +class RowParallelLinearWithShardedLoRA(RowParallelLinearWithLoRA):
205 + """
206 + Differs from RowParallelLinearWithLoRA by slicing the
207 + LoRA B's also.
208 +
209 + Based on S-LoRA, slicing happens along the output dim.
210 + This yields a combined partial sum from the row parallel base
211 + layer and column partitioned output from the LoRA.
212 + """
213 +
214 + def slice_lora_b(self, lora_b: torch.Tensor) -> torch.Tensor:
215 + shard_size = self.lora_b_stacked.shape[2]
216 + start_idx = self.tp_rank * shard_size
217 + end_idx = (self.tp_rank + 1) * shard_size
218 + lora_b = lora_b[:, start_idx:end_idx]
219 + return lora_b
220 +
221 + def apply_weights(self, x: torch.Tensor) -> torch.Tensor:
222 + output = self.base_layer.linear_method.apply_weights(
223 + self.base_layer, x)
224 +
225 + x = x.view(-1, x.shape[-1])
226 + output, out_orig_shape = output.view(-1,
227 + output.shape[-1]), output.shape
228 + buffer = torch.zeros((x.shape[0], self.lora_a_stacked.shape[2]),
229 + dtype=torch.float32,
230 + device=x.device)
231 + bgmv(buffer, x, self.lora_a_stacked,
232 + self.indices[:self.indices_len[0]], 0, 1.0)
233 + buffer = tensor_model_parallel_all_reduce(buffer)
234 +
235 + # following S-LoRA, allows the fusing of all_gather and all_reduce
236 + # by adding the column partitioned lora output to a slice of output
237 + # tensor, which is a partial sum due to row parallel. All that
238 + # remains is a standard all_reduce. User should be aware though that
239 + # the output is not the same as a normal row_parallel, it should be
240 + # reduced before being used
241 + shard_size = self.lora_b_stacked.shape[2]
242 + start_idx = self.tp_rank * shard_size
243 + dispatch_bgmv_low_level(output, buffer, self.lora_b_stacked,
244 + self.indices[:self.indices_len[0]], 0, 1.0,
245 + start_idx, shard_size)
246 +
247 + output = output.view(*out_orig_shape)
248 + return output
249 +
250 + @classmethod
251 + @_fully_sharded_can_replace
252 + def can_replace_layer(cls, source_layer: nn.Module,
253 + lora_config: LoRAConfig, packed_modules_list: List,
254 + model_config: Optional[PretrainedConfig]) -> bool:
255 + # specifying kwargs so they can be easily accessed in decorator
256 + return super().can_replace_layer(
257 + source_layer=source_layer,
258 + lora_config=lora_config,
259 + packed_modules_list=packed_modules_list,
260 + model_config=model_config,
261 + decorate=False,
262 + )
vllm/lora/layers.py
+146 -97
@@ -1,8 +1,7 @@
1 # pylint: disable=unused-argument
2 -import inspect
2 import math
3 from dataclasses import dataclass
5 -from typing import TYPE_CHECKING, List, Optional, Set, Tuple, Type
4 +from typing import TYPE_CHECKING, List, Optional, Tuple
5
6 import torch
7 import torch.nn as nn
@@ -16,6 +15,7 @@ from vllm.distributed import (get_tensor_model_parallel_rank,
15 tensor_model_parallel_all_gather,
16 tensor_model_parallel_all_reduce,
17 tensor_model_parallel_gather)
18 +from vllm.distributed.utils import divide
19 from vllm.lora.punica import add_lora, add_lora_slice, bgmv
20 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
21 MergedColumnParallelLinear,
@@ -23,7 +23,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
23 RowParallelLinear)
24 from vllm.model_executor.layers.logits_processor import LogitsProcessor
25 from vllm.model_executor.layers.vocab_parallel_embedding import (
26 - ParallelLMHead, VocabParallelEmbedding)
26 + VocabParallelEmbedding)
27
28 if TYPE_CHECKING:
29 pass
@@ -45,6 +45,21 @@ def _get_lora_device(base_layer: nn.Module) -> torch.device:
45 raise ValueError(f"Unsupported base layer: {base_layer}")
46
47
48 +def _not_fully_sharded_can_replace(can_replace):
49 + """
50 + decorator which adds the condition of not using fully sharded loras
51 + intended to wrap can_replace_layer()
52 + """
53 +
54 + def dec(*args, **kwargs):
55 + decorate = kwargs.pop('decorate') if 'decorate' in kwargs else True
56 + condition = (not kwargs['lora_config'].fully_sharded_loras
57 + if decorate else True)
58 + return can_replace(*args, **kwargs) and condition
59 +
60 + return dec
61 +
62 +
63 def _apply_lora(
64 x: torch.Tensor,
65 lora_a_stacked: torch.Tensor,
@@ -130,6 +145,14 @@ class LoRAMapping:
145
146 class BaseLayerWithLoRA(nn.Module):
147
148 + def slice_lora_a(self, lora_a: torch.Tensor) -> torch.Tensor:
149 + """Slice lora a if splitting for tensor parallelism."""
150 + ...
151 +
152 + def slice_lora_b(self, lora_b: torch.Tensor) -> torch.Tensor:
153 + """Slice lora b if splitting with tensor parallelism."""
154 + ...
155 +
156 def create_lora_weights(
157 self,
158 max_loras: int,
@@ -317,6 +340,11 @@ class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA):
340
341
342 class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
343 + """
344 + LoRA on top of ColumnParallelLinear layer.
345 +
346 + LoRA B is sliced for tensor parallelism.
347 + """
348
349 def __init__(self, base_layer: ColumnParallelLinear) -> None:
350 super().__init__()
@@ -331,10 +359,15 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
359 max_loras: int,
360 lora_config: LoRAConfig,
361 model_config: Optional[PretrainedConfig] = None) -> None:
362 + self.lora_config = lora_config
363 + self.tp_size = get_tensor_model_parallel_world_size()
364 + lora_a_output_size_per_partition = (
365 + lora_config.max_lora_rank if not lora_config.fully_sharded_loras
366 + else divide(lora_config.max_lora_rank, self.tp_size))
367 self.lora_a_stacked = torch.zeros(
368 max_loras,
369 1,
337 - lora_config.max_lora_rank,
370 + lora_a_output_size_per_partition,
371 self.input_size,
372 dtype=lora_config.lora_dtype,
373 device=self.device,
@@ -357,6 +390,17 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
390 self.lora_a_stacked[index] = 0
391 self.lora_b_stacked[index] = 0
392
393 + def slice_lora_a(self, lora_a: torch.Tensor) -> torch.Tensor:
394 + return lora_a
395 +
396 + def slice_lora_b(self, lora_b: torch.Tensor) -> torch.Tensor:
397 + tensor_model_parallel_rank = get_tensor_model_parallel_rank()
398 + shard_size = self.output_dim
399 + start_idx = tensor_model_parallel_rank * shard_size
400 + end_idx = (tensor_model_parallel_rank + 1) * shard_size
401 + lora_b = lora_b[:, start_idx:end_idx]
402 + return lora_b
403 +
404 def set_lora(
405 self,
406 index: int,
@@ -365,12 +409,11 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
409 embeddings_tensor: Optional[torch.Tensor],
410 ):
411 self.reset_lora(index)
412 +
413 if self.tp_size > 1:
369 - tensor_model_parallel_rank = get_tensor_model_parallel_rank()
370 - shard_size = self.output_dim
371 - start_idx = tensor_model_parallel_rank * shard_size
372 - end_idx = (tensor_model_parallel_rank + 1) * shard_size
373 - lora_b = lora_b[:, start_idx:end_idx]
414 + lora_a = self.slice_lora_a(lora_a)
415 + lora_b = self.slice_lora_b(lora_b)
416 +
417 self.lora_a_stacked[index,
418 0, :lora_a.shape[1], :lora_a.shape[0]].copy_(
419 lora_a.T, non_blocking=True)
@@ -426,6 +469,7 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
469 return output, output_bias
470
471 @classmethod
472 + @_not_fully_sharded_can_replace
473 def can_replace_layer(cls, source_layer: nn.Module,
474 lora_config: LoRAConfig, packed_modules_list: List,
475 model_config: Optional[PretrainedConfig]) -> bool:
@@ -451,6 +495,7 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
495 max_loras: int,
496 lora_config: LoRAConfig,
497 model_config: Optional[PretrainedConfig] = None) -> None:
498 + self.lora_config = lora_config
499 n_slices = 2
500 if not (len(self.base_layer.output_sizes) == n_slices
501 and self.base_layer.output_sizes[0]
@@ -459,12 +504,17 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
504 "LoRAColumnParallelLinear2Slice requires 2 slices with "
505 "the same size.")
506 self.tp_size = get_tensor_model_parallel_world_size()
507 + self.tp_rank = get_tensor_model_parallel_rank()
508 +
509 + lora_a_output_size_per_partition = (
510 + lora_config.max_lora_rank if not lora_config.fully_sharded_loras
511 + else divide(lora_config.max_lora_rank, self.tp_size))
512
513 self.lora_a_stacked = tuple(
514 torch.zeros(
515 max_loras,
516 1,
467 - lora_config.max_lora_rank,
517 + lora_a_output_size_per_partition,
518 self.input_size,
519 dtype=lora_config.lora_dtype,
520 device=self.device,
@@ -489,6 +539,18 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
539 self.lora_b_stacked[0][index] = 0
540 self.lora_b_stacked[1][index] = 0
541
542 + def slice_lora_a(self, lora_a: List[torch.Tensor]) -> List[torch.Tensor]:
543 + return lora_a
544 +
545 + def slice_lora_b(self, lora_b: List[torch.Tensor]) -> List[torch.Tensor]:
546 + shard_size = self.output_dim
547 + start_idx = self.tp_rank * shard_size
548 + end_idx = (self.tp_rank + 1) * shard_size
549 + lora_b = [
550 + lora_b[0][:, start_idx:end_idx], lora_b[1][:, start_idx:end_idx]
551 + ]
552 + return lora_b
553 +
554 def set_lora(
555 self,
556 index: int,
@@ -499,13 +561,8 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
561 self.reset_lora(index)
562
563 if self.tp_size > 1:
502 - tensor_model_parallel_rank = get_tensor_model_parallel_rank()
503 - shard_size = self.output_dim
504 - start_idx = tensor_model_parallel_rank * shard_size
505 - end_idx = (tensor_model_parallel_rank + 1) * shard_size
506 - lora_b = lora_b[0][:,
507 - start_idx:end_idx], lora_b[1][:,
508 - start_idx:end_idx]
564 + lora_a = self.slice_lora_a(lora_a)
565 + lora_b = self.slice_lora_b(lora_b)
566
567 if lora_a[0] is not None:
568 self.lora_a_stacked[0][
@@ -536,6 +593,7 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
593 return output
594
595 @classmethod
596 + @_not_fully_sharded_can_replace
597 def can_replace_layer(cls, source_layer: nn.Module,
598 lora_config: LoRAConfig, packed_modules_list: List,
599 model_config: Optional[PretrainedConfig]) -> bool:
@@ -627,21 +685,25 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
685 max_loras: int,
686 lora_config: LoRAConfig,
687 model_config: Optional[PretrainedConfig] = None) -> None:
688 + self.lora_config = lora_config
689 self.tp_size = get_tensor_model_parallel_world_size()
631 - tp_rank = get_tensor_model_parallel_rank()
690 + self.tp_rank = get_tensor_model_parallel_rank()
691 self.q_proj_shard_size = (self.base_layer.num_heads *
692 self.base_layer.head_size)
693 self.kv_proj_shard_size = (self.base_layer.num_kv_heads *
694 self.base_layer.head_size)
636 - self.q_shard_id = tp_rank
637 - self.kv_shard_id = tp_rank // self.base_layer.num_kv_head_replicas
695 + self.q_shard_id = self.tp_rank
696 + self.kv_shard_id = self.tp_rank // self.base_layer.num_kv_head_replicas
697
698 + lora_a_output_size_per_partition = (
699 + lora_config.max_lora_rank if not lora_config.fully_sharded_loras
700 + else divide(lora_config.max_lora_rank, self.tp_size))
701 # q, k, v
702 self.lora_a_stacked = (
703 torch.zeros(
704 max_loras,
705 1,
644 - lora_config.max_lora_rank,
706 + lora_a_output_size_per_partition,
707 self.input_size,
708 dtype=lora_config.lora_dtype,
709 device=self.device,
@@ -649,7 +711,7 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
711 torch.zeros(
712 max_loras,
713 1,
652 - lora_config.max_lora_rank,
714 + lora_a_output_size_per_partition,
715 self.input_size,
716 dtype=lora_config.lora_dtype,
717 device=self.device,
@@ -657,7 +719,7 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
719 torch.zeros(
720 max_loras,
721 1,
660 - lora_config.max_lora_rank,
722 + lora_a_output_size_per_partition,
723 self.input_size,
724 dtype=lora_config.lora_dtype,
725 device=self.device,
@@ -705,6 +767,25 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
767 self.lora_a_stacked[2][index] = 0
768 self.lora_b_stacked[2][index] = 0
769
770 + def slice_lora_a(self, lora_a: List[torch.Tensor]) -> List[torch.Tensor]:
771 + return lora_a
772 +
773 + def slice_lora_b(self, lora_b: List[torch.Tensor]) -> List[torch.Tensor]:
774 + if lora_b[0] is not None:
775 + lora_b_q = lora_b[0][:, self.q_proj_shard_size *
776 + self.q_shard_id:self.q_proj_shard_size *
777 + (self.q_shard_id + 1)]
778 + if lora_b[1] is not None:
779 + lora_b_k = lora_b[1][:, self.kv_proj_shard_size *
780 + self.kv_shard_id:self.kv_proj_shard_size *
781 + (self.kv_shard_id + 1)]
782 + if lora_b[2] is not None:
783 + lora_b_v = lora_b[2][:, self.kv_proj_shard_size *
784 + self.kv_shard_id:self.kv_proj_shard_size *
785 + (self.kv_shard_id + 1)]
786 + lora_b = [lora_b_q, lora_b_k, lora_b_v]
787 + return lora_b
788 +
789 def set_lora(
790 self,
791 index: int,
@@ -715,40 +796,24 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
796 self.reset_lora(index)
797
798 if self.tp_size > 1:
718 - if lora_b[0] is not None:
719 - lora_b_q = lora_b[0][:, self.q_proj_shard_size *
720 - self.q_shard_id:self.q_proj_shard_size *
721 - (self.q_shard_id + 1)]
722 - self.lora_b_stacked[0][
723 - index, 0, :lora_b_q.shape[1], :lora_b_q.shape[0]].copy_(
724 - lora_b_q.T, non_blocking=True)
725 - if lora_b[1] is not None:
726 - lora_b_k = lora_b[1][:, self.kv_proj_shard_size *
727 - self.kv_shard_id:self.kv_proj_shard_size *
728 - (self.kv_shard_id + 1)]
729 - self.lora_b_stacked[1][
730 - index, 0, :lora_b_k.shape[1], :lora_b_k.shape[0]].copy_(
731 - lora_b_k.T, non_blocking=True)
732 - if lora_b[2] is not None:
733 - lora_b_v = lora_b[2][:, self.kv_proj_shard_size *
734 - self.kv_shard_id:self.kv_proj_shard_size *
735 - (self.kv_shard_id + 1)]
736 - self.lora_b_stacked[2][
737 - index, 0, :lora_b_v.shape[1], :lora_b_v.shape[0]].copy_(
738 - lora_b_v.T, non_blocking=True)
739 - else:
740 - if lora_b[0] is not None:
741 - self.lora_b_stacked[0][
742 - index, 0, :lora_b[0].shape[1], :lora_b[0].shape[0]].copy_(
743 - lora_b[0].T, non_blocking=True)
744 - if lora_b[1] is not None:
745 - self.lora_b_stacked[1][
746 - index, 0, :lora_b[1].shape[1], :lora_b[1].shape[0]].copy_(
747 - lora_b[1].T, non_blocking=True)
748 - if lora_b[2] is not None:
749 - self.lora_b_stacked[2][
750 - index, 0, :lora_b[2].shape[1], :lora_b[2].shape[0]].copy_(
751 - lora_b[2].T, non_blocking=True)
799 + lora_a = self.slice_lora_a(lora_a)
800 + lora_b = self.slice_lora_b(lora_b)
801 +
802 + if lora_b[0] is not None:
803 + lora_b_q = lora_b[0]
804 + self.lora_b_stacked[0][
805 + index, 0, :lora_b_q.shape[1], :lora_b_q.shape[0]].copy_(
806 + lora_b_q.T, non_blocking=True)
807 + if lora_b[1] is not None:
808 + lora_b_k = lora_b[1]
809 + self.lora_b_stacked[1][
810 + index, 0, :lora_b_k.shape[1], :lora_b_k.shape[0]].copy_(
811 + lora_b_k.T, non_blocking=True)
812 + if lora_b[2] is not None:
813 + lora_b_v = lora_b[2]
814 + self.lora_b_stacked[2][
815 + index, 0, :lora_b_v.shape[1], :lora_b_v.shape[0]].copy_(
816 + lora_b_v.T, non_blocking=True)
817
818 if lora_a[0] is not None:
819 self.lora_a_stacked[0][
@@ -777,6 +842,7 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
842 return output
843
844 @classmethod
845 + @_not_fully_sharded_can_replace
846 def can_replace_layer(cls, source_layer: nn.Module,
847 lora_config: LoRAConfig, packed_modules_list: List,
848 model_config: Optional[PretrainedConfig]) -> bool:
@@ -798,6 +864,8 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
864 max_loras: int,
865 lora_config: LoRAConfig,
866 model_config: Optional[PretrainedConfig] = None) -> None:
867 + self.lora_config = lora_config
868 + self.tp_rank = get_tensor_model_parallel_rank()
869 self.lora_a_stacked = torch.zeros(
870 (
871 max_loras,
@@ -808,11 +876,16 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
876 dtype=lora_config.lora_dtype,
877 device=self.device,
878 )
879 + tp_size = get_tensor_model_parallel_world_size()
880 + lora_b_output_size_per_partition = (
881 + self.output_size if not lora_config.fully_sharded_loras else
882 + divide(self.output_size, tp_size))
883 +
884 self.lora_b_stacked = torch.zeros(
885 (
886 max_loras,
887 1,
815 - self.output_size,
888 + lora_b_output_size_per_partition,
889 lora_config.max_lora_rank,
890 ),
891 dtype=lora_config.lora_dtype,
@@ -826,6 +899,17 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
899 self.lora_a_stacked[index] = 0
900 self.lora_b_stacked[index] = 0
901
902 + def slice_lora_a(self, lora_a: torch.Tensor) -> torch.Tensor:
903 + tensor_model_parallel_rank = get_tensor_model_parallel_rank()
904 + shard_size = self.input_size
905 + start_idx = tensor_model_parallel_rank * shard_size
906 + end_idx = (tensor_model_parallel_rank + 1) * shard_size
907 + lora_a = lora_a[start_idx:end_idx, :]
908 + return lora_a
909 +
910 + def slice_lora_b(self, lora_b: torch.Tensor) -> torch.Tensor:
911 + return lora_b
912 +
913 def set_lora(
914 self,
915 index: int,
@@ -834,12 +918,10 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
918 embeddings_tensor: Optional[torch.Tensor],
919 ):
920 self.reset_lora(index)
921 +
922 if self.base_layer.tp_size > 1:
838 - tensor_model_parallel_rank = get_tensor_model_parallel_rank()
839 - shard_size = self.input_size
840 - start_idx = tensor_model_parallel_rank * shard_size
841 - end_idx = (tensor_model_parallel_rank + 1) * shard_size
842 - lora_a = lora_a[start_idx:end_idx, :]
923 + lora_a = self.slice_lora_a(lora_a)
924 + lora_b = self.slice_lora_b(lora_b)
925
926 self.lora_a_stacked[index,
927 0, :lora_a.shape[1], :lora_a.shape[0]].copy_(
@@ -915,6 +997,7 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
997 self.base_layer, "weight") else self.base_layer.qweight
998
999 @classmethod
1000 + @_not_fully_sharded_can_replace
1001 def can_replace_layer(cls, source_layer: nn.Module,
1002 lora_config: LoRAConfig, packed_modules_list: List,
1003 model_config: Optional[PretrainedConfig]) -> bool:
@@ -1096,37 +1179,3 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
1179 model_config: Optional[PretrainedConfig]) -> bool:
1180 # Special handling for the LogitsProcessor.
1181 return False
1099 -
1100 -
1101 -_all_lora_classes: Set[Type[BaseLayerWithLoRA]] = {
1102 - cls
1103 - for cls in globals().values() if inspect.isclass(cls)
1104 - and issubclass(cls, BaseLayerWithLoRA) and cls is not BaseLayerWithLoRA
1105 -}
1106 -
1107 -
1108 -def from_layer(layer: nn.Module,
1109 - max_loras: int,
1110 - lora_config: LoRAConfig,
1111 - packed_modules_list: List,
1112 - model_config: Optional[PretrainedConfig] = None) -> nn.Module:
1113 - for lora_cls in _all_lora_classes:
1114 - if lora_cls.can_replace_layer(layer, lora_config, packed_modules_list,
1115 - model_config):
1116 - ret = lora_cls(layer)
1117 - ret.create_lora_weights(max_loras, lora_config, model_config)
1118 - return ret
1119 - return layer
1120 -
1121 -
1122 -def from_layer_logits_processor(
1123 - layer: LogitsProcessor,
1124 - lm_head: ParallelLMHead,
1125 - max_loras: int,
1126 - lora_config: LoRAConfig,
1127 - model_config: Optional[PretrainedConfig] = None,
1128 -) -> LogitsProcessorWithLoRA:
1129 - ret = LogitsProcessorWithLoRA(layer, lm_head.embedding_dim,
1130 - lm_head.weight.dtype, lm_head.weight.device)
1131 - ret.create_lora_weights(max_loras, lora_config, model_config)
1132 - return ret
vllm/lora/models.py
+3 -3
@@ -11,10 +11,10 @@ from torch import nn
11
12 from vllm.config import LoRAConfig
13 from vllm.logger import init_logger
14 -from vllm.lora.layers import (BaseLayerWithLoRA, LoRAMapping, from_layer,
15 - from_layer_logits_processor)
14 +from vllm.lora.layers import BaseLayerWithLoRA, LoRAMapping
15 from vllm.lora.lora import LoRALayerWeights, PackedLoRALayerWeights
17 -from vllm.lora.utils import parse_fine_tuned_lora_name, replace_submodule
16 +from vllm.lora.utils import (from_layer, from_layer_logits_processor,
17 + parse_fine_tuned_lora_name, replace_submodule)
18 from vllm.utils import LRUCache, is_pin_memory_available
19
20 logger = init_logger(__name__)
vllm/lora/punica.py
+43
@@ -49,6 +49,49 @@ def bgmv(
49 punica_kernels.dispatch_bgmv(y, x, w_t_all, indicies, layer_idx, scale)
50
51
52 +def dispatch_bgmv_low_level(y: torch.Tensor, x: torch.Tensor,
53 + w_t_all: torch.Tensor, indicies: torch.LongTensor,
54 + layer_idx: int, scale: float, y_offset: int,
55 + y_slice_size: int):
56 + """
57 + Same as `bgmv` but you can operate on slices of y.
58 + Pass whole y, define y_offset and y_slice_size.
59 +
60 + Semantics:
61 + y[i] += (
62 + x[i].unsqueeze(0)
63 + @ w_t_all[indices[i], layer_idx, :, :].transpose(-1, -2)
64 + * scale
65 + ).squeeze(0)
66 +
67 + Args:
68 + y: Shape: `[B, H2]`. Output vectors. Will be changed in-place.
69 + x: Shape: `[B, H1]`. Input vectors.
70 + w_t_all: Shape: `[None, L, y_slice_size, H1]`. Column partition of
71 + all of the transposed LoRA matrices.
72 + indicies: Shape: `[B]`. Indices of the LoRA weights.
73 + layer_idx: Layer index of LoRA weights.
74 + scale: Scaling factor.
75 + y_offset: Offset to apply to the starting column of y.
76 + y_slice_size: Size of the y column slice.
77 + """
78 + try:
79 + import vllm._punica_C as punica_kernels
80 + except ImportError as e:
81 + _raise_import_error(e)
82 + punica_kernels.dispatch_bgmv_low_level(
83 + y,
84 + x,
85 + w_t_all,
86 + indicies,
87 + layer_idx,
88 + scale,
89 + x.size(1),
90 + y_slice_size,
91 + y_offset,
92 + )
93 +
94 +
95 def add_lora(y: torch.Tensor,
96 x: torch.Tensor,
97 wa_t_all: torch.Tensor,
vllm/lora/utils.py
+59 -1
@@ -1,11 +1,69 @@
1 -from typing import Tuple
1 +from typing import List, Optional, Set, Tuple, Type
2
3 from torch import nn
4 +from transformers import PretrainedConfig
5
6 +from vllm.config import LoRAConfig
7 from vllm.logger import init_logger
8 +from vllm.lora.fully_sharded_layers import (
9 + ColumnParallelLinearWithShardedLoRA,
10 + MergedColumnParallelLinearWithShardedLoRA,
11 + MergedQKVParallelLinearWithShardedLora, RowParallelLinearWithShardedLoRA)
12 +# being imported for _all_lora_classes below
13 +# yapf conflicts with isort for this block
14 +# yapf: disable
15 +from vllm.lora.layers import (BaseLayerWithLoRA, ColumnParallelLinearWithLoRA,
16 + LogitsProcessorWithLoRA,
17 + MergedColumnParallelLinearWithLoRA,
18 + MergedQKVParallelLinearWithLora,
19 + QKVParallelLinearWithLora,
20 + RowParallelLinearWithLoRA,
21 + VocabParallelEmbeddingWithLoRA)
22 +# yapf: enable
23 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
24 +from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
25
26 logger = init_logger(__name__)
27
28 +_all_lora_classes: Set[Type[BaseLayerWithLoRA]] = {
29 + VocabParallelEmbeddingWithLoRA, ColumnParallelLinearWithLoRA,
30 + MergedColumnParallelLinearWithLoRA, QKVParallelLinearWithLora,
31 + MergedQKVParallelLinearWithLora, RowParallelLinearWithLoRA,
32 + LogitsProcessorWithLoRA, ColumnParallelLinearWithShardedLoRA,
33 + MergedColumnParallelLinearWithShardedLoRA,
34 + MergedQKVParallelLinearWithShardedLora, RowParallelLinearWithShardedLoRA
35 +}
36 +
37 +
38 +def from_layer(layer: nn.Module,
39 + max_loras: int,
40 + lora_config: LoRAConfig,
41 + packed_modules_list: List,
42 + model_config: Optional[PretrainedConfig] = None) -> nn.Module:
43 + for lora_cls in _all_lora_classes:
44 + # specifying kwargs so they can be easily accessed in decorator
45 + if lora_cls.can_replace_layer(source_layer=layer,
46 + lora_config=lora_config,
47 + packed_modules_list=packed_modules_list,
48 + model_config=model_config):
49 + ret = lora_cls(layer)
50 + ret.create_lora_weights(max_loras, lora_config, model_config)
51 + return ret
52 + return layer
53 +
54 +
55 +def from_layer_logits_processor(
56 + layer: LogitsProcessor,
57 + lm_head: ParallelLMHead,
58 + max_loras: int,
59 + lora_config: LoRAConfig,
60 + model_config: Optional[PretrainedConfig] = None,
61 +) -> LogitsProcessorWithLoRA:
62 + ret = LogitsProcessorWithLoRA(layer, lm_head.embedding_dim,
63 + lm_head.weight.dtype, lm_head.weight.device)
64 + ret.create_lora_weights(max_loras, lora_config, model_config)
65 + return ret
66 +
67
68 def replace_submodule(model: nn.Module, module_name: str,
69 new_module: nn.Module) -> nn.Module: