A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Bugfix] Fixed error in slice_lora_b for MergedQKVParallelLinearWithLora (#4609)
Austin Veselka committed
May 7, 2024 at 12:59 UTC
10760da8003824e208c94fb2bfcdb6fdd0f4edda
2 files changed
+52
-32
vllm/lora/fully_sharded_layers.py
+30
-24
@@ -1,5 +1,5 @@
1
# pylint: disable=unused-argument
2
-from typing import TYPE_CHECKING, List, Optional
2
+from typing import TYPE_CHECKING, List, Optional, Union
3
4
import torch
5
import torch.nn as nn
@@ -51,10 +51,9 @@ class ColumnParallelLinearWithShardedLoRA(ColumnParallelLinearWithLoRA):
51
lora_a = lora_a[:, start_idx:start_idx + shard_size]
52
return lora_a
53
54
- def apply_weights(self, x: torch.Tensor,
55
- bias: Optional[torch.Tensor]) -> torch.Tensor:
56
- output = self.base_layer.linear_method.apply_weights(
57
- self.base_layer, x, bias)
54
+ def apply(self, x: torch.Tensor,
55
+ bias: Optional[torch.Tensor]) -> torch.Tensor:
56
+ output = self.base_layer.quant_method.apply(self.base_layer, x, bias)
57
58
x = x.view(-1, x.shape[-1])
59
output, out_orig_shape = output.view(-1,
@@ -88,7 +87,7 @@ class ColumnParallelLinearWithShardedLoRA(ColumnParallelLinearWithLoRA):
87
)
88
89
91
-def _mcp_apply_weights(x, bias, layer):
90
+def _mcp_apply(x, bias, layer):
91
"""
92
MergedColumnParallelLinearWithShardedLoRA and
93
QKVParallelLinearWithShardedLora share the same
@@ -100,8 +99,7 @@ def _mcp_apply_weights(x, bias, layer):
99
"""
100
# expecting 2 for column parallel and 3 for qkv
101
n = len(layer.lora_a_stacked)
103
- output = layer.base_layer.linear_method.apply_weights(
104
- layer.base_layer, x, bias)
102
+ output = layer.base_layer.quant_method.apply(layer.base_layer, x, bias)
103
104
x = x.view(-1, x.shape[-1])
105
output, out_orig_shape = output.view(-1, output.shape[-1]), output.shape
@@ -136,18 +134,23 @@ class MergedColumnParallelLinearWithShardedLoRA(
134
Based on S-LoRA, slicing happens along the rank dim.
135
"""
136
139
- def slice_lora_a(self, lora_a: List[torch.Tensor]) -> List[torch.Tensor]:
137
+ def slice_lora_a(
138
+ self, lora_a: List[Union[torch.Tensor, None]]
139
+ ) -> List[Union[torch.Tensor, None]]:
140
+ if lora_a[0] is None or lora_a[1] is None:
141
+ return lora_a
142
output_shard_size = self.lora_a_stacked[0].shape[2]
143
output_start_idx = self.tp_rank * output_shard_size
144
lora_a = [
143
- lora_a[i][:, output_start_idx:output_start_idx + output_shard_size]
144
- for i in range(2)
145
+ lora_a[0][:,
146
+ output_start_idx:output_start_idx + output_shard_size],
147
+ lora_a[1][:, output_start_idx:output_start_idx + output_shard_size]
148
]
149
return lora_a
150
148
- def apply_weights(self, x: torch.Tensor,
149
- bias: Optional[torch.Tensor]) -> torch.Tensor:
150
- return _mcp_apply_weights(x, bias, self)
151
+ def apply(self, x: torch.Tensor,
152
+ bias: Optional[torch.Tensor]) -> torch.Tensor:
153
+ return _mcp_apply(x, bias, self)
154
155
@classmethod
156
@_fully_sharded_can_replace
@@ -172,19 +175,23 @@ class MergedQKVParallelLinearWithShardedLora(MergedQKVParallelLinearWithLora):
175
Based on S-LoRA, slicing happens along the rank dim.
176
"""
177
175
- def slice_lora_a(self, lora_a: List[torch.Tensor]) -> List[torch.Tensor]:
178
+ def slice_lora_a(
179
+ self, lora_a: List[Union[torch.Tensor, None]]
180
+ ) -> List[Union[torch.Tensor, None]]:
181
+ if lora_a[0] is None or lora_a[1] is None or lora_a[2] is None:
182
+ return lora_a
183
shard_size = [self.lora_a_stacked[i].shape[2] for i in range(3)]
184
start_idx = [self.tp_rank * shard_size[i] for i in range(3)]
185
lora_a = [
179
- lora_a[i][:, start_idx[i]:start_idx[i] +
180
- shard_size[i]] if lora_a[i] is not None else None
181
- for i in range(3)
186
+ lora_a[0][:, start_idx[0]:start_idx[0] + shard_size[0]],
187
+ lora_a[1][:, start_idx[1]:start_idx[1] + shard_size[1]],
188
+ lora_a[2][:, start_idx[2]:start_idx[2] + shard_size[2]]
189
]
190
return lora_a
191
185
- def apply_weights(self, x: torch.Tensor,
186
- bias: Optional[torch.Tensor]) -> torch.Tensor:
187
- return _mcp_apply_weights(x, bias, self)
192
+ def apply(self, x: torch.Tensor,
193
+ bias: Optional[torch.Tensor]) -> torch.Tensor:
194
+ return _mcp_apply(x, bias, self)
195
196
@classmethod
197
@_fully_sharded_can_replace
@@ -218,9 +225,8 @@ class RowParallelLinearWithShardedLoRA(RowParallelLinearWithLoRA):
225
lora_b = lora_b[:, start_idx:end_idx]
226
return lora_b
227
221
- def apply_weights(self, x: torch.Tensor) -> torch.Tensor:
222
- output = self.base_layer.linear_method.apply_weights(
223
- self.base_layer, x)
228
+ def apply(self, x: torch.Tensor) -> torch.Tensor:
229
+ output = self.base_layer.quant_method.apply(self.base_layer, x)
230
231
x = x.view(-1, x.shape[-1])
232
output, out_orig_shape = output.view(-1,
vllm/lora/layers.py
+22
-8
@@ -1,7 +1,7 @@
1
# pylint: disable=unused-argument
2
import math
3
from dataclasses import dataclass
4
-from typing import TYPE_CHECKING, List, Optional, Tuple
4
+from typing import TYPE_CHECKING, List, Optional, Tuple, Union
5
6
import torch
7
import torch.nn as nn
@@ -145,11 +145,15 @@ class LoRAMapping:
145
146
class BaseLayerWithLoRA(nn.Module):
147
148
- def slice_lora_a(self, lora_a: torch.Tensor) -> torch.Tensor:
148
+ def slice_lora_a(
149
+ self, lora_a: Union[torch.Tensor, List[Union[torch.Tensor, None]]]
150
+ ) -> Union[torch.Tensor, List[Union[torch.Tensor, None]]]:
151
"""Slice lora a if splitting for tensor parallelism."""
152
...
153
152
- def slice_lora_b(self, lora_b: torch.Tensor) -> torch.Tensor:
154
+ def slice_lora_b(
155
+ self, lora_b: Union[torch.Tensor, List[Union[torch.Tensor, None]]]
156
+ ) -> Union[torch.Tensor, List[Union[torch.Tensor, None]]]:
157
"""Slice lora b if splitting with tensor parallelism."""
158
...
159
@@ -539,10 +543,16 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
543
self.lora_b_stacked[0][index] = 0
544
self.lora_b_stacked[1][index] = 0
545
542
- def slice_lora_a(self, lora_a: List[torch.Tensor]) -> List[torch.Tensor]:
546
+ def slice_lora_a(
547
+ self, lora_a: List[Union[torch.Tensor, None]]
548
+ ) -> List[Union[torch.Tensor, None]]:
549
return lora_a
550
545
- def slice_lora_b(self, lora_b: List[torch.Tensor]) -> List[torch.Tensor]:
551
+ def slice_lora_b(
552
+ self, lora_b: List[Union[torch.Tensor, None]]
553
+ ) -> List[Union[torch.Tensor, None]]:
554
+ if lora_b[0] is None or lora_b[1] is None:
555
+ return lora_b
556
shard_size = self.output_dim
557
start_idx = self.tp_rank * shard_size
558
end_idx = (self.tp_rank + 1) * shard_size
@@ -767,10 +777,15 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
777
self.lora_a_stacked[2][index] = 0
778
self.lora_b_stacked[2][index] = 0
779
770
- def slice_lora_a(self, lora_a: List[torch.Tensor]) -> List[torch.Tensor]:
780
+ def slice_lora_a(
781
+ self, lora_a: List[Union[torch.Tensor, None]]
782
+ ) -> List[Union[torch.Tensor, None]]:
783
return lora_a
784
773
- def slice_lora_b(self, lora_b: List[torch.Tensor]) -> List[torch.Tensor]:
785
+ def slice_lora_b(
786
+ self, lora_b: List[Union[torch.Tensor, None]]
787
+ ) -> List[Union[torch.Tensor, None]]:
788
+ lora_b_q, lora_b_k, lora_b_v = None, None, None
789
if lora_b[0] is not None:
790
lora_b_q = lora_b[0][:, self.q_proj_shard_size *
791
self.q_shard_id:self.q_proj_shard_size *
@@ -992,7 +1007,6 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
1007
1008
@property
1009
def weight(self):
995
-
1010
return self.base_layer.weight if hasattr(
1011
self.base_layer, "weight") else self.base_layer.qweight
1012