@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 10760da800
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Bugfix] Fixed error in slice_lora_b for MergedQKVParallelLinearWithLora (#4609)

Austin Veselka committed May 7, 2024 at 12:59 UTC 10760da8003824e208c94fb2bfcdb6fdd0f4edda
2 files changed +52 -32
vllm/lora/fully_sharded_layers.py
+30 -24
@@ -1,5 +1,5 @@
1 # pylint: disable=unused-argument
2 -from typing import TYPE_CHECKING, List, Optional
2 +from typing import TYPE_CHECKING, List, Optional, Union
3
4 import torch
5 import torch.nn as nn
@@ -51,10 +51,9 @@ class ColumnParallelLinearWithShardedLoRA(ColumnParallelLinearWithLoRA):
51 lora_a = lora_a[:, start_idx:start_idx + shard_size]
52 return lora_a
53
54 - def apply_weights(self, x: torch.Tensor,
55 - bias: Optional[torch.Tensor]) -> torch.Tensor:
56 - output = self.base_layer.linear_method.apply_weights(
57 - self.base_layer, x, bias)
54 + def apply(self, x: torch.Tensor,
55 + bias: Optional[torch.Tensor]) -> torch.Tensor:
56 + output = self.base_layer.quant_method.apply(self.base_layer, x, bias)
57
58 x = x.view(-1, x.shape[-1])
59 output, out_orig_shape = output.view(-1,
@@ -88,7 +87,7 @@ class ColumnParallelLinearWithShardedLoRA(ColumnParallelLinearWithLoRA):
87 )
88
89
91 -def _mcp_apply_weights(x, bias, layer):
90 +def _mcp_apply(x, bias, layer):
91 """
92 MergedColumnParallelLinearWithShardedLoRA and
93 QKVParallelLinearWithShardedLora share the same
@@ -100,8 +99,7 @@ def _mcp_apply_weights(x, bias, layer):
99 """
100 # expecting 2 for column parallel and 3 for qkv
101 n = len(layer.lora_a_stacked)
103 - output = layer.base_layer.linear_method.apply_weights(
104 - layer.base_layer, x, bias)
102 + output = layer.base_layer.quant_method.apply(layer.base_layer, x, bias)
103
104 x = x.view(-1, x.shape[-1])
105 output, out_orig_shape = output.view(-1, output.shape[-1]), output.shape
@@ -136,18 +134,23 @@ class MergedColumnParallelLinearWithShardedLoRA(
134 Based on S-LoRA, slicing happens along the rank dim.
135 """
136
139 - def slice_lora_a(self, lora_a: List[torch.Tensor]) -> List[torch.Tensor]:
137 + def slice_lora_a(
138 + self, lora_a: List[Union[torch.Tensor, None]]
139 + ) -> List[Union[torch.Tensor, None]]:
140 + if lora_a[0] is None or lora_a[1] is None:
141 + return lora_a
142 output_shard_size = self.lora_a_stacked[0].shape[2]
143 output_start_idx = self.tp_rank * output_shard_size
144 lora_a = [
143 - lora_a[i][:, output_start_idx:output_start_idx + output_shard_size]
144 - for i in range(2)
145 + lora_a[0][:,
146 + output_start_idx:output_start_idx + output_shard_size],
147 + lora_a[1][:, output_start_idx:output_start_idx + output_shard_size]
148 ]
149 return lora_a
150
148 - def apply_weights(self, x: torch.Tensor,
149 - bias: Optional[torch.Tensor]) -> torch.Tensor:
150 - return _mcp_apply_weights(x, bias, self)
151 + def apply(self, x: torch.Tensor,
152 + bias: Optional[torch.Tensor]) -> torch.Tensor:
153 + return _mcp_apply(x, bias, self)
154
155 @classmethod
156 @_fully_sharded_can_replace
@@ -172,19 +175,23 @@ class MergedQKVParallelLinearWithShardedLora(MergedQKVParallelLinearWithLora):
175 Based on S-LoRA, slicing happens along the rank dim.
176 """
177
175 - def slice_lora_a(self, lora_a: List[torch.Tensor]) -> List[torch.Tensor]:
178 + def slice_lora_a(
179 + self, lora_a: List[Union[torch.Tensor, None]]
180 + ) -> List[Union[torch.Tensor, None]]:
181 + if lora_a[0] is None or lora_a[1] is None or lora_a[2] is None:
182 + return lora_a
183 shard_size = [self.lora_a_stacked[i].shape[2] for i in range(3)]
184 start_idx = [self.tp_rank * shard_size[i] for i in range(3)]
185 lora_a = [
179 - lora_a[i][:, start_idx[i]:start_idx[i] +
180 - shard_size[i]] if lora_a[i] is not None else None
181 - for i in range(3)
186 + lora_a[0][:, start_idx[0]:start_idx[0] + shard_size[0]],
187 + lora_a[1][:, start_idx[1]:start_idx[1] + shard_size[1]],
188 + lora_a[2][:, start_idx[2]:start_idx[2] + shard_size[2]]
189 ]
190 return lora_a
191
185 - def apply_weights(self, x: torch.Tensor,
186 - bias: Optional[torch.Tensor]) -> torch.Tensor:
187 - return _mcp_apply_weights(x, bias, self)
192 + def apply(self, x: torch.Tensor,
193 + bias: Optional[torch.Tensor]) -> torch.Tensor:
194 + return _mcp_apply(x, bias, self)
195
196 @classmethod
197 @_fully_sharded_can_replace
@@ -218,9 +225,8 @@ class RowParallelLinearWithShardedLoRA(RowParallelLinearWithLoRA):
225 lora_b = lora_b[:, start_idx:end_idx]
226 return lora_b
227
221 - def apply_weights(self, x: torch.Tensor) -> torch.Tensor:
222 - output = self.base_layer.linear_method.apply_weights(
223 - self.base_layer, x)
228 + def apply(self, x: torch.Tensor) -> torch.Tensor:
229 + output = self.base_layer.quant_method.apply(self.base_layer, x)
230
231 x = x.view(-1, x.shape[-1])
232 output, out_orig_shape = output.view(-1,
vllm/lora/layers.py
+22 -8
@@ -1,7 +1,7 @@
1 # pylint: disable=unused-argument
2 import math
3 from dataclasses import dataclass
4 -from typing import TYPE_CHECKING, List, Optional, Tuple
4 +from typing import TYPE_CHECKING, List, Optional, Tuple, Union
5
6 import torch
7 import torch.nn as nn
@@ -145,11 +145,15 @@ class LoRAMapping:
145
146 class BaseLayerWithLoRA(nn.Module):
147
148 - def slice_lora_a(self, lora_a: torch.Tensor) -> torch.Tensor:
148 + def slice_lora_a(
149 + self, lora_a: Union[torch.Tensor, List[Union[torch.Tensor, None]]]
150 + ) -> Union[torch.Tensor, List[Union[torch.Tensor, None]]]:
151 """Slice lora a if splitting for tensor parallelism."""
152 ...
153
152 - def slice_lora_b(self, lora_b: torch.Tensor) -> torch.Tensor:
154 + def slice_lora_b(
155 + self, lora_b: Union[torch.Tensor, List[Union[torch.Tensor, None]]]
156 + ) -> Union[torch.Tensor, List[Union[torch.Tensor, None]]]:
157 """Slice lora b if splitting with tensor parallelism."""
158 ...
159
@@ -539,10 +543,16 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
543 self.lora_b_stacked[0][index] = 0
544 self.lora_b_stacked[1][index] = 0
545
542 - def slice_lora_a(self, lora_a: List[torch.Tensor]) -> List[torch.Tensor]:
546 + def slice_lora_a(
547 + self, lora_a: List[Union[torch.Tensor, None]]
548 + ) -> List[Union[torch.Tensor, None]]:
549 return lora_a
550
545 - def slice_lora_b(self, lora_b: List[torch.Tensor]) -> List[torch.Tensor]:
551 + def slice_lora_b(
552 + self, lora_b: List[Union[torch.Tensor, None]]
553 + ) -> List[Union[torch.Tensor, None]]:
554 + if lora_b[0] is None or lora_b[1] is None:
555 + return lora_b
556 shard_size = self.output_dim
557 start_idx = self.tp_rank * shard_size
558 end_idx = (self.tp_rank + 1) * shard_size
@@ -767,10 +777,15 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
777 self.lora_a_stacked[2][index] = 0
778 self.lora_b_stacked[2][index] = 0
779
770 - def slice_lora_a(self, lora_a: List[torch.Tensor]) -> List[torch.Tensor]:
780 + def slice_lora_a(
781 + self, lora_a: List[Union[torch.Tensor, None]]
782 + ) -> List[Union[torch.Tensor, None]]:
783 return lora_a
784
773 - def slice_lora_b(self, lora_b: List[torch.Tensor]) -> List[torch.Tensor]:
785 + def slice_lora_b(
786 + self, lora_b: List[Union[torch.Tensor, None]]
787 + ) -> List[Union[torch.Tensor, None]]:
788 + lora_b_q, lora_b_k, lora_b_v = None, None, None
789 if lora_b[0] is not None:
790 lora_b_q = lora_b[0][:, self.q_proj_shard_size *
791 self.q_shard_id:self.q_proj_shard_size *
@@ -992,7 +1007,6 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
1007
1008 @property
1009 def weight(self):
995 -
1010 return self.base_layer.weight if hasattr(
1011 self.base_layer, "weight") else self.base_layer.qweight
1012