A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[Bugfix] Fix fully sharded LoRA bug (#10352)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Nov 15, 2024 at 18:34 UTC
1d65ec7eeb35f03eb87ed080094f1aa5ff2ae3d3
3 files changed
+21
-19
vllm/lora/fully_sharded_layers.py
+12
-11
@@ -165,15 +165,14 @@ class MergedColumnParallelLinearWithShardedLoRA(
165
def slice_lora_a(
166
self, lora_a: List[Union[torch.Tensor, None]]
167
) -> List[Union[torch.Tensor, None]]:
168
- if lora_a[0] is None or lora_a[1] is None:
169
- return lora_a
168
+ #NOTE: lora_a contains 2 subloras, and each sublora could be None.
169
output_shard_size = self.lora_a_stacked[0].shape[2]
170
output_start_idx = self.tp_rank * output_shard_size
171
lora_a = [
173
- lora_a[0][:,
174
- output_start_idx:output_start_idx + output_shard_size],
175
- lora_a[1][:,
176
- output_start_idx:output_start_idx + output_shard_size],
172
+ lora_a[0][:, output_start_idx:output_start_idx +
173
+ output_shard_size] if lora_a[0] is not None else None,
174
+ lora_a[1][:, output_start_idx:output_start_idx +
175
+ output_shard_size] if lora_a[1] is not None else None,
176
]
177
return lora_a
178
@@ -261,14 +260,16 @@ class MergedQKVParallelLinearWithShardedLora(MergedQKVParallelLinearWithLora):
260
def slice_lora_a(
261
self, lora_a: List[Union[torch.Tensor, None]]
262
) -> List[Union[torch.Tensor, None]]:
264
- if lora_a[0] is None or lora_a[1] is None or lora_a[2] is None:
265
- return lora_a
263
+ # NOTE: lora_a contains 3 subloras, and each sublora could be None.
264
shard_size = [self.lora_a_stacked[i].shape[2] for i in range(3)]
265
start_idx = [self.tp_rank * shard_size[i] for i in range(3)]
266
lora_a = [
269
- lora_a[0][:, start_idx[0]:start_idx[0] + shard_size[0]],
270
- lora_a[1][:, start_idx[1]:start_idx[1] + shard_size[1]],
271
- lora_a[2][:, start_idx[2]:start_idx[2] + shard_size[2]],
267
+ lora_a[0][:, start_idx[0]:start_idx[0] +
268
+ shard_size[0]] if lora_a[0] is not None else None,
269
+ lora_a[1][:, start_idx[1]:start_idx[1] +
270
+ shard_size[1]] if lora_a[1] is not None else None,
271
+ lora_a[2][:, start_idx[2]:start_idx[2] +
272
+ shard_size[2]] if lora_a[2] is not None else None,
273
]
274
return lora_a
275
vllm/lora/layers.py
+8
-7
@@ -685,26 +685,27 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
685
def slice_lora_b(
686
self, lora_b: List[Union[torch.Tensor, None]]
687
) -> List[Union[torch.Tensor, None]]:
688
- if lora_b[0] is None or lora_b[1] is None:
689
- return lora_b
688
+ #NOTE: lora_b contains 2 subloras, and each sublora could be None.
689
shard_size = self.output_dim
690
start_idx = self.tp_rank * shard_size
691
end_idx = (self.tp_rank + 1) * shard_size
692
lora_b = [
694
- lora_b[0][:, start_idx:end_idx],
695
- lora_b[1][:, start_idx:end_idx],
693
+ lora_b[0][:, start_idx:end_idx] if lora_b[0] is not None else None,
694
+ lora_b[1][:, start_idx:end_idx] if lora_b[1] is not None else None,
695
]
696
return lora_b
697
698
def slice_bias(
699
self, bias: List[Union[torch.Tensor,
700
None]]) -> List[Union[torch.Tensor, None]]:
702
- if bias[0] is None or bias[1] is None:
703
- return bias
701
+ # NOTE : each bias could be None.
702
shard_size = self.output_dim
703
start_idx = self.tp_rank * shard_size
704
end_idx = (self.tp_rank + 1) * shard_size
707
- bias = [bias[0][start_idx:end_idx], bias[1][start_idx:end_idx]]
705
+ bias = [
706
+ bias[0][start_idx:end_idx] if bias[0] is not None else None,
707
+ bias[1][start_idx:end_idx] if bias[1] is not None else None
708
+ ]
709
return bias
710
711
def set_lora(
vllm/worker/worker.py
+1
-1
@@ -232,7 +232,7 @@ class Worker(LocalOrDistributedWorkerBase):
232
logger.info(
233
"Memory profiling results: total_gpu_memory=%.2fGiB"
234
" initial_memory_usage=%.2fGiB peak_torch_memory=%.2fGiB"
235
- " memory_usage_post_profile=%.2fGib"
235
+ " memory_usage_post_profile=%.2fGiB"
236
" non_torch_memory=%.2fGiB kv_cache_size=%.2fGiB"
237
" gpu_memory_utilization=%.2f", total_gpu_memory / (1024**3),
238
(total_gpu_memory - free_memory_pre_profile) / (1024**3),