A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[Bugfix] Fix ColumnParallelLinearWithLoRA slice (#11708)
Signed-off-by: ZincCat <zincchloride@outlook.com>
ZincCat committed
Jan 3, 2025 at 13:02 UTC
61fed92c7e646d6f2ec5d9de54568a860870e6a4
1 file changed
+2
-2
vllm/lora/layers.py
+2
-2
@@ -479,7 +479,7 @@ class ColumnParallelLinearWithLoRA(BaseLinearLayerWithLoRA):
479
# ColumnParallelLinear.
480
else:
481
tensor_model_parallel_rank = get_tensor_model_parallel_rank()
482
- shard_size = self.output_dim
482
+ shard_size = self.output_size
483
start_idx = tensor_model_parallel_rank * shard_size
484
end_idx = (tensor_model_parallel_rank + 1) * shard_size
485
lora_b = lora_b[:, start_idx:end_idx]
@@ -490,7 +490,7 @@ class ColumnParallelLinearWithLoRA(BaseLinearLayerWithLoRA):
490
if bias is None:
491
return bias
492
tensor_model_parallel_rank = get_tensor_model_parallel_rank()
493
- shard_size = self.output_dim
493
+ shard_size = self.output_size
494
start_idx = tensor_model_parallel_rank * shard_size
495
end_idx = (tensor_model_parallel_rank + 1) * shard_size
496
bias = bias[start_idx:end_idx]