@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 61fed92c7e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[Bugfix] Fix ColumnParallelLinearWithLoRA slice (#11708)

Signed-off-by: ZincCat <zincchloride@outlook.com>

ZincCat committed Jan 3, 2025 at 13:02 UTC 61fed92c7e646d6f2ec5d9de54568a860870e6a4
1 file changed +2 -2
vllm/lora/layers.py
+2 -2
@@ -479,7 +479,7 @@ class ColumnParallelLinearWithLoRA(BaseLinearLayerWithLoRA):
479 # ColumnParallelLinear.
480 else:
481 tensor_model_parallel_rank = get_tensor_model_parallel_rank()
482 - shard_size = self.output_dim
482 + shard_size = self.output_size
483 start_idx = tensor_model_parallel_rank * shard_size
484 end_idx = (tensor_model_parallel_rank + 1) * shard_size
485 lora_b = lora_b[:, start_idx:end_idx]
@@ -490,7 +490,7 @@ class ColumnParallelLinearWithLoRA(BaseLinearLayerWithLoRA):
490 if bias is None:
491 return bias
492 tensor_model_parallel_rank = get_tensor_model_parallel_rank()
493 - shard_size = self.output_dim
493 + shard_size = self.output_size
494 start_idx = tensor_model_parallel_rank * shard_size
495 end_idx = (tensor_model_parallel_rank + 1) * shard_size
496 bias = bias[start_idx:end_idx]