A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[Bugfix] Fix QKVParallelLinearWithShardedLora bias bug (#10844)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Dec 3, 2024 at 12:10 UTC
a4cf2561599448d4a5c3de4d79c73ca37cb8d647
2 files changed
+1
-9
.buildkite/test-pipeline.yaml
-1
@@ -481,7 +481,6 @@ steps:
481
482
- label: LoRA TP Test (Distributed)
483
num_gpus: 4
484
- soft_fail: true
484
source_file_dependencies:
485
- vllm/lora
486
- tests/lora
vllm/lora/fully_sharded_layers.py
+1
-8
@@ -77,13 +77,6 @@ class ColumnParallelLinearWithShardedLoRA(ColumnParallelLinearWithLoRA):
77
add_input=True)
78
# now have column partitioned output
79
80
- if self.bias_stacked is not None:
81
- self.bias_stacked = self.bias_stacked.view(
82
- -1, self.bias_stacked.shape[-1])
83
- self.bias_stacked = self.bias_stacked[
84
- self.punica_wrapper.token_lora_indices]
85
- output += self.bias_stacked
86
-
80
output = output.view(*out_orig_shape)
81
return output
82
@@ -222,7 +215,7 @@ class QKVParallelLinearWithShardedLora(QKVParallelLinearWithLora):
215
self.punica_wrapper.add_expand(output,
216
buffer,
217
self.lora_b_stacked,
225
- self.bias_all,
218
+ self.bias_stacked,
219
add_input=True)
220
# now have column partitioned output
221
output = output.view(*out_orig_shape)