@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a4cf256159
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Bugfix] Fix QKVParallelLinearWithShardedLora bias bug (#10844)

Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>

Jee Jee Li committed Dec 3, 2024 at 12:10 UTC a4cf2561599448d4a5c3de4d79c73ca37cb8d647
2 files changed +1 -9
.buildkite/test-pipeline.yaml
-1
@@ -481,7 +481,6 @@ steps:
481
482 - label: LoRA TP Test (Distributed)
483 num_gpus: 4
484 - soft_fail: true
484 source_file_dependencies:
485 - vllm/lora
486 - tests/lora
vllm/lora/fully_sharded_layers.py
+1 -8
@@ -77,13 +77,6 @@ class ColumnParallelLinearWithShardedLoRA(ColumnParallelLinearWithLoRA):
77 add_input=True)
78 # now have column partitioned output
79
80 - if self.bias_stacked is not None:
81 - self.bias_stacked = self.bias_stacked.view(
82 - -1, self.bias_stacked.shape[-1])
83 - self.bias_stacked = self.bias_stacked[
84 - self.punica_wrapper.token_lora_indices]
85 - output += self.bias_stacked
86 -
80 output = output.view(*out_orig_shape)
81 return output
82
@@ -222,7 +215,7 @@ class QKVParallelLinearWithShardedLora(QKVParallelLinearWithLora):
215 self.punica_wrapper.add_expand(output,
216 buffer,
217 self.lora_b_stacked,
225 - self.bias_all,
218 + self.bias_stacked,
219 add_input=True)
220 # now have column partitioned output
221 output = output.view(*out_orig_shape)