A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 22:15 UTC
[Bugfix] Fix precision loss in LoRA-wrapped RowParallelLinear by fusing bias into GEMM (#28972)
Signed-off-by: prashanth058 <prashanth.dannamaneni@uipath.com>
prashanth058 committed
Nov 19, 2025 at 19:50 UTC
0cca9b4d130b4caddb60086ef26a0d8741582dcb
1 file changed
+10
-15
vllm/lora/layers/row_parallel_linear.py
+10
-15
@@ -63,23 +63,18 @@ class RowParallelLinearWithLoRA(BaseLinearLayerWithLoRA):
63
input_parallel = splitted_input[self.tp_rank].contiguous()
64
65
# Matrix multiply.
66
- output_parallel = self.apply(input_parallel)
66
+ bias_ = (
67
+ None
68
+ if (self.tp_rank > 0 or self.base_layer.skip_bias_add)
69
+ else self.base_layer.bias
70
+ )
71
+ output_parallel = self.apply(input_parallel, bias_)
72
if self.base_layer.reduce_results and self.tp_size > 1:
68
- output_ = tensor_model_parallel_all_reduce(output_parallel)
69
- else:
70
- output_ = output_parallel
71
-
72
- if not self.base_layer.skip_bias_add:
73
- output = (
74
- output_ + self.base_layer.bias
75
- if self.base_layer.bias is not None
76
- else output_
77
- )
78
- output_bias = None
73
+ output = tensor_model_parallel_all_reduce(output_parallel)
74
else:
80
- output = output_
81
- output_bias = self.base_layer.bias
75
+ output = output_parallel
76
77
+ output_bias = self.base_layer.bias if self.base_layer.skip_bias_add else None
78
if not self.base_layer.return_bias:
79
return output
80
@@ -120,7 +115,7 @@ class RowParallelLinearWithShardedLoRA(RowParallelLinearWithLoRA):
115
return lora_b
116
117
def apply(self, x: torch.Tensor, bias: torch.Tensor | None = None) -> torch.Tensor:
123
- output = self.base_layer.quant_method.apply(self.base_layer, x)
118
+ output = self.base_layer.quant_method.apply(self.base_layer, x, bias)
119
120
x = x.view(-1, x.shape[-1])
121
output, out_orig_shape = output.view(-1, output.shape[-1]), output.shape