A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 07:15 UTC
[Bugfix] Fix for issue 17396 (#18773)
Signed-off-by: Fred Reiss <frreiss@us.ibm.com>
Fred Reiss committed
May 31, 2025 at 04:58 UTC
f2c3f66d59f9e38aa94985b54f370219222e7bd1
1 file changed
+5
-2
vllm/lora/ops/torch_ops/lora_ops.py
+5
-2
@@ -36,10 +36,13 @@ def bgmv_expand(inputs: torch.Tensor,
36
if outputs.shape[0] == 1 and output_tensor.shape[0] != 1:
37
limit = 1
38
39
+ # LoRA adapter and model may add different amounts of padding to output
40
+ common_len = min(outputs.shape[1], output_tensor.shape[1])
41
+
42
if add_inputs:
40
- output_tensor[:, :outputs.shape[1]] += outputs[:limit, :]
43
+ output_tensor[:, :common_len] += outputs[:limit, :common_len]
44
else:
42
- output_tensor[:, :outputs.shape[1]] = outputs[:limit, :]
45
+ output_tensor[:, :common_len] = outputs[:limit, :common_len]
46
47
48
def sgmv_shrink(