A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
Remove LoRA bias support (#25807)
Signed-off-by: Ashwin Phadke <ashwinphadke12@rediffmail.com> Signed-off-by: Ashwin Phadke <23502062+ashwin-phadke@users.noreply.github.com> Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
Ashwin Phadke committed
Oct 10, 2025 at 15:20 UTC
ab196edefb66a3f30bd21a75d4706937ca8a750a
20 files changed
+35
-366
tests/entrypoints/openai/test_lora_adapters.py
-5
@@ -23,11 +23,6 @@ BADREQUEST_CASES = [
23
{"r": 1024},
24
"is greater than max_lora_rank",
25
),
26
- (
27
- "test_bias",
28
- {"bias": "all"},
29
- "Adapter bias cannot be used without bias_enabled",
30
- ),
26
("test_dora", {"use_dora": True}, "does not yet support DoRA"),
27
(
28
"test_modules_to_save",
tests/lora/test_peft_helper.py
-5
@@ -16,11 +16,6 @@ ERROR_CASES = [
16
{"r": 1024},
17
"is greater than max_lora_rank",
18
),
19
- (
20
- "test_bias",
21
- {"bias": "all"},
22
- "Adapter bias cannot be used without bias_enabled",
23
- ),
19
("test_dora", {"use_dora": True}, "does not yet support DoRA"),
20
(
21
"test_modules_to_save",
tests/lora/test_utils.py
+2
-13
@@ -21,7 +21,6 @@ class LoRANameParserTestConfig(NamedTuple):
21
name: str
22
module_name: str
23
is_lora_a: bool
24
- is_bias: bool
24
weights_mapper: Optional[WeightsMapper] = None
25
26
@@ -37,44 +36,37 @@ def test_parse_fine_tuned_lora_name_valid():
36
"base_model.model.model.embed_tokens.lora_embedding_A",
37
"model.embed_tokens",
38
True,
40
- False,
39
),
40
LoRANameParserTestConfig(
41
"base_model.model.model.embed_tokens.lora_embedding_B",
42
"model.embed_tokens",
43
False,
46
- False,
44
),
45
LoRANameParserTestConfig(
46
"base_model.model.model.layers.9.mlp.down_proj.lora_A.weight",
47
"model.layers.9.mlp.down_proj",
48
True,
52
- False,
49
),
50
LoRANameParserTestConfig(
51
"base_model.model.model.layers.9.mlp.down_proj.lora_B.weight",
52
"model.layers.9.mlp.down_proj",
53
False,
58
- False,
54
),
55
LoRANameParserTestConfig(
56
"language_model.layers.9.mlp.down_proj.lora_A.weight",
57
"language_model.layers.9.mlp.down_proj",
58
True,
64
- False,
59
),
60
LoRANameParserTestConfig(
61
"language_model.layers.9.mlp.down_proj.lora_B.weight",
62
"language_model.layers.9.mlp.down_proj",
63
False,
70
- False,
64
),
65
# Test with WeightsMapper
66
LoRANameParserTestConfig(
67
"base_model.model.model.layers.9.mlp.down_proj.lora_A.weight",
68
"language_model.model.layers.9.mlp.down_proj",
69
True,
77
- False,
70
weights_mapper=WeightsMapper(
71
orig_to_new_prefix={"model.": "language_model.model."}
72
),
@@ -83,7 +75,6 @@ def test_parse_fine_tuned_lora_name_valid():
75
"base_model.model.model.layers.9.mlp.down_proj.lora_B.weight",
76
"language_model.model.layers.9.mlp.down_proj",
77
False,
86
- False,
78
weights_mapper=WeightsMapper(
79
orig_to_new_prefix={"model.": "language_model.model."}
80
),
@@ -92,7 +83,6 @@ def test_parse_fine_tuned_lora_name_valid():
83
"model.layers.9.mlp.down_proj.lora_A.weight",
84
"language_model.model.layers.9.mlp.down_proj",
85
True,
95
- False,
86
weights_mapper=WeightsMapper(
87
orig_to_new_prefix={"model.": "language_model.model."}
88
),
@@ -101,14 +91,13 @@ def test_parse_fine_tuned_lora_name_valid():
91
"model.layers.9.mlp.down_proj.lora_B.weight",
92
"language_model.model.layers.9.mlp.down_proj",
93
False,
104
- False,
94
weights_mapper=WeightsMapper(
95
orig_to_new_prefix={"model.": "language_model.model."}
96
),
97
),
98
]
110
- for name, module_name, is_lora_a, is_bias, weights_mapper in fixture:
111
- assert (module_name, is_lora_a, is_bias) == parse_fine_tuned_lora_name(
99
+ for name, module_name, is_lora_a, weights_mapper in fixture:
100
+ assert (module_name, is_lora_a) == parse_fine_tuned_lora_name(
101
name, weights_mapper
102
)
103
vllm/config/lora.py
+1
-7
@@ -70,12 +70,6 @@ class LoRAConfig:
70
per prompt. When run in offline mode, the lora IDs for n modalities
71
will be automatically assigned to 1-n with the names of the modalities
72
in alphabetic order."""
73
- bias_enabled: bool = Field(
74
- default=False,
75
- deprecated="`bias_enabled` is deprecated and will be removed in v0.12.0.",
76
- )
77
- """[DEPRECATED] Enable bias for LoRA adapters. This option will be
78
- removed in v0.12.0."""
73
74
def compute_hash(self) -> str:
75
"""
@@ -96,7 +90,7 @@ class LoRAConfig:
90
factors.append(self.lora_dtype)
91
factors.append(self.lora_extra_vocab_size)
92
factors.append(self.lora_vocab_padding_size)
99
- factors.append(self.bias_enabled)
93
+
94
hash_str = hashlib.md5(str(factors).encode(), usedforsecurity=False).hexdigest()
95
return hash_str
96
vllm/engine/arg_utils.py
-3
@@ -439,7 +439,6 @@ class EngineArgs:
439
video_pruning_rate: float = MultiModalConfig.video_pruning_rate
440
# LoRA fields
441
enable_lora: bool = False
442
- enable_lora_bias: bool = LoRAConfig.bias_enabled
442
max_loras: int = LoRAConfig.max_loras
443
max_lora_rank: int = LoRAConfig.max_lora_rank
444
default_mm_loras: Optional[dict[str, str]] = LoRAConfig.default_mm_loras
@@ -916,7 +915,6 @@ class EngineArgs:
915
action=argparse.BooleanOptionalAction,
916
help="If True, enable handling of LoRA adapters.",
917
)
919
- lora_group.add_argument("--enable-lora-bias", **lora_kwargs["bias_enabled"])
918
lora_group.add_argument("--max-loras", **lora_kwargs["max_loras"])
919
lora_group.add_argument("--max-lora-rank", **lora_kwargs["max_lora_rank"])
920
lora_group.add_argument(
@@ -1515,7 +1513,6 @@ class EngineArgs:
1513
1514
lora_config = (
1515
LoRAConfig(
1518
- bias_enabled=self.enable_lora_bias,
1516
max_lora_rank=self.max_lora_rank,
1517
max_loras=self.max_loras,
1518
default_mm_loras=self.default_mm_loras,
vllm/lora/layers/base.py
-1
@@ -45,7 +45,6 @@ class BaseLayerWithLoRA(nn.Module):
45
lora_a: torch.Tensor,
46
lora_b: torch.Tensor,
47
embeddings_tensor: Optional[torch.Tensor],
48
- bias: Optional[torch.Tensor] = None,
48
):
49
"""Overwrites lora tensors at index."""
50
...
vllm/lora/layers/base_linear.py
+2
-38
@@ -1,7 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
-from typing import Optional, cast
4
+from typing import Optional
5
6
import torch
7
from transformers import PretrainedConfig
@@ -29,7 +29,6 @@ class BaseLinearLayerWithLoRA(BaseLayerWithLoRA):
29
self.tp_size = self.base_layer.tp_size
30
self.tp_rank = self.base_layer.tp_rank
31
self.device = _get_lora_device(self.base_layer)
32
- self.lora_bias_stacked: Optional[tuple[torch.Tensor, ...]] = None
32
self.output_slices: tuple[int, ...]
33
self.output_size: int
34
self.n_slices: int
@@ -86,30 +85,12 @@ class BaseLinearLayerWithLoRA(BaseLayerWithLoRA):
85
)
86
for _ in range(self.n_slices)
87
)
89
- if lora_config.bias_enabled:
90
- lora_bias_out_size = lora_b_out_size
91
- self.lora_bias_stacked = tuple(
92
- torch.zeros(
93
- max_loras,
94
- 1,
95
- lora_bias_out_size,
96
- dtype=lora_config.lora_dtype,
97
- device=self.device,
98
- )
99
- for _ in range(self.n_slices)
100
- )
88
self.output_slices = (self.lora_b_stacked[0].shape[2],)
89
90
def reset_lora(self, index: int):
91
for s_index in range(self.n_slices):
92
self.lora_a_stacked[s_index][index] = 0
93
self.lora_b_stacked[s_index][index] = 0
107
- if self.lora_config.bias_enabled:
108
- # Make mypy happy
109
- self.lora_bias_stacked = cast(
110
- tuple[torch.Tensor, ...], self.lora_bias_stacked
111
- )
112
- self.lora_bias_stacked[s_index][index] = 0
94
95
def set_lora(
96
self,
@@ -117,7 +98,6 @@ class BaseLinearLayerWithLoRA(BaseLayerWithLoRA):
98
lora_a: torch.Tensor,
99
lora_b: torch.Tensor,
100
embeddings_tensor: Optional[torch.Tensor],
120
- lora_bias: Optional[torch.Tensor] = None,
101
):
102
# Except for QKVParallelLinearWithLoRA and
103
# MergedColumnParallelLinearWithLoRA, all other linear LoRA layers
@@ -131,8 +111,6 @@ class BaseLinearLayerWithLoRA(BaseLayerWithLoRA):
111
if self.tp_size > 1:
112
lora_a = self.slice_lora_a(lora_a)
113
lora_b = self.slice_lora_b(lora_b)
134
- if lora_bias is not None:
135
- lora_bias = self.slice_bias(lora_bias)
114
115
self.lora_a_stacked[0][index, 0, : lora_a.shape[0], : lora_a.shape[1]].copy_(
116
lora_a, non_blocking=True
@@ -140,14 +118,6 @@ class BaseLinearLayerWithLoRA(BaseLayerWithLoRA):
118
self.lora_b_stacked[0][index, 0, : lora_b.shape[0], : lora_b.shape[1]].copy_(
119
lora_b, non_blocking=True
120
)
143
- if lora_bias is not None:
144
- self.lora_bias_stacked = cast(
145
- tuple[torch.Tensor, ...], self.lora_bias_stacked
146
- )
147
- assert len(self.lora_bias_stacked)
148
- self.lora_bias_stacked[0][index, 0, : lora_bias.shape[0]].copy_(
149
- lora_bias, non_blocking=True
150
- )
121
122
def apply(
123
self, x: torch.Tensor, bias: Optional[torch.Tensor] = None
@@ -162,13 +132,7 @@ class BaseLinearLayerWithLoRA(BaseLayerWithLoRA):
132
x = x.flatten(0, 1)
133
134
lora_output: Optional[torch.Tensor] = self.punica_wrapper.add_lora_linear(
165
- output,
166
- x,
167
- self.lora_a_stacked,
168
- self.lora_b_stacked,
169
- self.lora_bias_stacked,
170
- 1.0,
171
- self.output_slices,
135
+ output, x, self.lora_a_stacked, self.lora_b_stacked, 1.0, self.output_slices
136
)
137
if not current_platform.can_update_inplace():
138
output = lora_output
vllm/lora/layers/column_parallel_linear.py
+1
-66
@@ -1,7 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
-from typing import Optional, Union, cast
4
+from typing import Optional, Union
5
6
import torch
7
import torch.nn as nn
@@ -32,8 +32,6 @@ def _mcp_apply(x, bias, layer: "ColumnParallelLinearWithLoRA"):
32
== len(layer.lora_b_stacked)
33
== len(layer.output_slices)
34
)
35
- if layer.lora_bias_stacked is not None:
36
- assert layer.n_slices == len(layer.lora_bias_stacked)
35
36
output = layer.base_layer.quant_method.apply(layer.base_layer, x, bias)
37
@@ -61,7 +59,6 @@ def _mcp_apply(x, bias, layer: "ColumnParallelLinearWithLoRA"):
59
output,
60
buffers,
61
layer.lora_b_stacked,
64
- layer.lora_bias_stacked,
62
layer.output_slices,
63
offset_start=0,
64
add_input=True,
@@ -122,16 +119,6 @@ class ColumnParallelLinearWithLoRA(BaseLinearLayerWithLoRA):
119
lora_b = lora_b[start_idx:end_idx, :]
120
return lora_b
121
125
- def slice_bias(self, bias: torch.Tensor) -> torch.Tensor:
126
- # TODO: Fix the slicing logic of bias.
127
- if bias is None:
128
- return bias
129
- shard_size = self.output_size
130
- start_idx = self.tp_rank * shard_size
131
- end_idx = (self.tp_rank + 1) * shard_size
132
- bias = bias[start_idx:end_idx]
133
- return bias
134
-
122
def forward(
123
self, input_: torch.Tensor
124
) -> Union[torch.Tensor, tuple[torch.Tensor, Optional[torch.Tensor]]]:
@@ -238,17 +225,6 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
225
)
226
for output_size in self.output_slices
227
)
241
- if lora_config.bias_enabled:
242
- self.lora_bias_stacked = tuple(
243
- torch.zeros(
244
- max_loras,
245
- 1,
246
- output_size,
247
- dtype=lora_config.lora_dtype,
248
- device=self.device,
249
- )
250
- for output_size in self.output_slices
251
- )
228
229
def slice_lora_a(
230
self, lora_a: list[Union[torch.Tensor, None]]
@@ -268,31 +244,18 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
244
]
245
return sliced_lora_b
246
271
- def slice_bias(
272
- self, bias: list[Union[torch.Tensor, None]]
273
- ) -> list[Union[torch.Tensor, None]]:
274
- for i, (shard_id, shard_size) in enumerate(
275
- zip(self.output_ids, self.output_slices)
276
- ):
277
- if (bias_i := bias[i]) is not None:
278
- bias[i] = bias_i[shard_size * shard_id : shard_size * (shard_id + 1)]
279
- return bias
280
-
247
def set_lora(
248
self,
249
index: int,
250
lora_a: torch.Tensor,
251
lora_b: torch.Tensor,
252
embeddings_tensor: Optional[torch.Tensor],
287
- lora_bias: Optional[torch.Tensor] = None,
253
):
254
self.reset_lora(index)
255
256
if self.tp_size > 1:
257
lora_a = self.slice_lora_a(lora_a)
258
lora_b = self.slice_lora_b(lora_b)
294
- if lora_bias is not None:
295
- lora_bias = self.slice_bias(lora_bias)
259
260
for i in range(self.n_slices):
261
if (lora_a_i := lora_a[i]) is not None:
@@ -304,16 +267,6 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
267
index, 0, : lora_b_i.shape[0], : lora_b_i.shape[1]
268
].copy_(lora_b_i, non_blocking=True)
269
307
- if lora_bias is not None:
308
- self.lora_bias_stacked = cast(
309
- tuple[torch.Tensor, ...], self.lora_bias_stacked
310
- )
311
- for i in range(self.n_slices):
312
- if (lora_bias_i := lora_bias[i]) is not None:
313
- self.lora_bias_stacked[i][index, 0, : lora_bias_i.shape[0]].copy_(
314
- lora_bias_i, non_blocking=True
315
- )
316
-
270
@classmethod
271
@_not_fully_sharded_can_replace
272
def can_replace_layer(
@@ -380,24 +333,6 @@ class QKVParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
333
lora_b = torch.cat([lora_b_q, lora_b_k, lora_b_v], dim=0)
334
return lora_b
335
383
- def slice_bias(self, bias: torch.Tensor) -> torch.Tensor:
384
- bias_q = bias[
385
- self.q_proj_shard_size * self.q_shard_id : self.q_proj_shard_size
386
- * (self.q_shard_id + 1)
387
- ]
388
- k_offset = self.q_proj_total_size
389
- bias_k = bias[
390
- k_offset + self.kv_proj_shard_size * self.kv_shard_id : k_offset
391
- + self.kv_proj_shard_size * (self.kv_shard_id + 1)
392
- ]
393
- v_offset = k_offset + self.kv_proj_total_size
394
- bias_v = bias[
395
- v_offset + self.kv_proj_shard_size * self.kv_shard_id : v_offset
396
- + self.kv_proj_shard_size * (self.kv_shard_id + 1)
397
- ]
398
- bias = torch.cat([bias_q, bias_k, bias_v], dim=1)
399
- return bias
400
-
336
@classmethod
337
@_not_fully_sharded_can_replace
338
def can_replace_layer(
vllm/lora/layers/logits_processor.py
-1
@@ -143,7 +143,6 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
143
lora_a: torch.Tensor,
144
lora_b: torch.Tensor,
145
embeddings_tensor: Optional[torch.Tensor],
146
- bias: Optional[torch.Tensor] = None,
146
):
147
self.reset_lora(index)
148
self.lora_a_stacked[index, 0, : lora_a.shape[0], : lora_a.shape[1]].copy_(
vllm/lora/layers/row_parallel_linear.py
+1
-15
@@ -1,7 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
-from typing import Optional, Union, cast
4
+from typing import Optional, Union
5
6
import torch
7
import torch.nn as nn
@@ -39,9 +39,6 @@ class RowParallelLinearWithLoRA(BaseLinearLayerWithLoRA):
39
def slice_lora_b(self, lora_b: torch.Tensor) -> torch.Tensor:
40
return lora_b
41
42
- def slice_bias(self, bias: torch.Tensor) -> torch.Tensor:
43
- return bias
44
-
42
def forward(
43
self, input_: torch.Tensor
44
) -> Union[torch.Tensor, tuple[torch.Tensor, Optional[torch.Tensor]]]:
@@ -123,16 +120,6 @@ class RowParallelLinearWithShardedLoRA(RowParallelLinearWithLoRA):
120
lora_b = lora_b[start_idx:end_idx, :]
121
return lora_b
122
126
- def slice_bias(self, bias: torch.Tensor) -> torch.Tensor:
127
- if bias is None:
128
- return bias
129
- self.lora_bias_stacked = cast(tuple[torch.Tensor, ...], self.lora_bias_stacked)
130
- shard_size = self.lora_bias_stacked[0].shape[2]
131
- start_idx = self.tp_rank * shard_size
132
- end_idx = (self.tp_rank + 1) * shard_size
133
- bias = bias[start_idx:end_idx]
134
- return bias
135
-
123
def apply(
124
self, x: torch.Tensor, bias: Optional[torch.Tensor] = None
125
) -> torch.Tensor:
@@ -167,7 +154,6 @@ class RowParallelLinearWithShardedLoRA(RowParallelLinearWithLoRA):
154
output,
155
buffer,
156
self.lora_b_stacked,
170
- self.lora_bias_stacked,
157
self.output_slices,
158
offset_start=offset_start,
159
add_input=True,
vllm/lora/layers/vocal_parallel_embedding.py
-1
@@ -91,7 +91,6 @@ class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA):
91
lora_a: torch.Tensor,
92
lora_b: torch.Tensor,
93
embeddings_tensor: Optional[torch.Tensor],
94
- bias: Optional[torch.Tensor] = None,
94
):
95
self.reset_lora(index)
96
# NOTE self.lora_a_stacked is row-major, and lora_a is col-major,
vllm/lora/lora_weights.py
+1
-14
@@ -21,7 +21,6 @@ class LoRALayerWeights:
21
lora_alpha: int,
22
lora_a: torch.Tensor,
23
lora_b: torch.Tensor,
24
- bias: Optional[torch.Tensor] = None,
24
embeddings_tensor: Optional[torch.Tensor] = None,
25
scaling: Optional[float] = None,
26
) -> None:
@@ -30,7 +29,6 @@ class LoRALayerWeights:
29
self.lora_alpha = lora_alpha
30
self.lora_a = lora_a
31
self.lora_b = lora_b
33
- self.bias = bias
32
self.embeddings_tensor = embeddings_tensor
33
34
if scaling is None:
@@ -71,13 +69,13 @@ class LoRALayerWeights:
69
peft_helper: PEFTHelper,
70
embeddings_tensor: Optional[torch.Tensor] = None,
71
) -> "LoRALayerWeights":
72
+ # lora_a and lora_b are set to None for config-based construction
73
return cls(
74
module_name,
75
peft_helper.r,
76
peft_helper.lora_alpha,
77
None,
78
None,
80
- None,
79
embeddings_tensor,
80
peft_helper.vllm_lora_scaling_factor,
81
)
@@ -92,7 +90,6 @@ class LoRALayerWeights:
90
dtype: torch.dtype,
91
device: torch.types.Device,
92
embeddings_tensor_dim: Optional[int] = None,
95
- bias_enabled: Optional[bool] = False,
93
) -> "LoRALayerWeights":
94
pin_memory = str(device) == "cpu" and is_pin_memory_available()
95
lora_a = torch.zeros(
@@ -101,12 +98,6 @@ class LoRALayerWeights:
98
lora_b = torch.zeros(
99
[output_dim, rank], dtype=dtype, device=device, pin_memory=pin_memory
100
)
104
- if bias_enabled:
105
- bias = torch.zeros(
106
- [output_dim], dtype=dtype, device=device, pin_memory=pin_memory
107
- )
108
- else:
109
- bias = None
101
102
embeddings_tensor = (
103
torch.rand(
@@ -125,7 +116,6 @@ class LoRALayerWeights:
116
lora_alpha=1,
117
lora_a=lora_a,
118
lora_b=lora_b,
128
- bias=bias,
119
embeddings_tensor=embeddings_tensor,
120
)
121
@@ -140,7 +130,6 @@ class PackedLoRALayerWeights(LoRALayerWeights):
130
lora_alphas: list[Optional[int]],
131
lora_a: list[Optional[torch.Tensor]],
132
lora_b: list[Optional[torch.Tensor]],
143
- bias: Optional[list[Optional[torch.Tensor]]] = None,
133
scaling: Optional[list[float]] = None,
134
) -> None:
135
super().__init__(
@@ -149,7 +138,6 @@ class PackedLoRALayerWeights(LoRALayerWeights):
138
lora_alpha=0,
139
lora_a=lora_a,
140
lora_b=lora_b,
152
- bias=bias,
141
scaling=scaling, # type: ignore
142
embeddings_tensor=None,
143
)
@@ -181,7 +169,6 @@ class PackedLoRALayerWeights(LoRALayerWeights):
169
[lora.lora_alpha if lora is not None else None for lora in loras],
170
[lora.lora_a if lora is not None else None for lora in loras],
171
[lora.lora_b if lora is not None else None for lora in loras],
184
- [lora.bias if lora is not None else None for lora in loras],
172
scaling=[
173
1 if lora is not None else None # type: ignore
174
for lora in loras
vllm/lora/models.py
+3
-28
@@ -3,7 +3,6 @@
3
4
import math
5
import os
6
-from collections.abc import Sequence
6
from typing import Callable, Optional, TypeVar, Union
7
8
import regex as re
@@ -140,7 +139,7 @@ class LoRAModel:
139
pin_memory = str(device) == "cpu" and is_pin_memory_available()
140
loras: dict[str, LoRALayerWeights] = {}
141
for tensor_name, tensor in tensors.items():
143
- module_name, is_lora_a, is_bias = parse_fine_tuned_lora_name(
142
+ module_name, is_lora_a = parse_fine_tuned_lora_name(
143
tensor_name, weights_mapper
144
)
145
if module_name not in loras:
@@ -160,13 +159,7 @@ class LoRAModel:
159
module_name, peft_helper, lora_embeddings_tensor
160
)
161
163
- if is_bias:
164
- loras[module_name].bias = tensor.to(device=device, dtype=dtype)
165
- bias = tensor.to(device=device, dtype=dtype)
166
- if pin_memory:
167
- bias = bias.pin_memory()
168
- loras[module_name].bias = bias
169
- elif is_lora_a:
162
+ if is_lora_a:
163
loras[module_name].lora_a = tensor.to(device=device, dtype=dtype)
164
if pin_memory:
165
loras[module_name].lora_a = loras[module_name].lora_a.pin_memory()
@@ -234,9 +227,7 @@ class LoRAModel:
227
228
def check_unexpected_modules(modules: dict):
229
for lora_module in modules.keys(): # noqa
237
- module_name, _, _ = parse_fine_tuned_lora_name(
238
- lora_module, weights_mapper
239
- )
230
+ module_name, _ = parse_fine_tuned_lora_name(lora_module, weights_mapper)
231
part_name = module_name.split(".")[-1]
232
if part_name not in expected_lora_modules:
233
unexpected_modules.append(module_name)
@@ -439,23 +430,11 @@ class LoRAModelManager:
430
module_lora = self._get_lora_layer_weights(lora_model, module_name)
431
if module_lora:
432
module_lora.optimize()
442
- # Bias is not explicitly enabled with the flag enable_lora_bias.
443
- bias = module_lora.bias
444
- if (
445
- torch.is_tensor(bias)
446
- or (isinstance(bias, Sequence) and any(b is not None for b in bias))
447
- ) and not self.lora_config.bias_enabled:
448
- module_lora.bias = None
449
- raise ValueError(
450
- f"Adapter bias cannot be used for {module_name}"
451
- " without --enable-lora-bias."
452
- )
433
module.set_lora(
434
index,
435
module_lora.lora_a,
436
module_lora.lora_b,
437
module_lora.embeddings_tensor,
458
- module_lora.bias,
438
)
439
else:
440
module.reset_lora(index)
@@ -581,7 +560,6 @@ class LoRAModelManager:
560
"""Create zero-initialized LoRAModel for warmup."""
561
model = LoRAModel(lora_id, rank, {})
562
for module_name, module in self.model.named_modules():
584
- bias_enabled = self.lora_config.bias_enabled
563
if (
564
not self._match_target_modules(module_name)
565
or not isinstance(module, BaseLayerWithLoRA)
@@ -616,7 +594,6 @@ class LoRAModelManager:
594
module.lora_a_stacked[0].dtype,
595
"cpu",
596
embeddings_tensor_dim=embeddings_tensor_dim,
619
- bias_enabled=bias_enabled,
597
)
598
else:
599
lora = LoRALayerWeights.create_dummy_lora_weights(
@@ -626,7 +603,6 @@ class LoRAModelManager:
603
rank,
604
module.lora_a_stacked[0].dtype,
605
"cpu",
629
- bias_enabled=bias_enabled,
606
)
607
else:
608
parts = module_name.split(".")
@@ -640,7 +616,6 @@ class LoRAModelManager:
616
rank,
617
module.lora_a_stacked[i].dtype,
618
"cpu",
643
- bias_enabled=bias_enabled,
619
)
620
subloras.append(lora)
621
lora = PackedLoRALayerWeights.pack(subloras)
vllm/lora/peft_helper.py
+3
-3
@@ -29,7 +29,7 @@ class PEFTHelper:
29
lora_alpha: int
30
target_modules: Union[list[str], str]
31
32
- bias: Literal["none", "all", "lora_only"] = field(default="none")
32
+ bias: Literal["none"] = field(default="none")
33
modules_to_save: Optional[list[str]] = field(default=None)
34
# True to use Rank-Stabilized LoRA (rsLoRA, see: https://arxiv.org/abs/2312.03732)
35
use_rslora: bool = field(default=False)
@@ -122,7 +122,7 @@ class PEFTHelper:
122
f"LoRA rank {self.r} is greater than max_lora_rank"
123
f" {lora_config.max_lora_rank}."
124
)
125
- if self.bias != "none" and not lora_config.bias_enabled:
126
- error_msg.append("Adapter bias cannot be used without bias_enabled.")
125
+ if self.bias != "none":
126
+ error_msg.append("Adapter bias is not supported.")
127
if error_msg:
128
raise ValueError(f"{' '.join(error_msg)}")
vllm/lora/punica_wrapper/punica_base.py
+4
-44
@@ -60,14 +60,13 @@ class PunicaWrapperABC(ABC):
60
y: torch.Tensor,
61
x: Union[tuple[torch.Tensor, ...], torch.Tensor],
62
lora_b_stacked: tuple[torch.Tensor, ...],
63
- lora_bias_stacked: Optional[tuple[torch.Tensor, ...]],
63
output_slices: tuple[int, ...],
64
offset_start: int = 0,
65
add_inputs=True,
66
**kwargs,
67
) -> Optional[torch.Tensor]:
68
"""
70
- Performs GEMM and bias addition for multiple slices of lora_b.
69
+ Performs GEMM for multiple slices of lora_b.
70
"""
71
raise NotImplementedError
72
@@ -93,7 +92,6 @@ class PunicaWrapperABC(ABC):
92
x: torch.Tensor,
93
lora_a_stacked: tuple[torch.Tensor, ...],
94
lora_b_stacked: tuple[torch.Tensor, ...],
96
- lora_bias_stacked: Optional[tuple[torch.Tensor, ...]],
95
scale: float,
96
output_slices: tuple[int, ...],
97
*,
@@ -222,38 +220,6 @@ class PunicaWrapperBase(PunicaWrapperABC):
220
self.token_nums = token_nums
221
self.no_lora = no_lora
222
225
- def _apply_bias(
226
- self,
227
- indices: torch.Tensor,
228
- output: torch.Tensor,
229
- output_slices: tuple[int, ...],
230
- lora_bias_stacked: tuple[Optional[torch.Tensor], ...],
231
- ):
232
- """Applies bias to output
233
-
234
- Input shapes:
235
- lora_bias_stacked: 3 element tuple of (num_loras, output_dim)
236
- indices: (batch_size)
237
- output: (batch_size, q_slice_size + 2*kv_slice_size)
238
- output_slices: n-1 element tuple of (slice_size...),
239
- where n is number of slices
240
- """
241
- org_output = output
242
- output = output.view(-1, output.shape[-1])
243
- indices = indices.view(-1)
244
-
245
- offset_left = 0
246
- for slice_idx, slice in enumerate(output_slices):
247
- bias = lora_bias_stacked[slice_idx]
248
- if bias is not None:
249
- bias = bias.view(-1, bias.shape[-1])
250
- bias = bias[indices]
251
- bias[indices == -1] = 0
252
- output[:, offset_left : offset_left + slice] += bias
253
- offset_left += slice
254
-
255
- return output.view_as(org_output)
256
-
223
@property
224
def prefill_metadata(
225
self,
@@ -365,29 +331,25 @@ class PunicaWrapperBase(PunicaWrapperABC):
331
y: torch.Tensor,
332
x: Union[tuple[torch.Tensor, ...], torch.Tensor],
333
lora_b_stacked: tuple[torch.Tensor, ...],
368
- lora_bias_stacked: Optional[tuple[torch.Tensor, ...]],
334
output_slices: tuple[int, ...],
335
offset_start: int = 0,
336
add_inputs=True,
337
**kwargs,
338
) -> Optional[torch.Tensor]:
339
"""
375
- Performs GEMM and bias addition for multiple slices of lora_b.
340
+ Performs GEMM for multiple slices of lora_b.
341
342
Semantics:
343
offset = offset_start
344
for i in range(len(lora_b_stacked)):
345
slice = output_slices[i]
381
- y[:, offset:offset+slice] += x[i] @ lora_b_stacked[i] +
382
- lora_bias_stacked[i]
346
+ y[:, offset:offset+slice] += x[i] @ lora_b_stacked[i]
347
offset += slice
348
349
Args:
350
y (torch.Tensor): Output tensor.
351
x (Union[tuple[torch.Tensor, ...], torch.Tensor]): Input tensors
352
lora_b_stacked (tuple[torch.Tensor, ...]): lora_b's weight
389
- lora_bias_stacked (Optional[tuple[torch.Tensor, ...]]):
390
- bias's weight
353
output_slices (tuple[int, ...]): Every slice's size
354
offset_start (int): The starting position of y, defaults to 0
355
add_inputs (bool): Defaults to True.
@@ -427,7 +389,6 @@ class PunicaWrapperBase(PunicaWrapperABC):
389
x: torch.Tensor,
390
lora_a_stacked: tuple[torch.Tensor, ...],
391
lora_b_stacked: tuple[torch.Tensor, ...],
430
- lora_bias_stacked: Optional[tuple[torch.Tensor, ...]],
392
scale: float,
393
output_slices: tuple[int, ...],
394
*,
@@ -444,14 +405,13 @@ class PunicaWrapperBase(PunicaWrapperABC):
405
@ lora_a_stacked[indices[i], layer_idx, :, :]
406
@ lora_b_stacked[indices[i], layer_idx, :, :]
407
* scale
447
- ).squeeze(0)+lora_bias_stacked[i]
408
+ ).squeeze(0)
409
410
Args:
411
y (torch.Tensor): Output tensor. Will be changed in-place.
412
x (torch.Tensor): Input tensor
413
lora_a_stacked (tuple[torch.Tensor, ...]): lora_a's weight.
414
lora_b_stacked (tuple[torch.Tensor, ...]): lora_b's weight.
454
- lora_bias_stacked (Optional[tuple[torch.Tensor, ...]]): lora's bias.
415
scale (float): Scaling factor.
416
output_slices (tuple[int, ...]): Every slice's size.
417
buffer (Optional[tuple[torch.Tensor, ...]]): Defaults to None.
vllm/lora/punica_wrapper/punica_cpu.py
+4
-19
@@ -199,38 +199,30 @@ class PunicaWrapperCPU(PunicaWrapperBase):
199
y: torch.Tensor,
200
x: Union[tuple[torch.Tensor, ...], torch.Tensor],
201
lora_b_stacked: tuple[torch.Tensor, ...],
202
- lora_bias_stacked: Optional[tuple[torch.Tensor, ...]],
202
output_slices: tuple[int, ...],
203
offset_start: int = 0,
204
add_inputs=True,
205
**kwargs,
206
) -> None:
207
"""
209
- Performs GEMM and bias addition for multiple slices of lora_b.
208
+ Performs GEMM for multiple slices of lora_b.
209
210
Semantics:
211
for i in range(len(lora_b_stacked)):
212
slice = output_slices[i]
214
- y[:, offset:offset+slice] += x[i] @ lora_b_stacked[i] +
215
- lora_bias_stacked[i]
213
+ y[:, offset:offset+slice] += x[i] @ lora_b_stacked[i]
214
offset += slice
215
216
Args:
217
y (torch.Tensor): Output tensor.
218
x (Union[tuple[torch.Tensor, ...], torch.Tensor]): Input tensors
219
lora_b_stacked (tuple[torch.Tensor, ...]): lora_b's weight
222
- lora_bias_stacked (Optional[tuple[torch.Tensor, ...]]):
223
- bias's weight
220
output_slices (tuple[int, ...]): Every slice's size
221
add_inputs (bool): Defaults to True.
222
"""
223
y_org = y
224
y = y.view(-1, y.shape[-1])
225
offset_left = offset_start
230
- if lora_bias_stacked is not None:
231
- self._apply_bias(
232
- self.token_lora_indices, y, output_slices, lora_bias_stacked
233
- )
226
for slice_idx in range(len(lora_b_stacked)):
227
self._apply_expand(
228
y,
@@ -276,7 +268,6 @@ class PunicaWrapperCPU(PunicaWrapperBase):
268
x: torch.Tensor,
269
lora_a_stacked: tuple[torch.Tensor, ...],
270
lora_b_stacked: tuple[torch.Tensor, ...],
279
- lora_bias_stacked: Optional[tuple[torch.Tensor, ...]],
271
scale: float,
272
output_slices: tuple[int, ...],
273
*,
@@ -293,25 +284,19 @@ class PunicaWrapperCPU(PunicaWrapperBase):
284
@ lora_a_stacked[indices[i], layer_idx, :, :]
285
@ lora_b_stacked[indices[i], layer_idx, :, :]
286
* scale
296
- ).squeeze(0)+lora_bias_stacked[i]
287
+ ).squeeze(0)
288
289
Args:
290
y (torch.Tensor): Output tensor. Will be changed in-place.
291
x (torch.Tensor): Input tensor
292
lora_a_stacked (tuple[torch.Tensor, ...]): lora_a's weight.
293
lora_b_stacked (tuple[torch.Tensor, ...]): lora_b's weight.
303
- lora_bias_stacked (Optional[tuple[torch.Tensor, ...]]): lora's bias.
294
scale (float): Scaling factor.
295
output_slices (tuple[int, ...]): Every slice's size.
296
buffer (Optional[tuple[torch.Tensor, ...]]): Defaults to None.
297
"""
298
299
assert len(lora_a_stacked) == len(lora_b_stacked) == len(output_slices)
310
- if lora_bias_stacked is not None:
311
- assert len(lora_bias_stacked) == len(output_slices)
312
- y = self._apply_bias(
313
- self.token_lora_indices, y, output_slices, lora_bias_stacked
314
- )
300
301
if buffer is None:
302
r = lora_b_stacked[0].size(-1)
@@ -323,7 +308,7 @@ class PunicaWrapperCPU(PunicaWrapperBase):
308
)
309
self.add_shrink(buffer, x, lora_a_stacked, scale, **kwargs)
310
self.add_expand(
326
- y, buffer, lora_b_stacked, None, output_slices, add_inputs=True, **kwargs
311
+ y, buffer, lora_b_stacked, output_slices, add_inputs=True, **kwargs
312
)
313
314
def add_lora_logits(
vllm/lora/punica_wrapper/punica_gpu.py
+3
-20
@@ -101,36 +101,29 @@ class PunicaWrapperGPU(PunicaWrapperBase):
101
y: torch.Tensor,
102
x: torch.Tensor,
103
lora_b_stacked: tuple[torch.Tensor, ...],
104
- lora_bias_stacked: Optional[tuple[torch.Tensor, ...]],
104
output_slices: tuple[int, ...],
105
offset_start: int = 0,
106
add_inputs=True,
107
**kwargs,
108
) -> None:
109
"""
111
- Performs GEMM and bias addition for multiple slices of lora_b.
110
+ Performs GEMM for multiple slices of lora_b.
111
112
Semantics:
113
for i in range(len(lora_b_stacked)):
114
slice = output_slices[i]
116
- y[:, offset:offset+slice] += x[i] @ lora_b_stacked[i] +
117
- lora_bias_stacked[i]
115
+ y[:, offset:offset+slice] += x[i] @ lora_b_stacked[i]
116
offset += slice
117
118
Args:
119
y (torch.Tensor): Output tensor.
120
x (torch.Tensor): Input tensors
121
lora_b_stacked (tuple[torch.Tensor, ...]): lora_b's weight
124
- lora_bias_stacked (Optional[tuple[torch.Tensor, ...]]):
125
- bias's weight
122
output_slices (tuple[int, ...]): Every slice's size
123
add_inputs (bool): Defaults to True.
124
"""
125
y_org = y
126
y = y.view(-1, y.shape[-1])
131
- if lora_bias_stacked is not None:
132
- token_lora_indices = torch.narrow(self._token_lora_indices, 0, 0, y.size(0))
133
- self._apply_bias(token_lora_indices, y, output_slices, lora_bias_stacked)
127
128
assert x.ndim == 3
129
assert x.size(0) == len(output_slices)
@@ -183,7 +176,6 @@ class PunicaWrapperGPU(PunicaWrapperBase):
176
x: torch.Tensor,
177
lora_a_stacked: tuple[torch.Tensor, ...],
178
lora_b_stacked: tuple[torch.Tensor, ...],
186
- lora_bias_stacked: Optional[tuple[torch.Tensor, ...]],
179
scale: float,
180
output_slices: tuple[int, ...],
181
*,
@@ -200,26 +192,18 @@ class PunicaWrapperGPU(PunicaWrapperBase):
192
@ lora_a_stacked[indices[i], layer_idx, :, :]
193
@ lora_b_stacked[indices[i], layer_idx, :, :]
194
* scale
203
- ).squeeze(0)+lora_bias_stacked[i]
204
-
195
+ ).squeeze(0)
196
Args:
197
y (torch.Tensor): Output tensor. Will be changed in-place.
198
x (torch.Tensor): Input tensor
199
lora_a_stacked (tuple[torch.Tensor, ...]): lora_a's weight.
200
lora_b_stacked (tuple[torch.Tensor, ...]): lora_b's weight.
210
- lora_bias_stacked (Optional[tuple[torch.Tensor, ...]]): lora's bias.
201
scale (float): Scaling factor.
202
output_slices (tuple[int, ...]): Every slice's size.
203
buffer (Optional[torch.Tensor]): Defaults to None.
204
"""
205
206
assert len(lora_a_stacked) == len(lora_b_stacked) == len(output_slices)
217
- if lora_bias_stacked is not None:
218
- assert len(lora_bias_stacked) == len(output_slices)
219
- token_lora_indices = torch.narrow(self._token_lora_indices, 0, 0, y.size(0))
220
- y = self._apply_bias(
221
- token_lora_indices, y, output_slices, lora_bias_stacked
222
- )
207
208
if buffer is None:
209
r = lora_b_stacked[0].size(-1)
@@ -241,7 +225,6 @@ class PunicaWrapperGPU(PunicaWrapperBase):
225
y,
226
buffer, # type: ignore
227
lora_b_stacked,
244
- None,
228
output_slices,
229
add_inputs=True,
230
**kwargs,
vllm/lora/punica_wrapper/punica_tpu.py
+4
-56
@@ -139,28 +139,24 @@ class PunicaWrapperTPU(PunicaWrapperBase):
139
y: torch.Tensor,
140
x: Union[tuple[torch.Tensor, ...], torch.Tensor],
141
lora_b_stacked: tuple[torch.Tensor, ...],
142
- lora_bias_stacked: Optional[tuple[torch.Tensor, ...]],
142
output_slices: tuple[int, ...],
143
offset_start: int = 0,
144
add_inputs=True,
145
**kwargs,
146
) -> torch.Tensor:
147
"""
149
- Performs GEMM and bias addition for multiple slices of lora_b.
148
+ Performs GEMM for multiple slices of lora_b.
149
150
Semantics:
151
for i in range(len(lora_b_stacked)):
152
slice = output_slices[i]
154
- y[:, offset:offset+slice] += x[i] @ lora_b_stacked[i] +
155
- lora_bias_stacked[i]
153
+ y[:, offset:offset+slice] += x[i] @ lora_b_stacked[i]
154
offset += slice
155
156
Args:
157
y (torch.Tensor): Output tensor.
158
x (Union[tuple[torch.Tensor, ...], torch.Tensor]): Input tensors
159
lora_b_stacked (tuple[torch.Tensor, ...]): lora_b's weight
162
- lora_bias_stacked (Optional[tuple[torch.Tensor, ...]]):
163
- bias's weight
160
output_slices (tuple[int, ...]): Every slice's size
161
add_inputs (bool): Defaults to True.
162
"""
@@ -168,10 +164,6 @@ class PunicaWrapperTPU(PunicaWrapperBase):
164
y = y.view(-1, y.shape[-1])
165
offset_left = 0
166
171
- if lora_bias_stacked is not None:
172
- y = self._apply_bias(
173
- self._get_token_lora_indices(y), y, output_slices, lora_bias_stacked
174
- )
167
for slice_idx in range(len(lora_b_stacked)):
168
y = self.expand_slice(
169
y,
@@ -214,7 +206,6 @@ class PunicaWrapperTPU(PunicaWrapperBase):
206
x: torch.Tensor,
207
lora_a_stacked: tuple[torch.Tensor, ...],
208
lora_b_stacked: tuple[torch.Tensor, ...],
217
- lora_bias_stacked: Optional[tuple[torch.Tensor, ...]],
209
scale: float,
210
output_slices: tuple[int, ...],
211
*,
@@ -231,25 +222,19 @@ class PunicaWrapperTPU(PunicaWrapperBase):
222
@ lora_a_stacked[indices[i], layer_idx, :, :]
223
@ lora_b_stacked[indices[i], layer_idx, :, :]
224
* scale
234
- ).squeeze(0)+lora_bias_stacked[i]
225
+ ).squeeze(0)
226
227
Args:
228
y (torch.Tensor): Output tensor. Will not be changed in-place.
229
x (torch.Tensor): Input tensor (T, E)
230
lora_a_stacked (tuple[torch.Tensor, ...]): lora_a's weight.
231
lora_b_stacked (tuple[torch.Tensor, ...]): lora_b's weight.
241
- lora_bias_stacked (Optional[tuple[torch.Tensor, ...]]): lora's bias.
232
scale (float): Scaling factor.
233
output_slices (tuple[int, ...]): Every slice's size.
234
buffer (Optional[tuple[torch.Tensor, ...]]): Defaults to None.
235
"""
236
237
assert len(lora_a_stacked) == len(lora_b_stacked) == len(output_slices)
248
- if lora_bias_stacked is not None:
249
- assert len(lora_bias_stacked) == len(output_slices)
250
- y = self._apply_bias(
251
- self._get_token_lora_indices(y), y, output_slices, lora_bias_stacked
252
- )
238
239
if buffer is None:
240
r = lora_b_stacked[0].size(-1)
@@ -261,7 +246,7 @@ class PunicaWrapperTPU(PunicaWrapperBase):
246
)
247
buffer = self.add_shrink(buffer, x, lora_a_stacked, scale, **kwargs)
248
return self.add_expand(
264
- y, buffer, lora_b_stacked, None, output_slices, add_inputs=True, **kwargs
249
+ y, buffer, lora_b_stacked, output_slices, add_inputs=True, **kwargs
250
)
251
252
def add_lora_logits(
@@ -299,43 +284,6 @@ class PunicaWrapperTPU(PunicaWrapperBase):
284
y = bgmv_expand(buffer, lora_b_stacked, y, sampler_indices, add_inputs=True)
285
return y.view_as(y_org)
286
302
- def _apply_bias(
303
- self,
304
- indices: torch.Tensor,
305
- output: torch.Tensor,
306
- output_slices: tuple[int, ...],
307
- lora_bias_stacked: tuple[Optional[torch.Tensor], ...],
308
- ):
309
- """Applies bias to output
310
-
311
- Input shapes:
312
- lora_bias_stacked: 3 element tuple of (num_loras, output_dim)
313
- indices: (batch_size)
314
- output: (batch_size, q_slice_size + 2*kv_slice_size)
315
- output_slices: n-1 element tuple of (slice_size...),
316
- where n is number of slices
317
- """
318
- org_output = output
319
- output = output.view(-1, output.shape[-1])
320
- indices = indices.view(-1)
321
-
322
- offset_left = 0
323
- for slice_idx, slice in enumerate(output_slices):
324
- bias = lora_bias_stacked[slice_idx]
325
- if bias is not None:
326
- bias = bias.view(-1, bias.shape[-1])
327
- bias = bias[indices]
328
- bias = torch.where(indices[:, None] == -1, 0, bias)
329
-
330
- bias = F.pad(
331
- bias, (offset_left, output.shape[1] - (offset_left + slice), 0, 0)
332
- )
333
-
334
- output += bias
335
- offset_left += slice
336
-
337
- return output.view_as(org_output)
338
-
287
# This performs the same tensor ops as the base method, except it does them
288
# on the CPU then transfers the results to the TPU
289
def _update_base_metadata(
vllm/lora/punica_wrapper/punica_xpu.py
+3
-19
@@ -108,36 +108,29 @@ class PunicaWrapperXPU(PunicaWrapperBase):
108
y: torch.Tensor,
109
x: torch.Tensor,
110
lora_b_stacked: tuple[torch.Tensor, ...],
111
- lora_bias_stacked: Optional[tuple[torch.Tensor, ...]],
111
output_slices: tuple[int, ...],
112
offset_start: int = 0,
113
add_inputs=True,
114
**kwargs,
115
) -> None:
116
"""
118
- Performs GEMM and bias addition for multiple slices of lora_b.
117
+ Performs GEMM for multiple slices of lora_b.
118
119
Semantics:
120
for i in range(len(lora_b_stacked)):
121
slice = output_slices[i]
123
- y[:, offset:offset+slice] += x[i] @ lora_b_stacked[i] +
124
- lora_bias_stacked[i]
122
+ y[:, offset:offset+slice] += x[i] @ lora_b_stacked[i]
123
offset += slice
124
125
Args:
126
y (torch.Tensor): Output tensor.
127
x (torch.Tensor): Input tensors
128
lora_b_stacked (tuple[torch.Tensor, ...]): lora_b's weight
131
- lora_bias_stacked (Optional[tuple[torch.Tensor, ...]]):
132
- bias's weight
129
output_slices (tuple[int, ...]): Every slice's size
130
add_inputs (bool): Defaults to True.
131
"""
132
y_org = y
133
y = y.view(-1, y.shape[-1])
138
- if lora_bias_stacked is not None:
139
- token_lora_indices = self._get_token_lora_indices(y)
140
- self._apply_bias(token_lora_indices, y, output_slices, lora_bias_stacked)
134
135
assert x.ndim == 3
136
assert x.size(0) == len(output_slices)
@@ -184,7 +177,6 @@ class PunicaWrapperXPU(PunicaWrapperBase):
177
x: torch.Tensor,
178
lora_a_stacked: tuple[torch.Tensor, ...],
179
lora_b_stacked: tuple[torch.Tensor, ...],
187
- lora_bias_stacked: Optional[tuple[torch.Tensor, ...]],
180
scale: float,
181
output_slices: tuple[int, ...],
182
*,
@@ -201,26 +193,19 @@ class PunicaWrapperXPU(PunicaWrapperBase):
193
@ lora_a_stacked[indices[i], layer_idx, :, :]
194
@ lora_b_stacked[indices[i], layer_idx, :, :]
195
* scale
204
- ).squeeze(0)+lora_bias_stacked[i]
196
+ ).squeeze(0)
197
198
Args:
199
y (torch.Tensor): Output tensor. Will be changed in-place.
200
x (torch.Tensor): Input tensor
201
lora_a_stacked (tuple[torch.Tensor, ...]): lora_a's weight.
202
lora_b_stacked (tuple[torch.Tensor, ...]): lora_b's weight.
211
- lora_bias_stacked (Optional[tuple[torch.Tensor, ...]]): lora's bias.
203
scale (float): Scaling factor.
204
output_slices (tuple[int, ...]): Every slice's size.
205
buffer (Optional[torch.Tensor]): Defaults to None.
206
"""
207
208
assert len(lora_a_stacked) == len(lora_b_stacked) == len(output_slices)
218
- if lora_bias_stacked is not None:
219
- assert len(lora_bias_stacked) == len(output_slices)
220
- token_lora_indices = self._get_token_lora_indices(y)
221
- y = self._apply_bias(
222
- token_lora_indices, y, output_slices, lora_bias_stacked
223
- )
209
210
if buffer is None:
211
r = lora_b_stacked[0].size(-1)
@@ -242,7 +227,6 @@ class PunicaWrapperXPU(PunicaWrapperBase):
227
y,
228
buffer, # type: ignore
229
lora_b_stacked,
245
- None,
230
output_slices,
231
add_inputs=True,
232
**kwargs,
vllm/lora/utils.py
+3
-8
@@ -112,7 +112,7 @@ def replace_submodule(
112
113
def parse_fine_tuned_lora_name(
114
name: str, weights_mapper: Optional["WeightsMapper"] = None
115
-) -> tuple[str, bool, bool]:
115
+) -> tuple[str, bool]:
116
"""Parse the name of lora weights.
117
118
args:
@@ -124,7 +124,6 @@ def parse_fine_tuned_lora_name(
124
tuple(module_name, is_lora_a):
125
module_name: the name of the module, e.g. model.dense1,
126
is_lora_a whether the tensor is lora_a or lora_b.
127
- is_bias whether the tensor is lora bias.
127
"""
128
129
# LoRA weight qualified name usually starts with `base_model.model.`,
@@ -146,15 +145,11 @@ def parse_fine_tuned_lora_name(
145
parts = name.split(".")
146
if parts[-1] == "weight" and (parts[-2] == "lora_A" or parts[-2] == "lora_B"):
147
new_name = ".".join(parts[start_index:-2])
149
- return new_name, parts[-2] == "lora_A", False
148
+ return new_name, parts[-2] == "lora_A"
149
150
if parts[-1] == "lora_embedding_A" or parts[-1] == "lora_embedding_B":
151
new_name = ".".join(parts[start_index:-1])
153
- return new_name, parts[-1] == "lora_embedding_A", False
154
-
155
- if parts[-1] == "bias":
156
- new_name = ".".join(parts[start_index:-2])
157
- return new_name, False, True
152
+ return new_name, parts[-1] == "lora_embedding_A"
153
154
raise ValueError(f"{name} is unsupported LoRA weight")
155