A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 13:30 UTC
[Bugfix] Fix Lora Name Parsing (#17196)
Signed-off-by: Alex-Brooks <Alex.Brooks@ibm.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
Alex Brooks committed
Apr 27, 2025 at 06:33 UTC
756848e79e1bd557d36b20e70cbd48ddba48ea51
2 files changed
+21
-4
tests/lora/test_utils.py
+12
@@ -39,6 +39,18 @@ def test_parse_fine_tuned_lora_name_valid():
39
False,
40
False,
41
),
42
+ (
43
+ "language_model.layers.9.mlp.down_proj.lora_A.weight",
44
+ "language_model.layers.9.mlp.down_proj",
45
+ True,
46
+ False,
47
+ ),
48
+ (
49
+ "language_model.layers.9.mlp.down_proj.lora_B.weight",
50
+ "language_model.layers.9.mlp.down_proj",
51
+ False,
52
+ False,
53
+ ),
54
}
55
for name, module_name, is_lora_a, is_bias in fixture:
56
assert (module_name, is_lora_a,
vllm/lora/utils.py
+9
-4
@@ -114,7 +114,7 @@ def parse_fine_tuned_lora_name(
114
is_bias whether the tensor is lora bias.
115
"""
116
117
- # LoRA weight qualified name always starts with `base_model.model.`,
117
+ # LoRA weight qualified name usually starts with `base_model.model.`,
118
# so we remove the prefix `base_model.model.` to make the following
119
# mapping correctly.
120
if "base_model.model." in name:
@@ -123,18 +123,23 @@ def parse_fine_tuned_lora_name(
123
# recover the prefix `base_model.model.`
124
name = "base_model.model." + name
125
126
+ # In some situations, we may not start with `base_model.model.`.
127
+ # If we don't (e.g., ibm-granite/granite-speech-3.3-8b),
128
+ # we should keep the prefix intact.
129
+ start_index = 2 if "base_model.model." in name else 0
130
+
131
parts = name.split(".")
132
if parts[-1] == "weight" and (parts[-2] == "lora_A"
133
or parts[-2] == "lora_B"):
129
- new_name = ".".join(parts[2:-2])
134
+ new_name = ".".join(parts[start_index:-2])
135
return new_name, parts[-2] == "lora_A", False
136
137
if parts[-1] == "lora_embedding_A" or parts[-1] == "lora_embedding_B":
133
- new_name = ".".join(parts[2:-1])
138
+ new_name = ".".join(parts[start_index:-1])
139
return new_name, parts[-1] == "lora_embedding_A", False
140
141
if parts[-1] == "bias":
137
- new_name = ".".join(parts[2:-2])
142
+ new_name = ".".join(parts[start_index:-2])
143
return new_name, False, True
144
145
raise ValueError(f"{name} is unsupported LoRA weight")