A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 11:00 UTC
[Bugfix] Fix missing lora name mapping for lora without prefix (#17793)
Signed-off-by: Isotr0py <2037008807@qq.com>
Isotr0py committed
May 8, 2025 at 00:17 UTC
f98e3075880bde73f5bdc20ab688e224352f6880
2 files changed
+61
-14
tests/lora/test_utils.py
+57
-12
@@ -1,6 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
from collections import OrderedDict
4
+from typing import NamedTuple, Optional
5
from unittest.mock import patch
6
7
import pytest
@@ -9,52 +10,96 @@ from torch import nn
10
11
from vllm.lora.utils import (get_adapter_absolute_path,
12
parse_fine_tuned_lora_name, replace_submodule)
13
+from vllm.model_executor.models.utils import WeightsMapper
14
+
15
+
16
+class LoRANameParserTestConfig(NamedTuple):
17
+ name: str
18
+ module_name: str
19
+ is_lora_a: bool
20
+ is_bias: bool
21
+ weights_mapper: Optional[WeightsMapper] = None
22
23
24
def test_parse_fine_tuned_lora_name_valid():
15
- fixture = {
16
- ("base_model.model.lm_head.lora_A.weight", "lm_head", True, False),
17
- ("base_model.model.lm_head.lora_B.weight", "lm_head", False, False),
18
- (
25
+ fixture = [
26
+ LoRANameParserTestConfig("base_model.model.lm_head.lora_A.weight",
27
+ "lm_head", True, False),
28
+ LoRANameParserTestConfig("base_model.model.lm_head.lora_B.weight",
29
+ "lm_head", False, False),
30
+ LoRANameParserTestConfig(
31
"base_model.model.model.embed_tokens.lora_embedding_A",
32
"model.embed_tokens",
33
True,
34
False,
35
),
24
- (
36
+ LoRANameParserTestConfig(
37
"base_model.model.model.embed_tokens.lora_embedding_B",
38
"model.embed_tokens",
39
False,
40
False,
41
),
30
- (
42
+ LoRANameParserTestConfig(
43
"base_model.model.model.layers.9.mlp.down_proj.lora_A.weight",
44
"model.layers.9.mlp.down_proj",
45
True,
46
False,
47
),
36
- (
48
+ LoRANameParserTestConfig(
49
"base_model.model.model.layers.9.mlp.down_proj.lora_B.weight",
50
"model.layers.9.mlp.down_proj",
51
False,
52
False,
53
),
42
- (
54
+ LoRANameParserTestConfig(
55
"language_model.layers.9.mlp.down_proj.lora_A.weight",
56
"language_model.layers.9.mlp.down_proj",
57
True,
58
False,
59
),
48
- (
60
+ LoRANameParserTestConfig(
61
"language_model.layers.9.mlp.down_proj.lora_B.weight",
62
"language_model.layers.9.mlp.down_proj",
63
False,
64
False,
65
),
54
- }
55
- for name, module_name, is_lora_a, is_bias in fixture:
66
+ # Test with WeightsMapper
67
+ LoRANameParserTestConfig(
68
+ "base_model.model.model.layers.9.mlp.down_proj.lora_A.weight",
69
+ "language_model.model.layers.9.mlp.down_proj",
70
+ True,
71
+ False,
72
+ weights_mapper=WeightsMapper(
73
+ orig_to_new_prefix={"model.": "language_model.model."}),
74
+ ),
75
+ LoRANameParserTestConfig(
76
+ "base_model.model.model.layers.9.mlp.down_proj.lora_B.weight",
77
+ "language_model.model.layers.9.mlp.down_proj",
78
+ False,
79
+ False,
80
+ weights_mapper=WeightsMapper(
81
+ orig_to_new_prefix={"model.": "language_model.model."}),
82
+ ),
83
+ LoRANameParserTestConfig(
84
+ "model.layers.9.mlp.down_proj.lora_A.weight",
85
+ "language_model.model.layers.9.mlp.down_proj",
86
+ True,
87
+ False,
88
+ weights_mapper=WeightsMapper(
89
+ orig_to_new_prefix={"model.": "language_model.model."}),
90
+ ),
91
+ LoRANameParserTestConfig(
92
+ "model.layers.9.mlp.down_proj.lora_B.weight",
93
+ "language_model.model.layers.9.mlp.down_proj",
94
+ False,
95
+ False,
96
+ weights_mapper=WeightsMapper(
97
+ orig_to_new_prefix={"model.": "language_model.model."}),
98
+ ),
99
+ ]
100
+ for name, module_name, is_lora_a, is_bias, weights_mapper in fixture:
101
assert (module_name, is_lora_a,
57
- is_bias) == parse_fine_tuned_lora_name(name)
102
+ is_bias) == parse_fine_tuned_lora_name(name, weights_mapper)
103
104
105
def test_parse_fine_tuned_lora_name_invalid():
vllm/lora/utils.py
+4
-2
@@ -117,16 +117,18 @@ def parse_fine_tuned_lora_name(
117
# LoRA weight qualified name usually starts with `base_model.model.`,
118
# so we remove the prefix `base_model.model.` to make the following
119
# mapping correctly.
120
- if "base_model.model." in name:
120
+ if name.startswith("base_model.model."):
121
name = name.replace("base_model.model.", "")
122
name = weights_mapper._map_name(name) if weights_mapper else name
123
# recover the prefix `base_model.model.`
124
name = "base_model.model." + name
125
+ else:
126
+ name = weights_mapper._map_name(name) if weights_mapper else name
127
128
# In some situations, we may not start with `base_model.model.`.
129
# If we don't (e.g., ibm-granite/granite-speech-3.3-8b),
130
# we should keep the prefix intact.
129
- start_index = 2 if "base_model.model." in name else 0
131
+ start_index = 2 if name.startswith("base_model.model.") else 0
132
133
parts = name.split(".")
134
if parts[-1] == "weight" and (parts[-2] == "lora_A"