A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Misc][LoRA] Support loading LoRA weights for target_modules in reg format (#9275)
Jee Jee Li committed
Oct 11, 2024 at 20:31 UTC
36ea79079bc499cd8fb07d3fe82fe069564e5570
4 files changed
+59
-5
tests/lora/conftest.py
+5
@@ -199,6 +199,11 @@ def baichuan_zero_lora_files():
199
return snapshot_download(repo_id="jeeejeee/baichuan7b-zero-init")
200
201
202
+@pytest.fixture(scope="session")
203
+def baichuan_regex_lora_files():
204
+ return snapshot_download(repo_id="jeeejeee/baichuan-7b-lora-zero-regex")
205
+
206
+
207
@pytest.fixture(scope="session")
208
def minicpmv_lora_files():
209
return snapshot_download(repo_id="jeeejeee/minicpmv25-lora-pokemon")
tests/lora/test_lora_checkpoints.py
+15
-2
@@ -5,7 +5,9 @@ import pytest
5
from vllm.lora.models import LoRAModel
6
from vllm.model_executor.models.baichuan import BaiChuanBaseForCausalLM
7
8
-lora_lst = ["baichuan7B", "baichuan7B-zero", "chatglm3-6b"]
8
+lora_lst = [
9
+ "baichuan7B", "baichuan7B-zero", "baichuan7B-zero-regex", "chatglm3-6b"
10
+]
11
12
13
@pytest.mark.parametrize("lora_name", lora_lst)
@@ -13,6 +15,7 @@ def test_load_checkpoints(
15
lora_name,
16
baichuan_lora_files,
17
baichuan_zero_lora_files,
18
+ baichuan_regex_lora_files,
19
chatglm3_lora_files,
20
):
21
supported_lora_modules = BaiChuanBaseForCausalLM.supported_lora_modules
@@ -36,7 +39,7 @@ def test_load_checkpoints(
39
embedding_modules=embedding_modules,
40
embedding_padding_modules=embed_padding_modules)
41
elif lora_name == "baichuan7B-zero":
39
- #Test that the target_modules contain prefix
42
+ # Test that the target_modules contain prefix
43
# such as "model.layers.0.self_atten.W_pack", and
44
# the test should pass.
45
LoRAModel.from_local_checkpoint(
@@ -46,6 +49,16 @@ def test_load_checkpoints(
49
device="cpu",
50
embedding_modules=embedding_modules,
51
embedding_padding_modules=embed_padding_modules)
52
+ elif lora_name == "baichuan7B-zero-regex":
53
+ # Test that the `target_modules` in the form of regular expressions,
54
+ # such as `model\\..*(W_pack|o_proj)`, and the test should pass.
55
+ LoRAModel.from_local_checkpoint(
56
+ baichuan_regex_lora_files,
57
+ expected_lora_modules,
58
+ lora_model_id=1,
59
+ device="cpu",
60
+ embedding_modules=embedding_modules,
61
+ embedding_padding_modules=embed_padding_modules)
62
else:
63
# For the baichuan7B model, load chatglm3-6b's LoRA,
64
# and the test should raise the following error.
vllm/lora/models.py
+5
-2
@@ -23,6 +23,7 @@ from vllm.lora.layers import (BaseLayerWithLoRA,
23
from vllm.lora.lora import LoRALayerWeights, PackedLoRALayerWeights
24
from vllm.lora.punica import PunicaWrapper
25
from vllm.lora.utils import (from_layer, from_layer_logits_processor,
26
+ is_regex_target_modules,
27
parse_fine_tuned_lora_name, replace_submodule)
28
from vllm.model_executor.models import SupportsLoRA, supports_multimodal
29
from vllm.model_executor.models.module_mapping import MultiModelKeys
@@ -233,6 +234,8 @@ class LoRAModel(AdapterModel):
234
# modules.
235
unexpected_modules = []
236
target_modules = config["target_modules"]
237
+ if not isinstance(target_modules, list):
238
+ target_modules = [target_modules]
239
for module in target_modules:
240
# Compatible with more modules,
241
# such as:layers.11.self_attn.k_proj
@@ -243,8 +246,8 @@ class LoRAModel(AdapterModel):
246
# expected_lora_modules. It is not reliable. See
247
# https://github.com/vllm-project/vllm/pull/5909. But there's no
248
# other better mechanism.
246
- if unexpected_modules:
247
- print(unexpected_modules, "modules")
249
+ if unexpected_modules and not is_regex_target_modules(
250
+ config["target_modules"], expected_lora_modules):
251
raise ValueError(
252
f"While loading {lora_dir}, expected"
253
f" target modules in {expected_lora_modules}"
vllm/lora/utils.py
+34
-1
@@ -1,5 +1,6 @@
1
import os
2
-from typing import List, Optional, Set, Tuple, Type
2
+import re
3
+from typing import List, Optional, Set, Tuple, Type, Union
4
5
import huggingface_hub
6
from huggingface_hub.utils import (EntryNotFoundError, HfHubHTTPError,
@@ -113,6 +114,38 @@ def parse_fine_tuned_lora_name(name: str) -> Tuple[str, bool]:
114
raise ValueError(f"{name} is unsupported LoRA weight")
115
116
117
+def is_regex_target_modules(load_modules: Union[str, List[str]],
118
+ expected_lora_modules: List[str]) -> bool:
119
+ """
120
+ PEFT supports passing `target_modules` in the form of regular expressions,
121
+ such as `model.*(q_proj|k_proj|v_proj)$`. This function is mainly used to
122
+ determine whether the suffix in the regular expression is present in the
123
+ `expected_lora_modules`.
124
+ """
125
+
126
+ def is_valid_regex(pattern):
127
+ try:
128
+ re.compile(pattern)
129
+ return True
130
+ except re.error:
131
+ return False
132
+
133
+ def is_subset(sub_list, full_list):
134
+ return set(sub_list).issubset(set(full_list))
135
+
136
+ # Similar to PEFT's processing logic, regex-related operations are only
137
+ # executed when the load_modules is a `str`.
138
+ if not isinstance(load_modules, str):
139
+ return False
140
+
141
+ if is_valid_regex(load_modules):
142
+ match = re.search(r"\((.*?)\)\$?$", load_modules)
143
+ if match:
144
+ suffix = match.group(1).split("|")
145
+ return is_subset(suffix, expected_lora_modules)
146
+ return False
147
+
148
+
149
def get_adapter_absolute_path(lora_path: str) -> str:
150
"""
151
Resolves the given lora_path to an absolute local path.