@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 36ea79079b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Misc][LoRA] Support loading LoRA weights for target_modules in reg format (#9275)

Jee Jee Li committed Oct 11, 2024 at 20:31 UTC 36ea79079bc499cd8fb07d3fe82fe069564e5570
4 files changed +59 -5
tests/lora/conftest.py
+5
@@ -199,6 +199,11 @@ def baichuan_zero_lora_files():
199 return snapshot_download(repo_id="jeeejeee/baichuan7b-zero-init")
200
201
202 +@pytest.fixture(scope="session")
203 +def baichuan_regex_lora_files():
204 + return snapshot_download(repo_id="jeeejeee/baichuan-7b-lora-zero-regex")
205 +
206 +
207 @pytest.fixture(scope="session")
208 def minicpmv_lora_files():
209 return snapshot_download(repo_id="jeeejeee/minicpmv25-lora-pokemon")
tests/lora/test_lora_checkpoints.py
+15 -2
@@ -5,7 +5,9 @@ import pytest
5 from vllm.lora.models import LoRAModel
6 from vllm.model_executor.models.baichuan import BaiChuanBaseForCausalLM
7
8 -lora_lst = ["baichuan7B", "baichuan7B-zero", "chatglm3-6b"]
8 +lora_lst = [
9 + "baichuan7B", "baichuan7B-zero", "baichuan7B-zero-regex", "chatglm3-6b"
10 +]
11
12
13 @pytest.mark.parametrize("lora_name", lora_lst)
@@ -13,6 +15,7 @@ def test_load_checkpoints(
15 lora_name,
16 baichuan_lora_files,
17 baichuan_zero_lora_files,
18 + baichuan_regex_lora_files,
19 chatglm3_lora_files,
20 ):
21 supported_lora_modules = BaiChuanBaseForCausalLM.supported_lora_modules
@@ -36,7 +39,7 @@ def test_load_checkpoints(
39 embedding_modules=embedding_modules,
40 embedding_padding_modules=embed_padding_modules)
41 elif lora_name == "baichuan7B-zero":
39 - #Test that the target_modules contain prefix
42 + # Test that the target_modules contain prefix
43 # such as "model.layers.0.self_atten.W_pack", and
44 # the test should pass.
45 LoRAModel.from_local_checkpoint(
@@ -46,6 +49,16 @@ def test_load_checkpoints(
49 device="cpu",
50 embedding_modules=embedding_modules,
51 embedding_padding_modules=embed_padding_modules)
52 + elif lora_name == "baichuan7B-zero-regex":
53 + # Test that the `target_modules` in the form of regular expressions,
54 + # such as `model\\..*(W_pack|o_proj)`, and the test should pass.
55 + LoRAModel.from_local_checkpoint(
56 + baichuan_regex_lora_files,
57 + expected_lora_modules,
58 + lora_model_id=1,
59 + device="cpu",
60 + embedding_modules=embedding_modules,
61 + embedding_padding_modules=embed_padding_modules)
62 else:
63 # For the baichuan7B model, load chatglm3-6b's LoRA,
64 # and the test should raise the following error.
vllm/lora/models.py
+5 -2
@@ -23,6 +23,7 @@ from vllm.lora.layers import (BaseLayerWithLoRA,
23 from vllm.lora.lora import LoRALayerWeights, PackedLoRALayerWeights
24 from vllm.lora.punica import PunicaWrapper
25 from vllm.lora.utils import (from_layer, from_layer_logits_processor,
26 + is_regex_target_modules,
27 parse_fine_tuned_lora_name, replace_submodule)
28 from vllm.model_executor.models import SupportsLoRA, supports_multimodal
29 from vllm.model_executor.models.module_mapping import MultiModelKeys
@@ -233,6 +234,8 @@ class LoRAModel(AdapterModel):
234 # modules.
235 unexpected_modules = []
236 target_modules = config["target_modules"]
237 + if not isinstance(target_modules, list):
238 + target_modules = [target_modules]
239 for module in target_modules:
240 # Compatible with more modules,
241 # such as:layers.11.self_attn.k_proj
@@ -243,8 +246,8 @@ class LoRAModel(AdapterModel):
246 # expected_lora_modules. It is not reliable. See
247 # https://github.com/vllm-project/vllm/pull/5909. But there's no
248 # other better mechanism.
246 - if unexpected_modules:
247 - print(unexpected_modules, "modules")
249 + if unexpected_modules and not is_regex_target_modules(
250 + config["target_modules"], expected_lora_modules):
251 raise ValueError(
252 f"While loading {lora_dir}, expected"
253 f" target modules in {expected_lora_modules}"
vllm/lora/utils.py
+34 -1
@@ -1,5 +1,6 @@
1 import os
2 -from typing import List, Optional, Set, Tuple, Type
2 +import re
3 +from typing import List, Optional, Set, Tuple, Type, Union
4
5 import huggingface_hub
6 from huggingface_hub.utils import (EntryNotFoundError, HfHubHTTPError,
@@ -113,6 +114,38 @@ def parse_fine_tuned_lora_name(name: str) -> Tuple[str, bool]:
114 raise ValueError(f"{name} is unsupported LoRA weight")
115
116
117 +def is_regex_target_modules(load_modules: Union[str, List[str]],
118 + expected_lora_modules: List[str]) -> bool:
119 + """
120 + PEFT supports passing `target_modules` in the form of regular expressions,
121 + such as `model.*(q_proj|k_proj|v_proj)$`. This function is mainly used to
122 + determine whether the suffix in the regular expression is present in the
123 + `expected_lora_modules`.
124 + """
125 +
126 + def is_valid_regex(pattern):
127 + try:
128 + re.compile(pattern)
129 + return True
130 + except re.error:
131 + return False
132 +
133 + def is_subset(sub_list, full_list):
134 + return set(sub_list).issubset(set(full_list))
135 +
136 + # Similar to PEFT's processing logic, regex-related operations are only
137 + # executed when the load_modules is a `str`.
138 + if not isinstance(load_modules, str):
139 + return False
140 +
141 + if is_valid_regex(load_modules):
142 + match = re.search(r"\((.*?)\)\$?$", load_modules)
143 + if match:
144 + suffix = match.group(1).split("|")
145 + return is_subset(suffix, expected_lora_modules)
146 + return False
147 +
148 +
149 def get_adapter_absolute_path(lora_path: str) -> str:
150 """
151 Resolves the given lora_path to an absolute local path.