A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Bugfix] Fix LoRA loading check (#4138)
Co-authored-by: simon-mo <simon.mo@hey.com>
Jee Li committed
Apr 19, 2024 at 15:59 UTC
d17c8477f1fb337ea9fcf439bcab4d323058c1b4
3 files changed
+29
-3
tests/lora/conftest.py
+6
@@ -143,6 +143,12 @@ def baichuan_lora_files():
143
return snapshot_download(repo_id="jeeejeee/baichuan7b-text2sql-spider")
144
145
146
+@pytest.fixture(scope="session")
147
+def baichuan_zero_lora_files():
148
+ # all the lora_B weights are initialized to zero.
149
+ return snapshot_download(repo_id="jeeejeee/baichuan7b-zero-init")
150
+
151
+
152
@pytest.fixture(scope="session")
153
def tinyllama_lora_files():
154
return snapshot_download(repo_id="jashing/tinyllama-colorist-lora")
tests/lora/test_lora_checkpoints.py
+20
-2
@@ -3,9 +3,16 @@ import pytest
3
from vllm.lora.models import LoRAModel
4
from vllm.model_executor.models.baichuan import BaiChuanBaseForCausalLM
5
6
+lora_lst = ["baichuan7B", "baichuan7B-zero", "chatglm3-6b"]
7
7
-@pytest.mark.parametrize("lora_name", ["baichuan7B", "chatglm3-6b"])
8
-def test_load_checkpoints(lora_name, chatglm3_lora_files, baichuan_lora_files):
8
+
9
+@pytest.mark.parametrize("lora_name", lora_lst)
10
+def test_load_checkpoints(
11
+ lora_name,
12
+ baichuan_lora_files,
13
+ baichuan_zero_lora_files,
14
+ chatglm3_lora_files,
15
+):
16
supported_lora_modules = BaiChuanBaseForCausalLM.supported_lora_modules
17
packed_modules_mapping = BaiChuanBaseForCausalLM.packed_modules_mapping
18
embedding_modules = BaiChuanBaseForCausalLM.embedding_modules
@@ -26,6 +33,17 @@ def test_load_checkpoints(lora_name, chatglm3_lora_files, baichuan_lora_files):
33
device="cpu",
34
embedding_modules=embedding_modules,
35
embedding_padding_modules=embed_padding_modules)
36
+ elif lora_name == "baichuan7B-zero":
37
+ #Test that the target_modules contain prefix
38
+ # such as "model.layers.0.self_atten.W_pack", and
39
+ # the test should pass.
40
+ LoRAModel.from_local_checkpoint(
41
+ baichuan_zero_lora_files,
42
+ expected_lora_modules,
43
+ lora_model_id=1,
44
+ device="cpu",
45
+ embedding_modules=embedding_modules,
46
+ embedding_padding_modules=embed_padding_modules)
47
else:
48
# For the baichuan7B model, load chatglm3-6b's LoRA,
49
# and the test should raise the following error.
vllm/lora/models.py
+3
-1
@@ -212,7 +212,9 @@ class LoRAModel:
212
target_modules = config["target_modules"]
213
unexpected_modules = []
214
for module in target_modules:
215
- if module not in expected_lora_modules:
215
+ # Compatible with more modules, such as:layers.11.self_attn.k_proj
216
+ part_name = module.split(".")[-1]
217
+ if part_name not in expected_lora_modules:
218
unexpected_modules.append(module)
219
# loaded lora's target modules must be a subset of expected_lora_modules
220
if unexpected_modules: