@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d17c8477f1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Bugfix] Fix LoRA loading check (#4138)

Co-authored-by: simon-mo <simon.mo@hey.com>

Jee Li committed Apr 19, 2024 at 15:59 UTC d17c8477f1fb337ea9fcf439bcab4d323058c1b4
3 files changed +29 -3
tests/lora/conftest.py
+6
@@ -143,6 +143,12 @@ def baichuan_lora_files():
143 return snapshot_download(repo_id="jeeejeee/baichuan7b-text2sql-spider")
144
145
146 +@pytest.fixture(scope="session")
147 +def baichuan_zero_lora_files():
148 + # all the lora_B weights are initialized to zero.
149 + return snapshot_download(repo_id="jeeejeee/baichuan7b-zero-init")
150 +
151 +
152 @pytest.fixture(scope="session")
153 def tinyllama_lora_files():
154 return snapshot_download(repo_id="jashing/tinyllama-colorist-lora")
tests/lora/test_lora_checkpoints.py
+20 -2
@@ -3,9 +3,16 @@ import pytest
3 from vllm.lora.models import LoRAModel
4 from vllm.model_executor.models.baichuan import BaiChuanBaseForCausalLM
5
6 +lora_lst = ["baichuan7B", "baichuan7B-zero", "chatglm3-6b"]
7
7 -@pytest.mark.parametrize("lora_name", ["baichuan7B", "chatglm3-6b"])
8 -def test_load_checkpoints(lora_name, chatglm3_lora_files, baichuan_lora_files):
8 +
9 +@pytest.mark.parametrize("lora_name", lora_lst)
10 +def test_load_checkpoints(
11 + lora_name,
12 + baichuan_lora_files,
13 + baichuan_zero_lora_files,
14 + chatglm3_lora_files,
15 +):
16 supported_lora_modules = BaiChuanBaseForCausalLM.supported_lora_modules
17 packed_modules_mapping = BaiChuanBaseForCausalLM.packed_modules_mapping
18 embedding_modules = BaiChuanBaseForCausalLM.embedding_modules
@@ -26,6 +33,17 @@ def test_load_checkpoints(lora_name, chatglm3_lora_files, baichuan_lora_files):
33 device="cpu",
34 embedding_modules=embedding_modules,
35 embedding_padding_modules=embed_padding_modules)
36 + elif lora_name == "baichuan7B-zero":
37 + #Test that the target_modules contain prefix
38 + # such as "model.layers.0.self_atten.W_pack", and
39 + # the test should pass.
40 + LoRAModel.from_local_checkpoint(
41 + baichuan_zero_lora_files,
42 + expected_lora_modules,
43 + lora_model_id=1,
44 + device="cpu",
45 + embedding_modules=embedding_modules,
46 + embedding_padding_modules=embed_padding_modules)
47 else:
48 # For the baichuan7B model, load chatglm3-6b's LoRA,
49 # and the test should raise the following error.
vllm/lora/models.py
+3 -1
@@ -212,7 +212,9 @@ class LoRAModel:
212 target_modules = config["target_modules"]
213 unexpected_modules = []
214 for module in target_modules:
215 - if module not in expected_lora_modules:
215 + # Compatible with more modules, such as:layers.11.self_attn.k_proj
216 + part_name = module.split(".")[-1]
217 + if part_name not in expected_lora_modules:
218 unexpected_modules.append(module)
219 # loaded lora's target modules must be a subset of expected_lora_modules
220 if unexpected_modules: