@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 11dd6ebb89
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Misc] Avoid loading incorrect LoRA config (#3777)

Jee Li committed Apr 10, 2024 at 10:47 UTC 11dd6ebb8950a66c371f6fa5d2489eb01fc4f6d5
3 files changed +66 -2
tests/lora/test_lora_checkpoints.py new
+40
@@ -0,0 +1,40 @@
1 +import pytest
2 +
3 +from vllm.lora.models import LoRAModel
4 +from vllm.model_executor.models.baichuan import BaiChuanBaseForCausalLM
5 +
6 +
7 +@pytest.mark.parametrize("lora_name", ["baichuan7B", "chatglm3-6b"])
8 +def test_load_checkpoints(lora_name, chatglm3_lora_files, baichuan_lora_files):
9 + supported_lora_modules = BaiChuanBaseForCausalLM.supported_lora_modules
10 + packed_modules_mapping = BaiChuanBaseForCausalLM.packed_modules_mapping
11 + embedding_modules = BaiChuanBaseForCausalLM.embedding_modules
12 + embed_padding_modules = BaiChuanBaseForCausalLM.embedding_padding_modules
13 + expected_lora_modules = []
14 + for module in supported_lora_modules:
15 + if module in packed_modules_mapping:
16 + expected_lora_modules.extend(packed_modules_mapping[module])
17 + else:
18 + expected_lora_modules.append(module)
19 + if lora_name == "baichuan7B":
20 + # For the baichuan7B model, load it's LoRA,
21 + # and the test should pass.
22 + LoRAModel.from_local_checkpoint(
23 + baichuan_lora_files,
24 + expected_lora_modules,
25 + lora_model_id=1,
26 + device="cpu",
27 + embedding_modules=embedding_modules,
28 + embedding_padding_modules=embed_padding_modules)
29 + else:
30 + # For the baichuan7B model, load chatglm3-6b's LoRA,
31 + # and the test should raise the following error.
32 + expected_error = "Please verify that the loaded LoRA module is correct" # noqa: E501
33 + with pytest.raises(ValueError, match=expected_error):
34 + LoRAModel.from_local_checkpoint(
35 + chatglm3_lora_files,
36 + expected_lora_modules,
37 + lora_model_id=1,
38 + device="cpu",
39 + embedding_modules=embedding_modules,
40 + embedding_padding_modules=embed_padding_modules)
vllm/lora/models.py
+15 -2
@@ -191,6 +191,7 @@ class LoRAModel:
191 def from_local_checkpoint(
192 cls,
193 lora_dir: str,
194 + expected_lora_modules: List[str],
195 lora_model_id: Optional[int] = None,
196 device: str = "cuda",
197 dtype: Optional[torch.dtype] = None,
@@ -206,6 +207,20 @@ class LoRAModel:
207 lora_dir, "new_embeddings.safetensors")
208 new_embeddings_bin_file_path = os.path.join(lora_dir,
209 "new_embeddings.bin")
210 + with open(lora_config_path) as f:
211 + config = json.load(f)
212 + target_modules = config["target_modules"]
213 + unexpected_modules = []
214 + for module in target_modules:
215 + if module not in expected_lora_modules:
216 + unexpected_modules.append(module)
217 + # loaded lora's target modules must be a subset of expected_lora_modules
218 + if unexpected_modules:
219 + raise ValueError(
220 + f"While loading {lora_dir}, expected"
221 + f" target modules in {expected_lora_modules}"
222 + f" but received {unexpected_modules}."
223 + f" Please verify that the loaded LoRA module is correct")
224 if os.path.isfile(lora_tensor_path):
225 tensors = safetensors.torch.load_file(lora_tensor_path)
226 elif os.path.isfile(lora_bin_file_path):
@@ -220,8 +235,6 @@ class LoRAModel:
235 elif os.path.isfile(new_embeddings_bin_file_path):
236 embeddings = torch.load(new_embeddings_bin_file_path)
237
223 - with open(lora_config_path) as f:
224 - config = json.load(f)
238 rank = config["r"]
239 lora_alpha = config["lora_alpha"]
240 return cls.from_lora_tensors(
vllm/lora/worker_manager.py
+11
@@ -136,8 +136,19 @@ class WorkerLoRAManager(AbstractWorkerLoRAManager):
136
137 def _load_lora(self, lora_request: LoRARequest) -> LoRAModel:
138 try:
139 + model = self._lora_manager.model
140 + supported_lora_modules = model.supported_lora_modules
141 + packed_modules_mapping = model.packed_modules_mapping
142 + expected_lora_modules = []
143 + for module in supported_lora_modules:
144 + if module in packed_modules_mapping:
145 + expected_lora_modules.extend(
146 + packed_modules_mapping[module])
147 + else:
148 + expected_lora_modules.append(module)
149 lora = self._lora_model_cls.from_local_checkpoint(
150 lora_request.lora_local_path,
151 + expected_lora_modules,
152 lora_model_id=lora_request.lora_int_id,
153 device="cpu",
154 dtype=self.lora_config.lora_dtype,