A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Misc] Avoid loading incorrect LoRA config (#3777)
Jee Li committed
Apr 10, 2024 at 10:47 UTC
11dd6ebb8950a66c371f6fa5d2489eb01fc4f6d5
3 files changed
+66
-2
tests/lora/test_lora_checkpoints.py
new
+40
@@ -0,0 +1,40 @@
1
+import pytest
2
+
3
+from vllm.lora.models import LoRAModel
4
+from vllm.model_executor.models.baichuan import BaiChuanBaseForCausalLM
5
+
6
+
7
+@pytest.mark.parametrize("lora_name", ["baichuan7B", "chatglm3-6b"])
8
+def test_load_checkpoints(lora_name, chatglm3_lora_files, baichuan_lora_files):
9
+ supported_lora_modules = BaiChuanBaseForCausalLM.supported_lora_modules
10
+ packed_modules_mapping = BaiChuanBaseForCausalLM.packed_modules_mapping
11
+ embedding_modules = BaiChuanBaseForCausalLM.embedding_modules
12
+ embed_padding_modules = BaiChuanBaseForCausalLM.embedding_padding_modules
13
+ expected_lora_modules = []
14
+ for module in supported_lora_modules:
15
+ if module in packed_modules_mapping:
16
+ expected_lora_modules.extend(packed_modules_mapping[module])
17
+ else:
18
+ expected_lora_modules.append(module)
19
+ if lora_name == "baichuan7B":
20
+ # For the baichuan7B model, load it's LoRA,
21
+ # and the test should pass.
22
+ LoRAModel.from_local_checkpoint(
23
+ baichuan_lora_files,
24
+ expected_lora_modules,
25
+ lora_model_id=1,
26
+ device="cpu",
27
+ embedding_modules=embedding_modules,
28
+ embedding_padding_modules=embed_padding_modules)
29
+ else:
30
+ # For the baichuan7B model, load chatglm3-6b's LoRA,
31
+ # and the test should raise the following error.
32
+ expected_error = "Please verify that the loaded LoRA module is correct" # noqa: E501
33
+ with pytest.raises(ValueError, match=expected_error):
34
+ LoRAModel.from_local_checkpoint(
35
+ chatglm3_lora_files,
36
+ expected_lora_modules,
37
+ lora_model_id=1,
38
+ device="cpu",
39
+ embedding_modules=embedding_modules,
40
+ embedding_padding_modules=embed_padding_modules)
vllm/lora/models.py
+15
-2
@@ -191,6 +191,7 @@ class LoRAModel:
191
def from_local_checkpoint(
192
cls,
193
lora_dir: str,
194
+ expected_lora_modules: List[str],
195
lora_model_id: Optional[int] = None,
196
device: str = "cuda",
197
dtype: Optional[torch.dtype] = None,
@@ -206,6 +207,20 @@ class LoRAModel:
207
lora_dir, "new_embeddings.safetensors")
208
new_embeddings_bin_file_path = os.path.join(lora_dir,
209
"new_embeddings.bin")
210
+ with open(lora_config_path) as f:
211
+ config = json.load(f)
212
+ target_modules = config["target_modules"]
213
+ unexpected_modules = []
214
+ for module in target_modules:
215
+ if module not in expected_lora_modules:
216
+ unexpected_modules.append(module)
217
+ # loaded lora's target modules must be a subset of expected_lora_modules
218
+ if unexpected_modules:
219
+ raise ValueError(
220
+ f"While loading {lora_dir}, expected"
221
+ f" target modules in {expected_lora_modules}"
222
+ f" but received {unexpected_modules}."
223
+ f" Please verify that the loaded LoRA module is correct")
224
if os.path.isfile(lora_tensor_path):
225
tensors = safetensors.torch.load_file(lora_tensor_path)
226
elif os.path.isfile(lora_bin_file_path):
@@ -220,8 +235,6 @@ class LoRAModel:
235
elif os.path.isfile(new_embeddings_bin_file_path):
236
embeddings = torch.load(new_embeddings_bin_file_path)
237
223
- with open(lora_config_path) as f:
224
- config = json.load(f)
238
rank = config["r"]
239
lora_alpha = config["lora_alpha"]
240
return cls.from_lora_tensors(
vllm/lora/worker_manager.py
+11
@@ -136,8 +136,19 @@ class WorkerLoRAManager(AbstractWorkerLoRAManager):
136
137
def _load_lora(self, lora_request: LoRARequest) -> LoRAModel:
138
try:
139
+ model = self._lora_manager.model
140
+ supported_lora_modules = model.supported_lora_modules
141
+ packed_modules_mapping = model.packed_modules_mapping
142
+ expected_lora_modules = []
143
+ for module in supported_lora_modules:
144
+ if module in packed_modules_mapping:
145
+ expected_lora_modules.extend(
146
+ packed_modules_mapping[module])
147
+ else:
148
+ expected_lora_modules.append(module)
149
lora = self._lora_model_cls.from_local_checkpoint(
150
lora_request.lora_local_path,
151
+ expected_lora_modules,
152
lora_model_id=lora_request.lora_int_id,
153
device="cpu",
154
dtype=self.lora_config.lora_dtype,