A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[Lora] Use safetensor keys instead of adapter_config.json to find unexpected modules. (#5909)
Co-authored-by: sang <sangcho@anyscale.com>
SangBin Cho committed
Jul 1, 2024 at 02:11 UTC
f5e73c9f1bfcffdac4ec97f038443c053ee6fed8
4 files changed
+52
-20
.buildkite/test-pipeline.yaml
+1
@@ -226,3 +226,4 @@ steps:
226
- pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.0.5/flashinfer-0.0.5+cu121torch2.3-cp310-cp310-linux_x86_64.whl
227
- VLLM_ATTENTION_BACKEND=FLASHINFER TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
228
- VLLM_ATTENTION_BACKEND=FLASHINFER TEST_DIST_MODEL=meta-llama/Meta-Llama-3-8B DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
229
+ - pytest -v -s -x lora/test_mixtral.py
tests/lora/conftest.py
+3
-1
@@ -165,7 +165,9 @@ def sql_lora_files():
165
166
@pytest.fixture(scope="session")
167
def mixtral_lora_files():
168
- return snapshot_download(repo_id="terrysun/mixtral-lora-adapter")
168
+ # Note: this module has incorrect adapter_config.json to test
169
+ # https://github.com/vllm-project/vllm/pull/5909/files.
170
+ return snapshot_download(repo_id="SangBinCho/mixtral-lora")
171
172
173
@pytest.fixture(scope="session")
tests/lora/test_mixtral.py
+2
-2
@@ -40,14 +40,14 @@ def test_mixtral_lora(mixtral_lora_files, tp_size):
40
enable_lora=True,
41
max_num_seqs=16,
42
max_loras=4,
43
+ distributed_executor_backend="ray",
44
tensor_parallel_size=tp_size)
45
46
expected_lora_output = [
47
"give_opinion(name[SpellForce 3], release_year[2017], developer[Grimlore Games], rating[poor])", # noqa: E501
47
- "give_opinion(name[SpellForce 3], release_year[2017], developer[Grimlore Games], rating[poor])", # noqa: E501
48
+ "give_opinion(name[SpellForce 3], developer[Grimlore Games], release_year[2017], rating[poor])", # noqa: E501
49
"inform(name[BioShock], release_year[2007], rating[good], genres[action-adventure, role-playing, shooter], platforms[PlayStation, Xbox, PC], available_on_steam[yes], has_linux_release[no], has_mac_release[yes])", # noqa: E501
50
]
50
-
51
assert do_sample(llm, mixtral_lora_files,
52
lora_id=1) == expected_lora_output
53
assert do_sample(llm, mixtral_lora_files,
vllm/lora/models.py
+46
-17
@@ -303,25 +303,54 @@ class LoRAModel:
303
"new_embeddings.bin")
304
with open(lora_config_path) as f:
305
config = json.load(f)
306
- target_modules = config["target_modules"]
307
- unexpected_modules = []
308
- for module in target_modules:
309
- # Compatible with more modules, such as:layers.11.self_attn.k_proj
310
- part_name = module.split(".")[-1]
311
- if part_name not in expected_lora_modules:
312
- unexpected_modules.append(module)
313
- # loaded lora's target modules must be a subset of expected_lora_modules
314
-
315
- if unexpected_modules:
316
- print(unexpected_modules, "modules")
317
- raise ValueError(
318
- f"While loading {lora_dir}, expected"
319
- f" target modules in {expected_lora_modules}"
320
- f" but received {unexpected_modules}."
321
- f" Please verify that the loaded LoRA module is correct")
306
if os.path.isfile(lora_tensor_path):
323
- tensors = safetensors.torch.load_file(lora_tensor_path)
307
+ tensors: Dict[str, torch.Tensor] = {}
308
+ # Find unexpected modules.
309
+ # Use safetensor key as a source of truth to find expected modules.
310
+ # in peft if you have target_modules A, B, C and C does not exist
311
+ # in the model it won’t error and model will be trained with A, B
312
+ # loraified. C won’t exist in the safetensor but it will exist in
313
+ # the target_modules of the adapter_config.json.
314
+ unexpected_modules = []
315
+ with safetensors.safe_open(lora_tensor_path,
316
+ framework="pt") as f: # type: ignore
317
+ for lora_module in f.keys(): # noqa
318
+ module_name, _ = parse_fine_tuned_lora_name(lora_module)
319
+ part_name = module_name.split(".")[-1]
320
+ if part_name not in expected_lora_modules:
321
+ unexpected_modules.append(module_name)
322
+ if unexpected_modules:
323
+ raise ValueError(
324
+ f"While loading {lora_dir}, expected"
325
+ f" target modules in {expected_lora_modules}"
326
+ f" but received {unexpected_modules}."
327
+ f" Please verify that the loaded LoRA module is correct"
328
+ )
329
+ # Load tensors if there are only expected modules.
330
+ for module in f.keys(): # noqa
331
+ tensors[module] = f.get_tensor(module)
332
elif os.path.isfile(lora_bin_file_path):
333
+ # When a bin file is provided, we rely on config to find unexpected
334
+ # modules.
335
+ unexpected_modules = []
336
+ target_modules = config["target_modules"]
337
+ for module in target_modules:
338
+ # Compatible with more modules,
339
+ # such as:layers.11.self_attn.k_proj
340
+ part_name = module.split(".")[-1]
341
+ if part_name not in expected_lora_modules:
342
+ unexpected_modules.append(module)
343
+ # loaded lora's target modules must be a subset of
344
+ # expected_lora_modules. It is not reliable. See
345
+ # https://github.com/vllm-project/vllm/pull/5909. But there's no
346
+ # other better mechanism.
347
+ if unexpected_modules:
348
+ print(unexpected_modules, "modules")
349
+ raise ValueError(
350
+ f"While loading {lora_dir}, expected"
351
+ f" target modules in {expected_lora_modules}"
352
+ f" but received {unexpected_modules}."
353
+ f" Please verify that the loaded LoRA module is correct")
354
tensors = torch.load(lora_bin_file_path)
355
else:
356
raise ValueError(f"{lora_dir} doesn't contain tensors")