@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f5e73c9f1b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[Lora] Use safetensor keys instead of adapter_config.json to find unexpected modules. (#5909)

Co-authored-by: sang <sangcho@anyscale.com>

SangBin Cho committed Jul 1, 2024 at 02:11 UTC f5e73c9f1bfcffdac4ec97f038443c053ee6fed8
4 files changed +52 -20
.buildkite/test-pipeline.yaml
+1
@@ -226,3 +226,4 @@ steps:
226 - pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.0.5/flashinfer-0.0.5+cu121torch2.3-cp310-cp310-linux_x86_64.whl
227 - VLLM_ATTENTION_BACKEND=FLASHINFER TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
228 - VLLM_ATTENTION_BACKEND=FLASHINFER TEST_DIST_MODEL=meta-llama/Meta-Llama-3-8B DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
229 + - pytest -v -s -x lora/test_mixtral.py
tests/lora/conftest.py
+3 -1
@@ -165,7 +165,9 @@ def sql_lora_files():
165
166 @pytest.fixture(scope="session")
167 def mixtral_lora_files():
168 - return snapshot_download(repo_id="terrysun/mixtral-lora-adapter")
168 + # Note: this module has incorrect adapter_config.json to test
169 + # https://github.com/vllm-project/vllm/pull/5909/files.
170 + return snapshot_download(repo_id="SangBinCho/mixtral-lora")
171
172
173 @pytest.fixture(scope="session")
tests/lora/test_mixtral.py
+2 -2
@@ -40,14 +40,14 @@ def test_mixtral_lora(mixtral_lora_files, tp_size):
40 enable_lora=True,
41 max_num_seqs=16,
42 max_loras=4,
43 + distributed_executor_backend="ray",
44 tensor_parallel_size=tp_size)
45
46 expected_lora_output = [
47 "give_opinion(name[SpellForce 3], release_year[2017], developer[Grimlore Games], rating[poor])", # noqa: E501
47 - "give_opinion(name[SpellForce 3], release_year[2017], developer[Grimlore Games], rating[poor])", # noqa: E501
48 + "give_opinion(name[SpellForce 3], developer[Grimlore Games], release_year[2017], rating[poor])", # noqa: E501
49 "inform(name[BioShock], release_year[2007], rating[good], genres[action-adventure, role-playing, shooter], platforms[PlayStation, Xbox, PC], available_on_steam[yes], has_linux_release[no], has_mac_release[yes])", # noqa: E501
50 ]
50 -
51 assert do_sample(llm, mixtral_lora_files,
52 lora_id=1) == expected_lora_output
53 assert do_sample(llm, mixtral_lora_files,
vllm/lora/models.py
+46 -17
@@ -303,25 +303,54 @@ class LoRAModel:
303 "new_embeddings.bin")
304 with open(lora_config_path) as f:
305 config = json.load(f)
306 - target_modules = config["target_modules"]
307 - unexpected_modules = []
308 - for module in target_modules:
309 - # Compatible with more modules, such as:layers.11.self_attn.k_proj
310 - part_name = module.split(".")[-1]
311 - if part_name not in expected_lora_modules:
312 - unexpected_modules.append(module)
313 - # loaded lora's target modules must be a subset of expected_lora_modules
314 -
315 - if unexpected_modules:
316 - print(unexpected_modules, "modules")
317 - raise ValueError(
318 - f"While loading {lora_dir}, expected"
319 - f" target modules in {expected_lora_modules}"
320 - f" but received {unexpected_modules}."
321 - f" Please verify that the loaded LoRA module is correct")
306 if os.path.isfile(lora_tensor_path):
323 - tensors = safetensors.torch.load_file(lora_tensor_path)
307 + tensors: Dict[str, torch.Tensor] = {}
308 + # Find unexpected modules.
309 + # Use safetensor key as a source of truth to find expected modules.
310 + # in peft if you have target_modules A, B, C and C does not exist
311 + # in the model it won’t error and model will be trained with A, B
312 + # loraified. C won’t exist in the safetensor but it will exist in
313 + # the target_modules of the adapter_config.json.
314 + unexpected_modules = []
315 + with safetensors.safe_open(lora_tensor_path,
316 + framework="pt") as f: # type: ignore
317 + for lora_module in f.keys(): # noqa
318 + module_name, _ = parse_fine_tuned_lora_name(lora_module)
319 + part_name = module_name.split(".")[-1]
320 + if part_name not in expected_lora_modules:
321 + unexpected_modules.append(module_name)
322 + if unexpected_modules:
323 + raise ValueError(
324 + f"While loading {lora_dir}, expected"
325 + f" target modules in {expected_lora_modules}"
326 + f" but received {unexpected_modules}."
327 + f" Please verify that the loaded LoRA module is correct"
328 + )
329 + # Load tensors if there are only expected modules.
330 + for module in f.keys(): # noqa
331 + tensors[module] = f.get_tensor(module)
332 elif os.path.isfile(lora_bin_file_path):
333 + # When a bin file is provided, we rely on config to find unexpected
334 + # modules.
335 + unexpected_modules = []
336 + target_modules = config["target_modules"]
337 + for module in target_modules:
338 + # Compatible with more modules,
339 + # such as:layers.11.self_attn.k_proj
340 + part_name = module.split(".")[-1]
341 + if part_name not in expected_lora_modules:
342 + unexpected_modules.append(module)
343 + # loaded lora's target modules must be a subset of
344 + # expected_lora_modules. It is not reliable. See
345 + # https://github.com/vllm-project/vllm/pull/5909. But there's no
346 + # other better mechanism.
347 + if unexpected_modules:
348 + print(unexpected_modules, "modules")
349 + raise ValueError(
350 + f"While loading {lora_dir}, expected"
351 + f" target modules in {expected_lora_modules}"
352 + f" but received {unexpected_modules}."
353 + f" Please verify that the loaded LoRA module is correct")
354 tensors = torch.load(lora_bin_file_path)
355 else:
356 raise ValueError(f"{lora_dir} doesn't contain tensors")