A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Misc][LoRA] Fix LoRA weight mapper (#11495)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Dec 26, 2024 at 15:52 UTC
aa25985bd1e7a4925a7061fdfbc93893b492627b
5 files changed
+31
-29
tests/lora/test_lora_checkpoints.py
+10
-5
@@ -74,7 +74,7 @@ def test_load_checkpoints(
74
embedding_padding_modules=embed_padding_modules)
75
76
77
-def test_lora_weights_mapping(baichuan_lora_files, ):
77
+def test_lora_weights_mapping(baichuan_lora_files):
78
supported_lora_modules = BaiChuanBaseForCausalLM.supported_lora_modules
79
packed_modules_mapping = BaiChuanBaseForCausalLM.packed_modules_mapping
80
embedding_modules = BaiChuanBaseForCausalLM.embedding_modules
@@ -86,10 +86,14 @@ def test_lora_weights_mapping(baichuan_lora_files, ):
86
else:
87
expected_lora_modules.append(module)
88
89
- hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={
90
- "model.": "language_model.model.",
91
- }, )
92
-
89
+ hf_to_vllm_mapper = WeightsMapper(
90
+ orig_to_new_prefix={
91
+ "model.": "language_model.model.",
92
+ },
93
+ orig_to_new_substr={
94
+ ".layers.": ".baichuan_layers.",
95
+ },
96
+ )
97
lora_model = LoRAModel.from_local_checkpoint(
98
baichuan_lora_files,
99
expected_lora_modules,
@@ -101,3 +105,4 @@ def test_lora_weights_mapping(baichuan_lora_files, ):
105
)
106
for name in lora_model.loras:
107
assert name.startswith(hf_to_vllm_mapper.orig_to_new_prefix["model."])
108
+ assert ".baichuan_layers." in name
tests/lora/test_qwen2vl.py
+5
-1
@@ -22,7 +22,7 @@ IMAGE_ASSETS = [
22
23
# After fine-tuning with LoRA, all generated content should start begin `A`.
24
EXPECTED_OUTPUT = [
25
- "A stop sign stands prominently in the foreground, with a traditional Chinese gate and a black SUV in the background, illustrating a blend of modern and cultural elements.", # noqa: E501
25
+ "A red stop sign stands prominently in the foreground, with a traditional Chinese gate and a black SUV in the background, illustrating a blend of modern and cultural elements.", # noqa: E501
26
"A majestic skyscraper stands tall, partially obscured by a vibrant canopy of cherry blossoms, against a clear blue sky.", # noqa: E501
27
]
28
@@ -76,3 +76,7 @@ def test_qwen2vl_lora(qwen2vl_lora_files):
76
output1 = do_sample(llm, qwen2vl_lora_files, lora_id=1)
77
for i in range(len(EXPECTED_OUTPUT)):
78
assert EXPECTED_OUTPUT[i].startswith(output1[i])
79
+
80
+ output2 = do_sample(llm, qwen2vl_lora_files, lora_id=2)
81
+ for i in range(len(EXPECTED_OUTPUT)):
82
+ assert EXPECTED_OUTPUT[i].startswith(output2[i])
vllm/lora/models.py
+2
-1
@@ -231,7 +231,8 @@ class LoRAModel(AdapterModel):
231
with safetensors.safe_open(lora_tensor_path,
232
framework="pt") as f: # type: ignore
233
for lora_module in f.keys(): # noqa
234
- module_name, _, _ = parse_fine_tuned_lora_name(lora_module)
234
+ module_name, _, _ = parse_fine_tuned_lora_name(
235
+ lora_module, weights_mapper)
236
part_name = module_name.split(".")[-1]
237
if part_name not in expected_lora_modules:
238
unexpected_modules.append(module_name)
vllm/lora/utils.py
+12
-22
@@ -1,4 +1,3 @@
1
-import copy
1
import os
2
import re
3
from typing import List, Optional, Set, Tuple, Type, Union
@@ -32,7 +31,6 @@ from vllm.lora.layers import (BaseLayerWithLoRA, ColumnParallelLinearWithLoRA,
31
from vllm.model_executor.layers.logits_processor import LogitsProcessor
32
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
33
from vllm.model_executor.models.utils import WeightsMapper
35
-from vllm.utils import print_warning_once
34
35
logger = init_logger(__name__)
36
@@ -112,36 +110,28 @@ def parse_fine_tuned_lora_name(
110
is_bias whether the tensor is lora bias.
111
"""
112
115
- w_mapper = None
116
- if weights_mapper:
117
- w_mapper = copy.deepcopy(weights_mapper)
118
- # TODO: Currently only supports mapping for prefix, mapping for
119
- # substr and subfix will be supported in the future.
120
- for attr, mapping in [
121
- ("orig_to_new_substr", w_mapper.orig_to_new_substr),
122
- ("orig_to_new_suffix", w_mapper.orig_to_new_suffix),
123
- ]:
124
- if mapping:
125
- print_warning_once(
126
- f"vLLM currently does not support mapping of LoRA weights "
127
- f"for {mapping}.")
128
- setattr(w_mapper, attr, {})
129
-
130
- mapper = (lambda name: w_mapper._map_name(name)
131
- if w_mapper is not None else name)
113
+ # LoRA weight qualified name always starts with `base_model.model.`,
114
+ # so we remove the prefix `base_model.model.` to make the following
115
+ # mapping correctly.
116
+ if "base_model.model." in name:
117
+ name = name.replace("base_model.model.", "")
118
+ name = weights_mapper._map_name(name) if weights_mapper else name
119
+ # recover the prefix `base_model.model.`
120
+ name = "base_model.model." + name
121
+
122
parts = name.split(".")
123
if parts[-1] == "weight" and (parts[-2] == "lora_A"
124
or parts[-2] == "lora_B"):
125
new_name = ".".join(parts[2:-2])
136
- return mapper(new_name), parts[-2] == "lora_A", False
126
+ return new_name, parts[-2] == "lora_A", False
127
128
if parts[-1] == "lora_embedding_A" or parts[-1] == "lora_embedding_B":
129
new_name = ".".join(parts[2:-1])
140
- return mapper(new_name), parts[-1] == "lora_embedding_A", False
130
+ return new_name, parts[-1] == "lora_embedding_A", False
131
132
if parts[-1] == "bias":
133
new_name = ".".join(parts[2:-2])
144
- return mapper(new_name), False, True
134
+ return new_name, False, True
135
136
raise ValueError(f"{name} is unsupported LoRA weight")
137
vllm/lora/worker_manager.py
+2
@@ -91,6 +91,8 @@ class WorkerLoRAManager(AbstractWorkerManager):
91
packed_modules_mapping[module])
92
else:
93
expected_lora_modules.append(module)
94
+
95
+ expected_lora_modules = list(set(expected_lora_modules))
96
lora_path = get_adapter_absolute_path(lora_request.lora_path)
97
98
# For some models like Qwen2VL, we need to use hf_to_vllm_mapper