A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Bugfix] Fix Qwen2-VL LoRA weight loading (#11430)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Dec 24, 2024 at 17:56 UTC
b1b1038fbdc1f14b32c5e348194aae395124b43c
7 files changed
+168
-14
tests/lora/conftest.py
+5
@@ -200,6 +200,11 @@ def minicpmv_lora_files():
200
return snapshot_download(repo_id="jeeejeee/minicpmv25-lora-pokemon")
201
202
203
+@pytest.fixture(scope="session")
204
+def qwen2vl_lora_files():
205
+ return snapshot_download(repo_id="jeeejeee/qwen2-vl-lora-pokemon")
206
+
207
+
208
@pytest.fixture(scope="session")
209
def tinyllama_lora_files():
210
return snapshot_download(repo_id="jashing/tinyllama-colorist-lora")
tests/lora/test_lora_checkpoints.py
+30
@@ -4,6 +4,7 @@ import pytest
4
5
from vllm.lora.models import LoRAModel
6
from vllm.model_executor.models.baichuan import BaiChuanBaseForCausalLM
7
+from vllm.model_executor.models.utils import WeightsMapper
8
9
lora_lst = [
10
"baichuan7B", "baichuan7B-zero", "baichuan7B-zero-regex", "chatglm3-6b"
@@ -71,3 +72,32 @@ def test_load_checkpoints(
72
device="cpu",
73
embedding_modules=embedding_modules,
74
embedding_padding_modules=embed_padding_modules)
75
+
76
+
77
+def test_lora_weights_mapping(baichuan_lora_files, ):
78
+ supported_lora_modules = BaiChuanBaseForCausalLM.supported_lora_modules
79
+ packed_modules_mapping = BaiChuanBaseForCausalLM.packed_modules_mapping
80
+ embedding_modules = BaiChuanBaseForCausalLM.embedding_modules
81
+ embed_padding_modules = BaiChuanBaseForCausalLM.embedding_padding_modules
82
+ expected_lora_modules: List[str] = []
83
+ for module in supported_lora_modules:
84
+ if module in packed_modules_mapping:
85
+ expected_lora_modules.extend(packed_modules_mapping[module])
86
+ else:
87
+ expected_lora_modules.append(module)
88
+
89
+ hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={
90
+ "model.": "language_model.model.",
91
+ }, )
92
+
93
+ lora_model = LoRAModel.from_local_checkpoint(
94
+ baichuan_lora_files,
95
+ expected_lora_modules,
96
+ lora_model_id=1,
97
+ device="cpu",
98
+ embedding_modules=embedding_modules,
99
+ embedding_padding_modules=embed_padding_modules,
100
+ weights_mapper=hf_to_vllm_mapper,
101
+ )
102
+ for name in lora_model.loras:
103
+ assert name.startswith(hf_to_vllm_mapper.orig_to_new_prefix["model."])
tests/lora/test_qwen2vl.py
new
+78
@@ -0,0 +1,78 @@
1
+from typing import List
2
+
3
+import pytest
4
+
5
+import vllm
6
+from vllm.assets.image import ImageAsset
7
+from vllm.lora.request import LoRARequest
8
+from vllm.platforms import current_platform
9
+
10
+MODEL_PATH = "Qwen/Qwen2-VL-7B-Instruct"
11
+
12
+PROMPT_TEMPLATE = (
13
+ "<|im_start|>system\nYou are a helpful assistant.<|im_end|>"
14
+ "\n<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>"
15
+ "What is in the image?<|im_end|>\n"
16
+ "<|im_start|>assistant\n")
17
+
18
+IMAGE_ASSETS = [
19
+ ImageAsset("stop_sign"),
20
+ ImageAsset("cherry_blossom"),
21
+]
22
+
23
+# After fine-tuning with LoRA, all generated content should start begin `A`.
24
+EXPECTED_OUTPUT = [
25
+ "A stop sign stands prominently in the foreground, with a traditional Chinese gate and a black SUV in the background, illustrating a blend of modern and cultural elements.", # noqa: E501
26
+ "A majestic skyscraper stands tall, partially obscured by a vibrant canopy of cherry blossoms, against a clear blue sky.", # noqa: E501
27
+]
28
+
29
+
30
+def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:
31
+ sampling_params = vllm.SamplingParams(
32
+ temperature=0,
33
+ max_tokens=5,
34
+ )
35
+
36
+ inputs = [{
37
+ "prompt": PROMPT_TEMPLATE,
38
+ "multi_modal_data": {
39
+ "image": asset.pil_image
40
+ },
41
+ } for asset in IMAGE_ASSETS]
42
+
43
+ outputs = llm.generate(
44
+ inputs,
45
+ sampling_params,
46
+ lora_request=LoRARequest(str(lora_id), lora_id, lora_path)
47
+ if lora_id else None,
48
+ )
49
+ # Print the outputs.
50
+ generated_texts: List[str] = []
51
+ for output in outputs:
52
+ prompt = output.prompt
53
+ generated_text = output.outputs[0].text.strip()
54
+ generated_texts.append(generated_text)
55
+ print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
56
+ return generated_texts
57
+
58
+
59
+@pytest.mark.xfail(current_platform.is_rocm(),
60
+ reason="Qwen2-VL dependency xformers incompatible with ROCm"
61
+ )
62
+def test_qwen2vl_lora(qwen2vl_lora_files):
63
+ llm = vllm.LLM(
64
+ MODEL_PATH,
65
+ max_num_seqs=2,
66
+ enable_lora=True,
67
+ max_loras=2,
68
+ max_lora_rank=16,
69
+ trust_remote_code=True,
70
+ mm_processor_kwargs={
71
+ "min_pixels": 28 * 28,
72
+ "max_pixels": 1280 * 28 * 28,
73
+ },
74
+ max_model_len=4096,
75
+ )
76
+ output1 = do_sample(llm, qwen2vl_lora_files, lora_id=1)
77
+ for i in range(len(EXPECTED_OUTPUT)):
78
+ assert EXPECTED_OUTPUT[i].startswith(output1[i])
vllm/lora/models.py
+6
-3
@@ -28,7 +28,7 @@ from vllm.lora.utils import (from_layer, from_layer_logits_processor,
28
parse_fine_tuned_lora_name, replace_submodule)
29
from vllm.model_executor.models import SupportsLoRA, supports_multimodal
30
from vllm.model_executor.models.module_mapping import MultiModelKeys
31
-from vllm.model_executor.models.utils import PPMissingLayer
31
+from vllm.model_executor.models.utils import PPMissingLayer, WeightsMapper
32
from vllm.utils import is_pin_memory_available
33
34
logger = init_logger(__name__)
@@ -113,13 +113,14 @@ class LoRAModel(AdapterModel):
113
target_embedding_padding: Optional[int] = None,
114
embedding_modules: Optional[Dict[str, str]] = None,
115
embedding_padding_modules: Optional[List[str]] = None,
116
+ weights_mapper: Optional[WeightsMapper] = None,
117
) -> "LoRAModel":
118
"""Create a LoRAModel from a dictionary of tensors."""
119
pin_memory = str(device) == "cpu" and is_pin_memory_available()
120
loras: Dict[str, LoRALayerWeights] = {}
121
for tensor_name, tensor in tensors.items():
122
module_name, is_lora_a, is_bias = parse_fine_tuned_lora_name(
122
- tensor_name)
123
+ tensor_name, weights_mapper)
124
if module_name not in loras:
125
lora_embeddings_tensor = None
126
if embeddings:
@@ -187,6 +188,7 @@ class LoRAModel(AdapterModel):
188
target_embedding_padding: Optional[int] = None,
189
embedding_modules: Optional[Dict[str, str]] = None,
190
embedding_padding_modules: Optional[List[str]] = None,
191
+ weights_mapper: Optional[WeightsMapper] = None,
192
) -> "LoRAModel":
193
"""Create a LoRAModel from a local checkpoint.
194
@@ -289,7 +291,8 @@ class LoRAModel(AdapterModel):
291
embeddings=embeddings,
292
target_embedding_padding=target_embedding_padding,
293
embedding_modules=embedding_modules,
292
- embedding_padding_modules=embedding_padding_modules)
294
+ embedding_padding_modules=embedding_padding_modules,
295
+ weights_mapper=weights_mapper)
296
297
298
class LoRAModelManager(AdapterModelManager):
vllm/lora/utils.py
+33
-4
@@ -1,3 +1,4 @@
1
+import copy
2
import os
3
import re
4
from typing import List, Optional, Set, Tuple, Type, Union
@@ -30,6 +31,8 @@ from vllm.lora.layers import (BaseLayerWithLoRA, ColumnParallelLinearWithLoRA,
31
# yapf: enable
32
from vllm.model_executor.layers.logits_processor import LogitsProcessor
33
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
34
+from vllm.model_executor.models.utils import WeightsMapper
35
+from vllm.utils import print_warning_once
36
37
logger = init_logger(__name__)
38
@@ -91,28 +94,54 @@ def replace_submodule(model: nn.Module, module_name: str,
94
return new_module
95
96
94
-def parse_fine_tuned_lora_name(name: str) -> Tuple[str, bool, bool]:
97
+def parse_fine_tuned_lora_name(
98
+ name: str,
99
+ weights_mapper: Optional[WeightsMapper] = None
100
+) -> Tuple[str, bool, bool]:
101
"""Parse the name of lora weights.
102
103
args:
104
name: the name of the fine-tuned LoRA, e.g.
105
base_model.model.dense1.weight
106
+ weights_mapper: maps the name of weight, e.g.
107
+ `model.` -> `language_model.model.`,
108
return:
109
Tuple(module_name, is_lora_a):
110
module_name: the name of the module, e.g. model.dense1,
111
is_lora_a whether the tensor is lora_a or lora_b.
112
is_bias whether the tensor is lora bias.
113
"""
114
+
115
+ w_mapper = None
116
+ if weights_mapper:
117
+ w_mapper = copy.deepcopy(weights_mapper)
118
+ # TODO: Currently only supports mapping for prefix, mapping for
119
+ # substr and subfix will be supported in the future.
120
+ for attr, mapping in [
121
+ ("orig_to_new_substr", w_mapper.orig_to_new_substr),
122
+ ("orig_to_new_suffix", w_mapper.orig_to_new_suffix),
123
+ ]:
124
+ if mapping:
125
+ print_warning_once(
126
+ f"vLLM currently does not support mapping of LoRA weights "
127
+ f"for {mapping}.")
128
+ setattr(w_mapper, attr, {})
129
+
130
+ mapper = (lambda name: w_mapper._map_name(name)
131
+ if w_mapper is not None else name)
132
parts = name.split(".")
133
if parts[-1] == "weight" and (parts[-2] == "lora_A"
134
or parts[-2] == "lora_B"):
109
- return ".".join(parts[2:-2]), parts[-2] == "lora_A", False
135
+ new_name = ".".join(parts[2:-2])
136
+ return mapper(new_name), parts[-2] == "lora_A", False
137
138
if parts[-1] == "lora_embedding_A" or parts[-1] == "lora_embedding_B":
112
- return ".".join(parts[2:-1]), parts[-1] == "lora_embedding_A", False
139
+ new_name = ".".join(parts[2:-1])
140
+ return mapper(new_name), parts[-1] == "lora_embedding_A", False
141
142
if parts[-1] == "bias":
115
- return ".".join(parts[2:-2]), False, True
143
+ new_name = ".".join(parts[2:-2])
144
+ return mapper(new_name), False, True
145
146
raise ValueError(f"{name} is unsupported LoRA weight")
147
vllm/lora/worker_manager.py
+10
-1
@@ -92,6 +92,14 @@ class WorkerLoRAManager(AbstractWorkerManager):
92
else:
93
expected_lora_modules.append(module)
94
lora_path = get_adapter_absolute_path(lora_request.lora_path)
95
+
96
+ # For some models like Qwen2VL, we need to use hf_to_vllm_mapper
97
+ # to ensure correct loading of lora weights.
98
+ hf_to_vllm_mapper = None
99
+ if (hasattr(model, "hf_to_vllm_mapper")
100
+ and model.hf_to_vllm_mapper is not None):
101
+ hf_to_vllm_mapper = model.hf_to_vllm_mapper
102
+
103
lora = self._lora_model_cls.from_local_checkpoint(
104
lora_path,
105
expected_lora_modules,
@@ -103,7 +111,8 @@ class WorkerLoRAManager(AbstractWorkerManager):
111
self.lora_config.lora_extra_vocab_size,
112
embedding_modules=self.embedding_modules,
113
embedding_padding_modules=self.embedding_padding_modules,
106
- )
114
+ weights_mapper=hf_to_vllm_mapper)
115
+
116
except Exception as e:
117
raise RuntimeError(f"Loading lora {lora_path} failed") from e
118
if lora.rank > self.lora_config.max_lora_rank:
vllm/model_executor/models/qwen2_vl.py
+6
-6
@@ -901,6 +901,11 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
901
]
902
embedding_modules = {}
903
embedding_padding_modules = []
904
+ # To ensure correct weight loading and mapping.
905
+ hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={
906
+ "lm_head.": "language_model.lm_head.",
907
+ "model.": "language_model.model.",
908
+ })
909
910
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
911
super().__init__()
@@ -1190,11 +1195,6 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1195
1196
def load_weights(self, weights: Iterable[Tuple[str,
1197
torch.Tensor]]) -> Set[str]:
1193
- hf_to_vllm_mapper = WeightsMapper(
1194
- orig_to_new_prefix={
1195
- "lm_head.": "language_model.lm_head.",
1196
- "model.": "language_model.model.",
1197
- })
1198
1199
loader = AutoWeightsLoader(self)
1200
- return loader.load_weights(weights, mapper=hf_to_vllm_mapper)
1200
+ return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)