@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b1b1038fbd
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Bugfix] Fix Qwen2-VL LoRA weight loading (#11430)

Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>

Jee Jee Li committed Dec 24, 2024 at 17:56 UTC b1b1038fbdc1f14b32c5e348194aae395124b43c
7 files changed +168 -14
tests/lora/conftest.py
+5
@@ -200,6 +200,11 @@ def minicpmv_lora_files():
200 return snapshot_download(repo_id="jeeejeee/minicpmv25-lora-pokemon")
201
202
203 +@pytest.fixture(scope="session")
204 +def qwen2vl_lora_files():
205 + return snapshot_download(repo_id="jeeejeee/qwen2-vl-lora-pokemon")
206 +
207 +
208 @pytest.fixture(scope="session")
209 def tinyllama_lora_files():
210 return snapshot_download(repo_id="jashing/tinyllama-colorist-lora")
tests/lora/test_lora_checkpoints.py
+30
@@ -4,6 +4,7 @@ import pytest
4
5 from vllm.lora.models import LoRAModel
6 from vllm.model_executor.models.baichuan import BaiChuanBaseForCausalLM
7 +from vllm.model_executor.models.utils import WeightsMapper
8
9 lora_lst = [
10 "baichuan7B", "baichuan7B-zero", "baichuan7B-zero-regex", "chatglm3-6b"
@@ -71,3 +72,32 @@ def test_load_checkpoints(
72 device="cpu",
73 embedding_modules=embedding_modules,
74 embedding_padding_modules=embed_padding_modules)
75 +
76 +
77 +def test_lora_weights_mapping(baichuan_lora_files, ):
78 + supported_lora_modules = BaiChuanBaseForCausalLM.supported_lora_modules
79 + packed_modules_mapping = BaiChuanBaseForCausalLM.packed_modules_mapping
80 + embedding_modules = BaiChuanBaseForCausalLM.embedding_modules
81 + embed_padding_modules = BaiChuanBaseForCausalLM.embedding_padding_modules
82 + expected_lora_modules: List[str] = []
83 + for module in supported_lora_modules:
84 + if module in packed_modules_mapping:
85 + expected_lora_modules.extend(packed_modules_mapping[module])
86 + else:
87 + expected_lora_modules.append(module)
88 +
89 + hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={
90 + "model.": "language_model.model.",
91 + }, )
92 +
93 + lora_model = LoRAModel.from_local_checkpoint(
94 + baichuan_lora_files,
95 + expected_lora_modules,
96 + lora_model_id=1,
97 + device="cpu",
98 + embedding_modules=embedding_modules,
99 + embedding_padding_modules=embed_padding_modules,
100 + weights_mapper=hf_to_vllm_mapper,
101 + )
102 + for name in lora_model.loras:
103 + assert name.startswith(hf_to_vllm_mapper.orig_to_new_prefix["model."])
tests/lora/test_qwen2vl.py new
+78
@@ -0,0 +1,78 @@
1 +from typing import List
2 +
3 +import pytest
4 +
5 +import vllm
6 +from vllm.assets.image import ImageAsset
7 +from vllm.lora.request import LoRARequest
8 +from vllm.platforms import current_platform
9 +
10 +MODEL_PATH = "Qwen/Qwen2-VL-7B-Instruct"
11 +
12 +PROMPT_TEMPLATE = (
13 + "<|im_start|>system\nYou are a helpful assistant.<|im_end|>"
14 + "\n<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>"
15 + "What is in the image?<|im_end|>\n"
16 + "<|im_start|>assistant\n")
17 +
18 +IMAGE_ASSETS = [
19 + ImageAsset("stop_sign"),
20 + ImageAsset("cherry_blossom"),
21 +]
22 +
23 +# After fine-tuning with LoRA, all generated content should start begin `A`.
24 +EXPECTED_OUTPUT = [
25 + "A stop sign stands prominently in the foreground, with a traditional Chinese gate and a black SUV in the background, illustrating a blend of modern and cultural elements.", # noqa: E501
26 + "A majestic skyscraper stands tall, partially obscured by a vibrant canopy of cherry blossoms, against a clear blue sky.", # noqa: E501
27 +]
28 +
29 +
30 +def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:
31 + sampling_params = vllm.SamplingParams(
32 + temperature=0,
33 + max_tokens=5,
34 + )
35 +
36 + inputs = [{
37 + "prompt": PROMPT_TEMPLATE,
38 + "multi_modal_data": {
39 + "image": asset.pil_image
40 + },
41 + } for asset in IMAGE_ASSETS]
42 +
43 + outputs = llm.generate(
44 + inputs,
45 + sampling_params,
46 + lora_request=LoRARequest(str(lora_id), lora_id, lora_path)
47 + if lora_id else None,
48 + )
49 + # Print the outputs.
50 + generated_texts: List[str] = []
51 + for output in outputs:
52 + prompt = output.prompt
53 + generated_text = output.outputs[0].text.strip()
54 + generated_texts.append(generated_text)
55 + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
56 + return generated_texts
57 +
58 +
59 +@pytest.mark.xfail(current_platform.is_rocm(),
60 + reason="Qwen2-VL dependency xformers incompatible with ROCm"
61 + )
62 +def test_qwen2vl_lora(qwen2vl_lora_files):
63 + llm = vllm.LLM(
64 + MODEL_PATH,
65 + max_num_seqs=2,
66 + enable_lora=True,
67 + max_loras=2,
68 + max_lora_rank=16,
69 + trust_remote_code=True,
70 + mm_processor_kwargs={
71 + "min_pixels": 28 * 28,
72 + "max_pixels": 1280 * 28 * 28,
73 + },
74 + max_model_len=4096,
75 + )
76 + output1 = do_sample(llm, qwen2vl_lora_files, lora_id=1)
77 + for i in range(len(EXPECTED_OUTPUT)):
78 + assert EXPECTED_OUTPUT[i].startswith(output1[i])
vllm/lora/models.py
+6 -3
@@ -28,7 +28,7 @@ from vllm.lora.utils import (from_layer, from_layer_logits_processor,
28 parse_fine_tuned_lora_name, replace_submodule)
29 from vllm.model_executor.models import SupportsLoRA, supports_multimodal
30 from vllm.model_executor.models.module_mapping import MultiModelKeys
31 -from vllm.model_executor.models.utils import PPMissingLayer
31 +from vllm.model_executor.models.utils import PPMissingLayer, WeightsMapper
32 from vllm.utils import is_pin_memory_available
33
34 logger = init_logger(__name__)
@@ -113,13 +113,14 @@ class LoRAModel(AdapterModel):
113 target_embedding_padding: Optional[int] = None,
114 embedding_modules: Optional[Dict[str, str]] = None,
115 embedding_padding_modules: Optional[List[str]] = None,
116 + weights_mapper: Optional[WeightsMapper] = None,
117 ) -> "LoRAModel":
118 """Create a LoRAModel from a dictionary of tensors."""
119 pin_memory = str(device) == "cpu" and is_pin_memory_available()
120 loras: Dict[str, LoRALayerWeights] = {}
121 for tensor_name, tensor in tensors.items():
122 module_name, is_lora_a, is_bias = parse_fine_tuned_lora_name(
122 - tensor_name)
123 + tensor_name, weights_mapper)
124 if module_name not in loras:
125 lora_embeddings_tensor = None
126 if embeddings:
@@ -187,6 +188,7 @@ class LoRAModel(AdapterModel):
188 target_embedding_padding: Optional[int] = None,
189 embedding_modules: Optional[Dict[str, str]] = None,
190 embedding_padding_modules: Optional[List[str]] = None,
191 + weights_mapper: Optional[WeightsMapper] = None,
192 ) -> "LoRAModel":
193 """Create a LoRAModel from a local checkpoint.
194
@@ -289,7 +291,8 @@ class LoRAModel(AdapterModel):
291 embeddings=embeddings,
292 target_embedding_padding=target_embedding_padding,
293 embedding_modules=embedding_modules,
292 - embedding_padding_modules=embedding_padding_modules)
294 + embedding_padding_modules=embedding_padding_modules,
295 + weights_mapper=weights_mapper)
296
297
298 class LoRAModelManager(AdapterModelManager):
vllm/lora/utils.py
+33 -4
@@ -1,3 +1,4 @@
1 +import copy
2 import os
3 import re
4 from typing import List, Optional, Set, Tuple, Type, Union
@@ -30,6 +31,8 @@ from vllm.lora.layers import (BaseLayerWithLoRA, ColumnParallelLinearWithLoRA,
31 # yapf: enable
32 from vllm.model_executor.layers.logits_processor import LogitsProcessor
33 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
34 +from vllm.model_executor.models.utils import WeightsMapper
35 +from vllm.utils import print_warning_once
36
37 logger = init_logger(__name__)
38
@@ -91,28 +94,54 @@ def replace_submodule(model: nn.Module, module_name: str,
94 return new_module
95
96
94 -def parse_fine_tuned_lora_name(name: str) -> Tuple[str, bool, bool]:
97 +def parse_fine_tuned_lora_name(
98 + name: str,
99 + weights_mapper: Optional[WeightsMapper] = None
100 +) -> Tuple[str, bool, bool]:
101 """Parse the name of lora weights.
102
103 args:
104 name: the name of the fine-tuned LoRA, e.g.
105 base_model.model.dense1.weight
106 + weights_mapper: maps the name of weight, e.g.
107 + `model.` -> `language_model.model.`,
108 return:
109 Tuple(module_name, is_lora_a):
110 module_name: the name of the module, e.g. model.dense1,
111 is_lora_a whether the tensor is lora_a or lora_b.
112 is_bias whether the tensor is lora bias.
113 """
114 +
115 + w_mapper = None
116 + if weights_mapper:
117 + w_mapper = copy.deepcopy(weights_mapper)
118 + # TODO: Currently only supports mapping for prefix, mapping for
119 + # substr and subfix will be supported in the future.
120 + for attr, mapping in [
121 + ("orig_to_new_substr", w_mapper.orig_to_new_substr),
122 + ("orig_to_new_suffix", w_mapper.orig_to_new_suffix),
123 + ]:
124 + if mapping:
125 + print_warning_once(
126 + f"vLLM currently does not support mapping of LoRA weights "
127 + f"for {mapping}.")
128 + setattr(w_mapper, attr, {})
129 +
130 + mapper = (lambda name: w_mapper._map_name(name)
131 + if w_mapper is not None else name)
132 parts = name.split(".")
133 if parts[-1] == "weight" and (parts[-2] == "lora_A"
134 or parts[-2] == "lora_B"):
109 - return ".".join(parts[2:-2]), parts[-2] == "lora_A", False
135 + new_name = ".".join(parts[2:-2])
136 + return mapper(new_name), parts[-2] == "lora_A", False
137
138 if parts[-1] == "lora_embedding_A" or parts[-1] == "lora_embedding_B":
112 - return ".".join(parts[2:-1]), parts[-1] == "lora_embedding_A", False
139 + new_name = ".".join(parts[2:-1])
140 + return mapper(new_name), parts[-1] == "lora_embedding_A", False
141
142 if parts[-1] == "bias":
115 - return ".".join(parts[2:-2]), False, True
143 + new_name = ".".join(parts[2:-2])
144 + return mapper(new_name), False, True
145
146 raise ValueError(f"{name} is unsupported LoRA weight")
147
vllm/lora/worker_manager.py
+10 -1
@@ -92,6 +92,14 @@ class WorkerLoRAManager(AbstractWorkerManager):
92 else:
93 expected_lora_modules.append(module)
94 lora_path = get_adapter_absolute_path(lora_request.lora_path)
95 +
96 + # For some models like Qwen2VL, we need to use hf_to_vllm_mapper
97 + # to ensure correct loading of lora weights.
98 + hf_to_vllm_mapper = None
99 + if (hasattr(model, "hf_to_vllm_mapper")
100 + and model.hf_to_vllm_mapper is not None):
101 + hf_to_vllm_mapper = model.hf_to_vllm_mapper
102 +
103 lora = self._lora_model_cls.from_local_checkpoint(
104 lora_path,
105 expected_lora_modules,
@@ -103,7 +111,8 @@ class WorkerLoRAManager(AbstractWorkerManager):
111 self.lora_config.lora_extra_vocab_size,
112 embedding_modules=self.embedding_modules,
113 embedding_padding_modules=self.embedding_padding_modules,
106 - )
114 + weights_mapper=hf_to_vllm_mapper)
115 +
116 except Exception as e:
117 raise RuntimeError(f"Loading lora {lora_path} failed") from e
118 if lora.rank > self.lora_config.max_lora_rank:
vllm/model_executor/models/qwen2_vl.py
+6 -6
@@ -901,6 +901,11 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
901 ]
902 embedding_modules = {}
903 embedding_padding_modules = []
904 + # To ensure correct weight loading and mapping.
905 + hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={
906 + "lm_head.": "language_model.lm_head.",
907 + "model.": "language_model.model.",
908 + })
909
910 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
911 super().__init__()
@@ -1190,11 +1195,6 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1195
1196 def load_weights(self, weights: Iterable[Tuple[str,
1197 torch.Tensor]]) -> Set[str]:
1193 - hf_to_vllm_mapper = WeightsMapper(
1194 - orig_to_new_prefix={
1195 - "lm_head.": "language_model.lm_head.",
1196 - "model.": "language_model.model.",
1197 - })
1198
1199 loader = AutoWeightsLoader(self)
1200 - return loader.load_weights(weights, mapper=hf_to_vllm_mapper)
1200 + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)