@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3d49776bbb
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Model][LoRA]LoRA support added for MiniCPMV2.5 (#7199)

Jee Jee Li committed Sep 29, 2024 at 14:59 UTC 3d49776bbb25927abf91bb7c5537e0006c199c16
8 files changed +378 -31
tests/lora/conftest.py
+5
@@ -194,6 +194,11 @@ def baichuan_zero_lora_files():
194 return snapshot_download(repo_id="jeeejeee/baichuan7b-zero-init")
195
196
197 +@pytest.fixture(scope="session")
198 +def minicpmv_lora_files():
199 + return snapshot_download(repo_id="jeeejeee/minicpmv25-lora-pokemon")
200 +
201 +
202 @pytest.fixture(scope="session")
203 def tinyllama_lora_files():
204 return snapshot_download(repo_id="jashing/tinyllama-colorist-lora")
tests/lora/test_minicpmv.py new
+71
@@ -0,0 +1,71 @@
1 +from typing import List
2 +
3 +import vllm
4 +from vllm.assets.image import ImageAsset
5 +from vllm.lora.request import LoRARequest
6 +
7 +MODEL_PATH = "openbmb/MiniCPM-Llama3-V-2_5"
8 +
9 +PROMPT_TEMPLATE = (
10 + "<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n"
11 + "(<image>./</image>)\nWhat is in the image?<|eot_id|>"
12 + "<|start_header_id|>assistant<|end_header_id|>\n\n")
13 +
14 +IMAGE_ASSETS = [
15 + ImageAsset("stop_sign"),
16 + ImageAsset("cherry_blossom"),
17 +]
18 +
19 +# After fine-tuning with LoRA, all generated content should start begin `A`.
20 +EXPECTED_OUTPUT = [
21 + "A red and white stop sign with a Chinese archway in the background featuring red lanterns and gold accents.", # noqa: E501
22 + "A pink cherry blossom tree with a blue sky in the background.",
23 +]
24 +
25 +
26 +def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:
27 + sampling_params = vllm.SamplingParams(
28 + temperature=0,
29 + max_tokens=5,
30 + stop_token_ids=[128001, 128009], # eos_id, eot_id
31 + )
32 +
33 + inputs = [{
34 + "prompt": PROMPT_TEMPLATE,
35 + "multi_modal_data": {
36 + "image": asset.pil_image
37 + },
38 + } for asset in IMAGE_ASSETS]
39 +
40 + outputs = llm.generate(
41 + inputs,
42 + sampling_params,
43 + lora_request=LoRARequest(str(lora_id), lora_id, lora_path)
44 + if lora_id else None,
45 + )
46 + # Print the outputs.
47 + generated_texts: List[str] = []
48 + for output in outputs:
49 + prompt = output.prompt
50 + generated_text = output.outputs[0].text.strip()
51 + generated_texts.append(generated_text)
52 + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
53 + return generated_texts
54 +
55 +
56 +def test_minicpmv_lora(minicpmv_lora_files):
57 + llm = vllm.LLM(
58 + MODEL_PATH,
59 + max_num_seqs=2,
60 + enable_lora=True,
61 + max_loras=4,
62 + max_lora_rank=64,
63 + trust_remote_code=True,
64 + )
65 +
66 + output1 = do_sample(llm, minicpmv_lora_files, lora_id=1)
67 + for i in range(len(EXPECTED_OUTPUT)):
68 + assert EXPECTED_OUTPUT[i].startswith(output1[i])
69 + output2 = do_sample(llm, minicpmv_lora_files, lora_id=2)
70 + for i in range(len(EXPECTED_OUTPUT)):
71 + assert EXPECTED_OUTPUT[i].startswith(output2[i])
tests/lora/test_minicpmv_tp.py new
+95
@@ -0,0 +1,95 @@
1 +from typing import List
2 +
3 +import pytest
4 +
5 +import vllm
6 +from vllm.assets.image import ImageAsset
7 +from vllm.lora.request import LoRARequest
8 +
9 +from ..utils import multi_gpu_test
10 +
11 +MODEL_PATH = "openbmb/MiniCPM-Llama3-V-2_5"
12 +
13 +PROMPT_TEMPLATE = (
14 + "<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n"
15 + "(<image>./</image>)\nWhat is in the image?<|eot_id|>"
16 + "<|start_header_id|>assistant<|end_header_id|>\n\n")
17 +
18 +IMAGE_ASSETS = [
19 + ImageAsset("stop_sign"),
20 + ImageAsset("cherry_blossom"),
21 +]
22 +
23 +# After fine-tuning with LoRA, all generated content should start begin `A`.
24 +EXPECTED_OUTPUT = [
25 + "A red and white stop sign with a Chinese archway in the background featuring red lanterns and gold accents.", # noqa: E501
26 + "A pink cherry blossom tree with a blue sky in the background.",
27 +]
28 +
29 +
30 +def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:
31 + sampling_params = vllm.SamplingParams(
32 + temperature=0,
33 + max_tokens=5,
34 + stop_token_ids=[128001, 128009], # eos_id, eot_id
35 + )
36 +
37 + inputs = [{
38 + "prompt": PROMPT_TEMPLATE,
39 + "multi_modal_data": {
40 + "image": asset.pil_image
41 + },
42 + } for asset in IMAGE_ASSETS]
43 +
44 + outputs = llm.generate(
45 + inputs,
46 + sampling_params,
47 + lora_request=LoRARequest(str(lora_id), lora_id, lora_path)
48 + if lora_id else None,
49 + )
50 + # Print the outputs.
51 + generated_texts: List[str] = []
52 + for output in outputs:
53 + prompt = output.prompt
54 + generated_text = output.outputs[0].text.strip()
55 + generated_texts.append(generated_text)
56 + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
57 + return generated_texts
58 +
59 +
60 +@multi_gpu_test(num_gpus=2)
61 +@pytest.mark.parametrize("fully_sharded", [True, False])
62 +def test_minicpmv_tp2(minicpmv_lora_files, fully_sharded):
63 + llm = vllm.LLM(
64 + MODEL_PATH,
65 + enable_lora=True,
66 + max_num_seqs=2,
67 + max_loras=4,
68 + max_lora_rank=64,
69 + tensor_parallel_size=2,
70 + trust_remote_code=True,
71 + fully_sharded_loras=fully_sharded,
72 + )
73 +
74 + output_tp = do_sample(llm, minicpmv_lora_files, lora_id=1)
75 +
76 + for i in range(len(EXPECTED_OUTPUT)):
77 + assert EXPECTED_OUTPUT[i].startswith(output_tp[i])
78 +
79 +
80 +@multi_gpu_test(num_gpus=4)
81 +@pytest.mark.parametrize("fully_sharded", [True, False])
82 +def test_minicpmv_tp4(minicpmv_lora_files, fully_sharded):
83 + llm = vllm.LLM(
84 + MODEL_PATH,
85 + enable_lora=True,
86 + max_num_seqs=2,
87 + max_loras=4,
88 + max_lora_rank=64,
89 + tensor_parallel_size=4,
90 + trust_remote_code=True,
91 + fully_sharded_loras=fully_sharded,
92 + )
93 + output_tp = do_sample(llm, minicpmv_lora_files, lora_id=1)
94 + for i in range(len(EXPECTED_OUTPUT)):
95 + assert EXPECTED_OUTPUT[i].startswith(output_tp[i])
vllm/lora/models.py
+41 -4
@@ -24,7 +24,9 @@ from vllm.lora.lora import LoRALayerWeights, PackedLoRALayerWeights
24 from vllm.lora.punica import PunicaWrapper
25 from vllm.lora.utils import (from_layer, from_layer_logits_processor,
26 parse_fine_tuned_lora_name, replace_submodule)
27 -from vllm.model_executor.models.interfaces import SupportsLoRA
27 +from vllm.model_executor.models.interfaces import (SupportsLoRA,
28 + supports_multimodal)
29 +from vllm.model_executor.models.module_mapping import MultiModelKeys
30 from vllm.model_executor.models.utils import PPMissingLayer
31 from vllm.utils import is_pin_memory_available
32
@@ -332,6 +334,8 @@ class LoRAModelManager(AdapterModelManager):
334 self.supported_lora_modules.append("rotary_emb")
335 self.packed_modules_mapping = copy.deepcopy(
336 self.model.packed_modules_mapping)
337 + # Used to indicate whether the model is a multimodal model
338 + self.supports_mm: bool = supports_multimodal(self.model)
339 self.packed_modules: Dict[str, List[str]] = {}
340 self.modules: Dict[str, "BaseLayerWithLoRA"] = {}
341 # Dict instead of a Set for compatibility with LRUCache.
@@ -437,12 +441,22 @@ class LoRAModelManager(AdapterModelManager):
441 continue
442 if not self._match_target_modules(module_name):
443 continue
444 + # A temporary approach for multimodal models to support LoRA
445 + # TODO: Remove this restriction
446 + if self._filter_unsupported_mm_module(module_name):
447 + logger.warning(
448 + "Regarding multimodal models, vLLM currently only supports "
449 + "adding LoRA to language model, %s will be ignored.",
450 + module_name,
451 + )
452 + continue
453 parts = module_name.split(".")[-1]
454 packed_moduled_lst = self.packed_modules_mapping.get(parts, [])
455 new_module = replace_submodule(
456 self.model, module_name,
457 from_layer(module, self.lora_slots, self.lora_config,
458 packed_moduled_lst, self.model.config))
459 +
460 # LinearScalingRotaryEmbeddingWithLora is used to handle
461 # long context lora. Register relevant metadata.
462 if isinstance(new_module, LinearScalingRotaryEmbeddingWithLora):
@@ -460,6 +474,15 @@ class LoRAModelManager(AdapterModelManager):
474 module, self.lora_slots,
475 self.lora_config,
476 self.model.config))
477 +
478 + # In some models, especially multimodal ones, layers with the same
479 + # name may have different types, such as nn.Linear and
480 + # ReplicatedLinear. The nn.Linear layers cannot be replaced with
481 + # LoRA layers, leading to assertion error. The following check
482 + # aims to prevent this error
483 + if self.supports_mm and not isinstance(new_module,
484 + BaseLayerWithLoRA):
485 + continue
486 self.register_module(module_name, new_module)
487 self._register_packed_modules(module_name)
488 # All lora layers share the same punica_wrapper based on reference.
@@ -478,9 +501,10 @@ class LoRAModelManager(AdapterModelManager):
501 """Create zero-initialized LoRAModel for warmup."""
502 model = LoRAModel(lora_id, rank, {}, scaling_factor)
503 for module_name, module in self.model.named_modules():
481 - if not self._match_target_modules(module_name) or not isinstance(
482 - module, BaseLayerWithLoRA) or isinstance(
483 - module, LinearScalingRotaryEmbeddingWithLora):
504 + if (not self._match_target_modules(module_name)
505 + or not isinstance(module, BaseLayerWithLoRA)
506 + or isinstance(module, LinearScalingRotaryEmbeddingWithLora)
507 + or self._filter_unsupported_mm_module(module_name)):
508 continue
509 parts = module_name.split(".")
510 if module_name not in self.packed_modules:
@@ -541,6 +565,19 @@ class LoRAModelManager(AdapterModelManager):
565 module_name) or target_module == module_name
566 for target_module in self.supported_lora_modules)
567
568 + def _filter_unsupported_mm_module(self, module_name: str) -> bool:
569 + """
570 + Regarding multimodal models, vLLM currently only supports adding LoRA to
571 + language model. LoRA for other modules, such as the vision tower, will
572 + be filtered out.
573 + """
574 + if self.supports_mm:
575 + prefix = module_name.split(".")[0]
576 + module_mapping: MultiModelKeys = self.model.get_mm_mapping()
577 + return (prefix in module_mapping.connector
578 + or prefix in module_mapping.tower_model)
579 + return False
580 +
581 def _register_packed_modules(self, module_full_name: str) -> None:
582 parts = module_full_name.split(".")
583 module_name = parts[-1]
vllm/model_executor/models/minicpmv.py
+72 -22
@@ -36,7 +36,7 @@ from transformers import PretrainedConfig
36 from typing_extensions import NotRequired
37
38 from vllm.attention import AttentionMetadata
39 -from vllm.config import CacheConfig, MultiModalConfig
39 +from vllm.config import CacheConfig, LoRAConfig, MultiModalConfig
40 from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
41 from vllm.model_executor.layers.linear import ReplicatedLinear
42 from vllm.model_executor.layers.logits_processor import LogitsProcessor
@@ -50,7 +50,9 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
50 from vllm.model_executor.models.interfaces import SupportsMultiModal
51 from vllm.model_executor.models.llama import LlamaModel
52 from vllm.model_executor.models.minicpm import MiniCPMModel
53 +from vllm.model_executor.models.module_mapping import MultiModelKeys
54 from vllm.model_executor.models.qwen2 import Qwen2Model
55 +from vllm.model_executor.models.utils import LLMWrapper
56 from vllm.model_executor.sampling_metadata import SamplingMetadata
57 from vllm.multimodal import MULTIMODAL_REGISTRY
58 from vllm.multimodal.base import MultiModalInputs
@@ -59,10 +61,10 @@ from vllm.multimodal.utils import cached_get_tokenizer
61 from vllm.sequence import IntermediateTensors, SequenceData
62
63 from .idefics2_vision_model import Idefics2VisionTransformer
64 +from .interfaces import SupportsLoRA
65
66 _KEYS_TO_MODIFY_MAPPING = {
67 "llm.lm_head": "lm_head",
65 - "llm.model": "llm",
68 }
69
70
@@ -621,6 +623,14 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal):
623 default_weight_loader)
624 weight_loader(param, loaded_weight)
625
626 + def get_mm_mapping(self) -> MultiModelKeys:
627 + """
628 + Get the module prefix in multimodal models
629 + """
630 + return MultiModelKeys.from_string_field(language_model="llm",
631 + connector="resampler",
632 + tower_model="vpm")
633 +
634 def init_llm(
635 self,
636 config: PretrainedConfig,
@@ -669,9 +679,11 @@ class MiniCPMV2_0(MiniCPMVBaseModel):
679 cache_config: Optional[CacheConfig] = None,
680 quant_config: Optional[QuantizationConfig] = None,
681 ) -> nn.Module:
672 - return MiniCPMModel(config,
673 - cache_config=cache_config,
674 - quant_config=quant_config)
682 +
683 + return LLMWrapper(MiniCPMModel(config,
684 + cache_config=cache_config,
685 + quant_config=quant_config),
686 + name="model")
687
688 def init_vision_module(self) -> nn.Module:
689 # TODO :refactor this vision model
@@ -697,6 +709,9 @@ class MiniCPMV2_0(MiniCPMVBaseModel):
709
710 return model
711
712 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
713 + return self.model.embed_tokens(input_ids)
714 +
715 def init_resampler(self, embed_dim: int, vision_dim: int) -> nn.Module:
716 with set_default_torch_dtype(torch.float16):
717 resampler = Resampler2(
@@ -743,7 +758,34 @@ class MiniCPMV2_0(MiniCPMVBaseModel):
758 return "resampler" in name or "vpm" in name
759
760
746 -class MiniCPMV2_5(MiniCPMVBaseModel):
761 +class MiniCPMV2_5(MiniCPMVBaseModel, SupportsLoRA):
762 + packed_modules_mapping = {
763 + "qkv_proj": [
764 + "q_proj",
765 + "k_proj",
766 + "v_proj",
767 + ],
768 + "gate_up_proj": [
769 + "gate_proj",
770 + "up_proj",
771 + ],
772 + }
773 + # LoRA specific attributes
774 + supported_lora_modules = [
775 + # vision encoder
776 + "fc1",
777 + "fc2",
778 + "out_proj",
779 + # language model
780 + "qkv_proj", # same name with vision encoder
781 + "o_proj",
782 + "gate_up_proj",
783 + "down_proj",
784 + # resampler
785 + "kv_proj",
786 + ]
787 + embedding_modules = {}
788 + embedding_padding_modules = []
789
790 def __init__(
791 self,
@@ -751,6 +793,7 @@ class MiniCPMV2_5(MiniCPMVBaseModel):
793 multimodal_config: MultiModalConfig,
794 cache_config: Optional[CacheConfig] = None,
795 quant_config: Optional[QuantizationConfig] = None,
796 + lora_config: Optional[LoRAConfig] = None,
797 ):
798 super().__init__(config, multimodal_config, cache_config, quant_config)
799 assert self.version == (2, 5)
@@ -761,9 +804,10 @@ class MiniCPMV2_5(MiniCPMVBaseModel):
804 cache_config: Optional[CacheConfig] = None,
805 quant_config: Optional[QuantizationConfig] = None,
806 ) -> nn.Module:
764 - return LlamaModel(config,
765 - cache_config=cache_config,
766 - quant_config=quant_config)
807 + return LLMWrapper(LlamaModel(config,
808 + cache_config=cache_config,
809 + quant_config=quant_config),
810 + name="model")
811
812 def init_vision_module(self) -> nn.Module:
813 model = Idefics2VisionTransformer(self.config.vision_config)
@@ -843,9 +887,11 @@ class MiniCPMV2_6(MiniCPMVBaseModel):
887 cache_config: Optional[CacheConfig] = None,
888 quant_config: Optional[QuantizationConfig] = None,
889 ) -> nn.Module:
846 - return Qwen2Model(config,
847 - cache_config=cache_config,
848 - quant_config=quant_config)
890 +
891 + return LLMWrapper(Qwen2Model(config,
892 + cache_config=cache_config,
893 + quant_config=quant_config),
894 + name="model")
895
896 def init_vision_module(self) -> nn.Module:
897 # A custom version of SiglipVisionTransformer, won't work with TP
@@ -870,7 +916,6 @@ class MiniCPMV2_6(MiniCPMVBaseModel):
916 num_heads=embed_dim // 128,
917 kv_dim=vision_dim,
918 )
873 -
919 return resampler
920
921 def get_vision_embedding(
@@ -934,20 +979,25 @@ _SUPPORT_VERSION = {
979 @MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_minicpmv_image_tokens)
980 @INPUT_REGISTRY.register_dummy_data(dummy_data_for_minicpmv)
981 @INPUT_REGISTRY.register_input_processor(input_processor_for_minicpmv)
937 -class MiniCPMV(MiniCPMVBaseModel):
982 +class MiniCPMV(MiniCPMVBaseModel, SupportsLoRA):
983 """
984 Different versions of MiniCPMV use different visual encoders and LLMs,
985 which is not conducive to the current integration logic of LoRA and
986 bitsandbytes in vLLM. Therefore, it is necessary to separate them.
987 """
943 -
944 - def __new__(
945 - cls,
946 - config: PretrainedConfig,
947 - multimodal_config: MultiModalConfig,
948 - cache_config: Optional[CacheConfig] = None,
949 - quant_config: Optional[QuantizationConfig] = None,
950 - ):
988 + # Ensure that the LoRA support check passes when the class is not
989 + # initialized, but set all these attributes to empty.
990 + packed_modules_mapping = {}
991 + supported_lora_modules = []
992 + embedding_modules = {}
993 + embedding_padding_modules = []
994 +
995 + def __new__(cls,
996 + config: PretrainedConfig,
997 + multimodal_config: MultiModalConfig,
998 + cache_config: Optional[CacheConfig] = None,
999 + quant_config: Optional[QuantizationConfig] = None,
1000 + lora_config: Optional[LoRAConfig] = None):
1001 if not hasattr(config, "version"):
1002 if config.hidden_size == 2304 and config.query_num == 64:
1003 version = (2, 0)
vllm/model_executor/models/module_mapping.py new
+69
@@ -0,0 +1,69 @@
1 +# Adapted from
2 +# https://github.com/modelscope/ms-swift/blob/v2.4.2/swift/utils/module_mapping.py
3 +
4 +from dataclasses import dataclass, field
5 +from typing import List, Union
6 +
7 +
8 +@dataclass
9 +class ModelKeys:
10 + model_type: str = None
11 +
12 + module_list: str = None
13 +
14 + embedding: str = None
15 +
16 + mlp: str = None
17 +
18 + down_proj: str = None
19 +
20 + attention: str = None
21 +
22 + o_proj: str = None
23 +
24 + q_proj: str = None
25 +
26 + k_proj: str = None
27 +
28 + v_proj: str = None
29 +
30 + qkv_proj: str = None
31 +
32 + qk_proj: str = None
33 +
34 + qa_proj: str = None
35 +
36 + qb_proj: str = None
37 +
38 + kva_proj: str = None
39 +
40 + kvb_proj: str = None
41 +
42 + output: str = None
43 +
44 +
45 +@dataclass
46 +class MultiModelKeys(ModelKeys):
47 + language_model: List[str] = field(default_factory=list)
48 + connector: List[str] = field(default_factory=list)
49 + # vision tower and audio tower
50 + tower_model: List[str] = field(default_factory=list)
51 + generator: List[str] = field(default_factory=list)
52 +
53 + @staticmethod
54 + def from_string_field(language_model: Union[str, List[str]] = None,
55 + connector: Union[str, List[str]] = None,
56 + tower_model: Union[str, List[str]] = None,
57 + generator: Union[str, List[str]] = None,
58 + **kwargs) -> 'MultiModelKeys':
59 +
60 + def to_list(value):
61 + if value is None:
62 + return []
63 + return [value] if isinstance(value, str) else list(value)
64 +
65 + return MultiModelKeys(language_model=to_list(language_model),
66 + connector=to_list(connector),
67 + tower_model=to_list(tower_model),
68 + generator=to_list(generator),
69 + **kwargs)
vllm/model_executor/models/utils.py
+20 -2
@@ -1,7 +1,7 @@
1 import itertools
2 from collections import UserDict
3 -from typing import (Dict, Iterable, List, Literal, Optional, Protocol, Tuple,
4 - Union, overload)
3 +from typing import (Any, Dict, Iterable, List, Literal, Optional, Protocol,
4 + Tuple, Union, overload)
5
6 import torch
7 import torch.nn as nn
@@ -329,3 +329,21 @@ def make_empty_intermediate_tensors_factory(keys: List[str], hidden_size: int):
329 })
330
331 return make_empty_intermediate_tensors
332 +
333 +
334 +class LLMWrapper(nn.Module):
335 + """
336 + To align with the key names of LoRA trained with PEFT, we need to add an
337 + additional layer to the llm's implementation.
338 + """
339 +
340 + def __init__(self, llm: nn.Module, name: str) -> None:
341 + super().__init__()
342 + self.model_name = name
343 + setattr(self, name, llm)
344 +
345 + def forward(self, *args, **kwargs) -> Any:
346 + return getattr(self, self.model_name)(*args, **kwargs)
347 +
348 + def embed_tokens(self, *args, **kwargs) -> Any:
349 + return getattr(self, self.model_name).embed_tokens(*args, **kwargs)
vllm/worker/model_runner.py
+5 -3
@@ -1034,10 +1034,12 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1034 self.model_memory_usage / float(2**30))
1035
1036 if self.lora_config:
1037 - assert supports_lora(self.model), "Model does not support LoRA"
1038 - assert not supports_multimodal(
1037 + assert supports_lora(
1038 self.model
1040 - ), "To be tested: Multi-modal model with LoRA settings."
1039 + ), f"{self.model.__class__.__name__} does not support LoRA yet."
1040 + if supports_multimodal(self.model):
1041 + logger.warning("Regarding multimodal models, vLLM currently "
1042 + "only supports adding LoRA to language model.")
1043
1044 self.lora_manager = LRUCacheWorkerLoRAManager(
1045 self.scheduler_config.max_num_seqs,