@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7a4df5f200
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Model][LoRA]LoRA support added for Qwen (#9622)

Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>

Jee Jee Li committed Oct 29, 2024 at 12:14 UTC 7a4df5f200f0943113dd2d9be49cbcae38ad10bb
2 files changed +101 -14
vllm/lora/models.py
+3 -3
@@ -578,10 +578,10 @@ class LoRAModelManager(AdapterModelManager):
578 be filtered out.
579 """
580 if self.supports_mm:
581 - prefix = module_name.split(".")[0]
581 module_mapping: MultiModelKeys = self.model.get_mm_mapping()
583 - return (prefix in module_mapping.connector
584 - or prefix in module_mapping.tower_model)
582 + prefix_lst = module_mapping.connector + module_mapping.tower_model
583 + return any(
584 + [module_name.startswith(prefix) for prefix in prefix_lst])
585 return False
586
587 def _register_packed_modules(self, module_full_name: str) -> None:
vllm/model_executor/models/qwen.py
+98 -11
@@ -20,7 +20,7 @@ from torchvision.transforms import InterpolationMode
20 from transformers import PretrainedConfig
21
22 from vllm.attention import Attention, AttentionMetadata
23 -from vllm.config import CacheConfig, MultiModalConfig
23 +from vllm.config import CacheConfig, LoRAConfig, MultiModalConfig
24 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
25 from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
26 token_inputs)
@@ -30,6 +30,7 @@ from vllm.model_executor.layers.layernorm import RMSNorm
30 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
31 MergedColumnParallelLinear,
32 QKVParallelLinear,
33 + ReplicatedLinear,
34 RowParallelLinear)
35 from vllm.model_executor.layers.logits_processor import LogitsProcessor
36 from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -39,6 +40,7 @@ from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
40 from vllm.model_executor.layers.vocab_parallel_embedding import (
41 ParallelLMHead, VocabParallelEmbedding)
42 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
43 +from vllm.model_executor.models.module_mapping import MultiModelKeys
44 from vllm.model_executor.sampling_metadata import SamplingMetadata
45 from vllm.multimodal import MULTIMODAL_REGISTRY
46 from vllm.multimodal.base import MultiModalInputs
@@ -46,7 +48,7 @@ from vllm.multimodal.utils import cached_get_tokenizer
48 from vllm.sequence import IntermediateTensors, SequenceData
49 from vllm.utils import is_list_of
50
49 -from .interfaces import SupportsMultiModal, SupportsPP
51 +from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP
52 from .utils import (flatten_bn, is_pp_missing_parameter,
53 make_empty_intermediate_tensors_factory, make_layers)
54
@@ -122,8 +124,8 @@ class VisualAttention(nn.Module):
124 # Strided linear layer.
125 assert self._qkv_same_embed_dim, \
126 'Visual Attention implementation only supports self-attention'
125 - self.in_proj = nn.Linear(embed_dim, 3 * embed_dim)
126 - self.out_proj = nn.Linear(embed_dim, embed_dim)
127 + self.in_proj = ReplicatedLinear(embed_dim, 3 * embed_dim)
128 + self.out_proj = ReplicatedLinear(embed_dim, embed_dim)
129 self.norm_factor = math.sqrt(self.hidden_size_per_attention_head)
130
131 def forward(
@@ -133,7 +135,7 @@ class VisualAttention(nn.Module):
135 ) -> torch.Tensor:
136 # query/key/value: [sq, b, h]
137 sq, b, _ = x.size()
136 - mixed_x_layer = self.in_proj(x)
138 + mixed_x_layer, _ = self.in_proj(x)
139
140 # [sq, b, (np * 3 * hn)] --> [sq, b, np, 3 * hn]
141 new_tensor_shape = mixed_x_layer.size()[:-1] + \
@@ -182,7 +184,7 @@ class VisualAttention(nn.Module):
184 (self.hidden_size_per_partition,)
185 context_layer = context_layer.view(*new_context_layer_shape)
186
185 - output = self.out_proj(context_layer)
187 + output, _ = self.out_proj(context_layer)
188
189 return output
190
@@ -860,11 +862,7 @@ def dummy_data_for_qwen(
862 return seq_data, mm_data
863
864
863 -@MULTIMODAL_REGISTRY.register_image_input_mapper(input_mapper_for_qwen)
864 -@MULTIMODAL_REGISTRY.register_max_image_tokens(MAX_QWEN_IMG_TOKENS)
865 -@INPUT_REGISTRY.register_dummy_data(dummy_data_for_qwen)
866 -@INPUT_REGISTRY.register_input_processor(input_processor_for_qwen)
867 -class QWenLMHeadModel(nn.Module, SupportsMultiModal, SupportsPP):
865 +class QWenBaseModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
866
867 def __init__(
868 self,
@@ -872,6 +870,7 @@ class QWenLMHeadModel(nn.Module, SupportsMultiModal, SupportsPP):
870 multimodal_config: MultiModalConfig,
871 cache_config: Optional[CacheConfig] = None,
872 quant_config: Optional[QuantizationConfig] = None,
873 + lora_config: Optional[LoRAConfig] = None,
874 ):
875 super().__init__()
876 self.config = config
@@ -990,3 +989,91 @@ class QWenLMHeadModel(nn.Module, SupportsMultiModal, SupportsPP):
989 weight_loader = getattr(param, "weight_loader",
990 default_weight_loader)
991 weight_loader(param, loaded_weight)
992 +
993 +
994 +class QWenLLM(QWenBaseModel):
995 + packed_modules_mapping = {
996 + "c_attn": ["c_attn"],
997 + "gate_up_proj": [
998 + "w2",
999 + "w1",
1000 + ],
1001 + }
1002 + # LoRA specific attributes
1003 + supported_lora_modules = [
1004 + "c_attn",
1005 + "gate_up_proj",
1006 + "c_proj",
1007 + ]
1008 +
1009 + embedding_modules = {}
1010 + embedding_padding_modules = []
1011 +
1012 +
1013 +class QWenVL(QWenBaseModel):
1014 + packed_modules_mapping = {
1015 + "c_attn": ["c_attn"],
1016 + "gate_up_proj": [
1017 + "w2",
1018 + "w1",
1019 + ],
1020 + }
1021 + # LoRA specific attributes
1022 + supported_lora_modules = [
1023 + "c_attn",
1024 + "gate_up_proj",
1025 + "c_proj",
1026 + # visual module
1027 + "out_proj",
1028 + "in_proj",
1029 + "c_fc",
1030 + # resampler
1031 + "kv_proj",
1032 + ]
1033 +
1034 + embedding_modules = {}
1035 + embedding_padding_modules = []
1036 +
1037 + def get_mm_mapping(self) -> MultiModelKeys:
1038 + """
1039 + Get the module prefix in multimodal models
1040 + """
1041 + return MultiModelKeys.from_string_field(
1042 + language_model="transformer.h",
1043 + connector="transformer.visual.attn_pool",
1044 + tower_model="transformer.visual.transformer")
1045 +
1046 +
1047 +@MULTIMODAL_REGISTRY.register_image_input_mapper(input_mapper_for_qwen)
1048 +@MULTIMODAL_REGISTRY.register_max_image_tokens(MAX_QWEN_IMG_TOKENS)
1049 +@INPUT_REGISTRY.register_dummy_data(dummy_data_for_qwen)
1050 +@INPUT_REGISTRY.register_input_processor(input_processor_for_qwen)
1051 +class QWenLMHeadModel(QWenBaseModel):
1052 + """
1053 + QWenLMHeadModel is not only applicable to LLM but also to VL, which is not
1054 + conducive to the current integration logic of LoRA in vLLM. Therefore, it
1055 + is necessary to separate them.
1056 + """
1057 + # Ensure that the LoRA support check passes when the class is not
1058 + # initialized, but set all these attributes to empty.
1059 + packed_modules_mapping = {}
1060 + supported_lora_modules = []
1061 + embedding_modules = {}
1062 + embedding_padding_modules = []
1063 +
1064 + def __new__(
1065 + cls,
1066 + config: PretrainedConfig,
1067 + multimodal_config: MultiModalConfig,
1068 + cache_config: Optional[CacheConfig] = None,
1069 + quant_config: Optional[QuantizationConfig] = None,
1070 + lora_config: Optional[LoRAConfig] = None,
1071 + ):
1072 + # Initialize VL
1073 + if hasattr(config, "visual"):
1074 + return QWenVL(config, multimodal_config, cache_config,
1075 + quant_config, lora_config)
1076 + # Initialize LLM
1077 + else:
1078 + return QWenLLM(config, multimodal_config, cache_config,
1079 + quant_config, lora_config)