A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[Model][LoRA]LoRA support added for MolmoForCausalLM (#11439)
Signed-off-by: Matthias Vogler <matthias.vogler@joesecurity.org> Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Co-authored-by: Matthias Vogler <matthias.vogler@joesecurity.org> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
Matthias Vogler committed
Dec 31, 2024 at 02:33 UTC
a2a40bcd0d8275e19c46e9cc06ee994d8839b98d
2 files changed
+43
-4
docs/source/models/supported_models.md
+1
-1
@@ -666,7 +666,7 @@ See [this page](#generative-models) for more information on how to use generativ
666
- Molmo
667
- T + I
668
- `allenai/Molmo-7B-D-0924`, `allenai/Molmo-72B-0924`, etc.
669
- -
669
+ - ✅︎
670
- ✅︎
671
- ✅︎
672
* - `NVLM_D_Model`
vllm/model_executor/models/molmo.py
+42
-3
@@ -36,6 +36,7 @@ from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
36
from vllm.model_executor.layers.vocab_parallel_embedding import (
37
ParallelLMHead, VocabParallelEmbedding)
38
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
39
+from vllm.model_executor.models.module_mapping import MultiModelKeys
40
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
41
from vllm.multimodal.inputs import NestedTensors, PlaceholderRange
42
from vllm.multimodal.utils import cached_get_tokenizer
@@ -43,7 +44,7 @@ from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
44
SequenceData)
45
from vllm.transformers_utils.processor import get_processor
46
46
-from .interfaces import SupportsMultiModal, SupportsPP
47
+from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP
48
from .utils import (AutoWeightsLoader, WeightsMapper, is_pp_missing_parameter,
49
make_empty_intermediate_tensors_factory, make_layers,
50
maybe_prefix, merge_multimodal_embeddings)
@@ -1161,8 +1162,8 @@ def input_processor_for_molmo(ctx: InputContext, inputs: DecoderOnlyInputs):
1162
@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_molmo_image_tokens)
1163
@INPUT_REGISTRY.register_dummy_data(dummy_data_for_molmo)
1164
@INPUT_REGISTRY.register_input_processor(input_processor_for_molmo)
1164
-class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
1165
-
1165
+class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP,
1166
+ SupportsLoRA):
1167
hf_to_vllm_mapper = WeightsMapper(
1168
orig_to_new_substr={
1169
# vision backbone mapping
@@ -1191,6 +1192,32 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
1192
},
1193
)
1194
1195
+ packed_modules_mapping = {
1196
+ "qkv_proj": ["qkv_proj"],
1197
+ "gate_up_proj": ["gate_up_proj"], # language model
1198
+ "merged_linear": ["gate_proj", "up_proj"] # image_projector
1199
+ }
1200
+
1201
+ # LoRA specific attributes
1202
+ supported_lora_modules = [
1203
+ # language model
1204
+ "qkv_proj",
1205
+ "o_proj",
1206
+ "gate_up_proj",
1207
+ "down_proj", # same name with image_projector
1208
+ # vision tower
1209
+ "wq",
1210
+ "wk",
1211
+ "wv",
1212
+ "wo",
1213
+ "w1",
1214
+ "w2",
1215
+ # image_projector
1216
+ "merged_linear",
1217
+ ]
1218
+ embedding_modules = {}
1219
+ embedding_padding_modules = []
1220
+
1221
# BitandBytes specific attributes
1222
bitsandbytes_stacked_params_mapping = {
1223
"gate_proj": ("merged_linear", 0),
@@ -1202,8 +1229,10 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
1229
config = vllm_config.model_config.hf_config
1230
quant_config = vllm_config.quant_config
1231
multimodal_config = vllm_config.model_config.multimodal_config
1232
+ lora_config = vllm_config.lora_config
1233
self.config = config
1234
self.multimodal_config = multimodal_config
1235
+ self.lora_config = lora_config
1236
1237
vision_config = VisionBackboneConfig()
1238
self.vision_backbone = MolmoVisionBackbone(config, vision_config,
@@ -1377,6 +1406,16 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
1406
weights = _get_weights_with_merged_embedding(weights)
1407
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
1408
1409
+ def get_mm_mapping(self) -> MultiModelKeys:
1410
+ """
1411
+ Get the module prefix in multimodal models
1412
+ """
1413
+ return MultiModelKeys.from_string_field(
1414
+ language_model="model",
1415
+ connector="vision_backbone.image_projector",
1416
+ tower_model="vision_backbone",
1417
+ )
1418
+
1419
1420
def _get_weights_with_merged_embedding(
1421
weights: Iterable[Tuple[str, torch.Tensor]]