@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a2a40bcd0d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[Model][LoRA]LoRA support added for MolmoForCausalLM (#11439)

Signed-off-by: Matthias Vogler <matthias.vogler@joesecurity.org> Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Co-authored-by: Matthias Vogler <matthias.vogler@joesecurity.org> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>

Matthias Vogler committed Dec 31, 2024 at 02:33 UTC a2a40bcd0d8275e19c46e9cc06ee994d8839b98d
2 files changed +43 -4
docs/source/models/supported_models.md
+1 -1
@@ -666,7 +666,7 @@ See [this page](#generative-models) for more information on how to use generativ
666 - Molmo
667 - T + I
668 - `allenai/Molmo-7B-D-0924`, `allenai/Molmo-72B-0924`, etc.
669 - -
669 + - ✅︎
670 - ✅︎
671 - ✅︎
672 * - `NVLM_D_Model`
vllm/model_executor/models/molmo.py
+42 -3
@@ -36,6 +36,7 @@ from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
36 from vllm.model_executor.layers.vocab_parallel_embedding import (
37 ParallelLMHead, VocabParallelEmbedding)
38 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
39 +from vllm.model_executor.models.module_mapping import MultiModelKeys
40 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
41 from vllm.multimodal.inputs import NestedTensors, PlaceholderRange
42 from vllm.multimodal.utils import cached_get_tokenizer
@@ -43,7 +44,7 @@ from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
44 SequenceData)
45 from vllm.transformers_utils.processor import get_processor
46
46 -from .interfaces import SupportsMultiModal, SupportsPP
47 +from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP
48 from .utils import (AutoWeightsLoader, WeightsMapper, is_pp_missing_parameter,
49 make_empty_intermediate_tensors_factory, make_layers,
50 maybe_prefix, merge_multimodal_embeddings)
@@ -1161,8 +1162,8 @@ def input_processor_for_molmo(ctx: InputContext, inputs: DecoderOnlyInputs):
1162 @MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_molmo_image_tokens)
1163 @INPUT_REGISTRY.register_dummy_data(dummy_data_for_molmo)
1164 @INPUT_REGISTRY.register_input_processor(input_processor_for_molmo)
1164 -class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
1165 -
1165 +class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP,
1166 + SupportsLoRA):
1167 hf_to_vllm_mapper = WeightsMapper(
1168 orig_to_new_substr={
1169 # vision backbone mapping
@@ -1191,6 +1192,32 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
1192 },
1193 )
1194
1195 + packed_modules_mapping = {
1196 + "qkv_proj": ["qkv_proj"],
1197 + "gate_up_proj": ["gate_up_proj"], # language model
1198 + "merged_linear": ["gate_proj", "up_proj"] # image_projector
1199 + }
1200 +
1201 + # LoRA specific attributes
1202 + supported_lora_modules = [
1203 + # language model
1204 + "qkv_proj",
1205 + "o_proj",
1206 + "gate_up_proj",
1207 + "down_proj", # same name with image_projector
1208 + # vision tower
1209 + "wq",
1210 + "wk",
1211 + "wv",
1212 + "wo",
1213 + "w1",
1214 + "w2",
1215 + # image_projector
1216 + "merged_linear",
1217 + ]
1218 + embedding_modules = {}
1219 + embedding_padding_modules = []
1220 +
1221 # BitandBytes specific attributes
1222 bitsandbytes_stacked_params_mapping = {
1223 "gate_proj": ("merged_linear", 0),
@@ -1202,8 +1229,10 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
1229 config = vllm_config.model_config.hf_config
1230 quant_config = vllm_config.quant_config
1231 multimodal_config = vllm_config.model_config.multimodal_config
1232 + lora_config = vllm_config.lora_config
1233 self.config = config
1234 self.multimodal_config = multimodal_config
1235 + self.lora_config = lora_config
1236
1237 vision_config = VisionBackboneConfig()
1238 self.vision_backbone = MolmoVisionBackbone(config, vision_config,
@@ -1377,6 +1406,16 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
1406 weights = _get_weights_with_merged_embedding(weights)
1407 return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
1408
1409 + def get_mm_mapping(self) -> MultiModelKeys:
1410 + """
1411 + Get the module prefix in multimodal models
1412 + """
1413 + return MultiModelKeys.from_string_field(
1414 + language_model="model",
1415 + connector="vision_backbone.image_projector",
1416 + tower_model="vision_backbone",
1417 + )
1418 +
1419
1420 def _get_weights_with_merged_embedding(
1421 weights: Iterable[Tuple[str, torch.Tensor]]