[Bugfix][LoRA] Add multimodal module mapping for Muse-Glimmer (#53513)
Signed-off-by: linitra24 <Joy25810@foxmail.com>
linitra24 committed
Aug 24, 2026 at 13:51 UTC
8c2bbe00d58a930c6c09a80495728b26b79d9200
1 file changed
+11
vllm/model_executor/models/muse_glimmer.py
+11
@@ -69,6 +69,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
69
ParallelLMHead,
70
VocabParallelEmbedding,
71
)
72
+from vllm.model_executor.models.module_mapping import MultiModelKeys
73
from vllm.multimodal import MULTIMODAL_REGISTRY
74
from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems
75
from vllm.multimodal.parse import ImageSize, MultiModalDataItems
@@ -1646,3 +1647,13 @@ class MuseGlimmerForCausalLM(
1647
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
1648
loader = AutoWeightsLoader(self)
1649
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
1650
+
1651
+ def get_mm_mapping(self) -> MultiModelKeys:
1652
+ """
1653
+ Get the module prefix in multimodal models
1654
+ """
1655
+ return MultiModelKeys.from_string_field(
1656
+ language_model="model",
1657
+ connector=["vision_adapter.", "vision_projection."],
1658
+ tower_model="vision_encoder.",
1659
+ )