A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 08:30 UTC
[LoRA] Add LoRA support for InternVL (#18842)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
May 29, 2025 at 16:46 UTC
34d6c447c4b9cf2dd986df7788684cc40662e5e7
1 file changed
+23
-2
vllm/model_executor/models/internvl.py
+23
-2
@@ -22,6 +22,7 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
22
from vllm.model_executor.layers.quantization.awq import AWQConfig
23
from vllm.model_executor.models.intern_vit import (InternVisionModel,
24
InternVisionPatchModel)
25
+from vllm.model_executor.models.module_mapping import MultiModelKeys
26
from vllm.model_executor.sampling_metadata import SamplingMetadata
27
from vllm.multimodal import MULTIMODAL_REGISTRY
28
from vllm.multimodal.image import convert_image_mode
@@ -36,7 +37,8 @@ from vllm.multimodal.profiling import BaseDummyInputsBuilder
37
from vllm.sequence import IntermediateTensors
38
from vllm.transformers_utils.tokenizer import AnyTokenizer
39
39
-from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
40
+from .interfaces import (MultiModalEmbeddings, SupportsLoRA,
41
+ SupportsMultiModal, SupportsPP)
42
from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
43
maybe_prefix, merge_multimodal_embeddings)
44
@@ -1014,7 +1016,17 @@ class InternVLMultiModalProcessor(
1016
InternVLMultiModalProcessor,
1017
info=InternVLProcessingInfo,
1018
dummy_inputs=InternVLDummyInputsBuilder)
1017
-class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
1019
+class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP,
1020
+ SupportsLoRA):
1021
+
1022
+ packed_modules_mapping = {
1023
+ "wqkv": ["wqkv"],
1024
+ "qkv": ["qkv"],
1025
+ "gate_up_proj": [
1026
+ "w1",
1027
+ "w3",
1028
+ ],
1029
+ }
1030
1031
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
1032
super().__init__()
@@ -1403,3 +1415,12 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
1415
]
1416
loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
1417
return loader.load_weights(weights)
1418
+
1419
+ def get_mm_mapping(self) -> MultiModelKeys:
1420
+ """
1421
+ Get the module prefix in multimodal models
1422
+ """
1423
+ return MultiModelKeys.from_string_field(
1424
+ language_model="language_model",
1425
+ connector="mlp1",
1426
+ tower_model="vision_model")