@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 406d4cc480
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[Model][LoRA]LoRA support added for Qwen2VLForConditionalGeneration (#10022)

Signed-off-by: ericperfect <ericperfectttt@gmail.com>

Eric committed Nov 6, 2024 at 22:13 UTC 406d4cc480bbc01d41f34b83102548bae229671a
2 files changed +29 -5
docs/source/models/supported_models.rst
+1 -1
@@ -540,7 +540,7 @@ Text Generation
540 - Qwen2-VL
541 - T + I\ :sup:`E+` + V\ :sup:`+`
542 - :code:`Qwen/Qwen2-VL-2B-Instruct`, :code:`Qwen/Qwen2-VL-7B-Instruct`, :code:`Qwen/Qwen2-VL-72B-Instruct`, etc.
543 - -
543 + - ✅︎
544 - ✅︎
545 * - :code:`UltravoxModel`
546 - Ultravox
vllm/model_executor/models/qwen2_vl.py
+28 -4
@@ -40,7 +40,7 @@ from transformers.models.qwen2_vl.image_processing_qwen2_vl import (
40
41 from vllm.attention import AttentionMetadata
42 from vllm.attention.selector import _Backend
43 -from vllm.config import CacheConfig, MultiModalConfig
43 +from vllm.config import CacheConfig, LoRAConfig, MultiModalConfig
44 from vllm.distributed import get_pp_group, parallel_state
45 from vllm.distributed import utils as dist_utils
46 from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
@@ -65,7 +65,7 @@ from vllm.sequence import IntermediateTensors, SequenceData
65 from vllm.transformers_utils.config import uses_mrope
66 from vllm.transformers_utils.processor import cached_get_processor
67
68 -from .interfaces import SupportsMultiModal, SupportsPP
68 +from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP
69 from .utils import (PPMissingLayer, get_vit_attn_backend,
70 is_pp_missing_parameter,
71 make_empty_intermediate_tensors_factory)
@@ -927,13 +927,37 @@ def input_processor_for_qwen2_vl(
927 @INPUT_REGISTRY.register_dummy_data(dummy_data_for_qwen2_vl)
928 @INPUT_REGISTRY.register_input_processor(input_processor_for_qwen2_vl)
929 class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
930 - SupportsPP):
930 + SupportsLoRA, SupportsPP):
931 + packed_modules_mapping = {
932 + "qkv_proj": [
933 + "q_proj",
934 + "k_proj",
935 + "v_proj",
936 + ],
937 + "gate_up_proj": [
938 + "gate_proj",
939 + "up_proj",
940 + ],
941 + }
942 +
943 + # LoRA specific attributes
944 + # TODO Support LoRA for the visual encoder in the future.
945 + supported_lora_modules = [
946 + "qkv_proj",
947 + "o_proj",
948 + "gate_up_proj",
949 + "down_proj",
950 + ]
951 + embedding_modules = {}
952 + embedding_padding_modules = []
953
954 def __init__(self,
955 config: Qwen2VLConfig,
956 multimodal_config: MultiModalConfig,
957 cache_config: Optional[CacheConfig] = None,
936 - quant_config: Optional[QuantizationConfig] = None) -> None:
958 + quant_config: Optional[QuantizationConfig] = None,
959 + lora_config: Optional[LoRAConfig] = None) -> None:
960 +
961 super().__init__()
962
963 assert not cache_config.enable_prefix_caching, \