A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[Model][LoRA]LoRA support added for Qwen2VLForConditionalGeneration (#10022)
Signed-off-by: ericperfect <ericperfectttt@gmail.com>
Eric committed
Nov 6, 2024 at 22:13 UTC
406d4cc480bbc01d41f34b83102548bae229671a
2 files changed
+29
-5
docs/source/models/supported_models.rst
+1
-1
@@ -540,7 +540,7 @@ Text Generation
540
- Qwen2-VL
541
- T + I\ :sup:`E+` + V\ :sup:`+`
542
- :code:`Qwen/Qwen2-VL-2B-Instruct`, :code:`Qwen/Qwen2-VL-7B-Instruct`, :code:`Qwen/Qwen2-VL-72B-Instruct`, etc.
543
- -
543
+ - ✅︎
544
- ✅︎
545
* - :code:`UltravoxModel`
546
- Ultravox
vllm/model_executor/models/qwen2_vl.py
+28
-4
@@ -40,7 +40,7 @@ from transformers.models.qwen2_vl.image_processing_qwen2_vl import (
40
41
from vllm.attention import AttentionMetadata
42
from vllm.attention.selector import _Backend
43
-from vllm.config import CacheConfig, MultiModalConfig
43
+from vllm.config import CacheConfig, LoRAConfig, MultiModalConfig
44
from vllm.distributed import get_pp_group, parallel_state
45
from vllm.distributed import utils as dist_utils
46
from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
@@ -65,7 +65,7 @@ from vllm.sequence import IntermediateTensors, SequenceData
65
from vllm.transformers_utils.config import uses_mrope
66
from vllm.transformers_utils.processor import cached_get_processor
67
68
-from .interfaces import SupportsMultiModal, SupportsPP
68
+from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP
69
from .utils import (PPMissingLayer, get_vit_attn_backend,
70
is_pp_missing_parameter,
71
make_empty_intermediate_tensors_factory)
@@ -927,13 +927,37 @@ def input_processor_for_qwen2_vl(
927
@INPUT_REGISTRY.register_dummy_data(dummy_data_for_qwen2_vl)
928
@INPUT_REGISTRY.register_input_processor(input_processor_for_qwen2_vl)
929
class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
930
- SupportsPP):
930
+ SupportsLoRA, SupportsPP):
931
+ packed_modules_mapping = {
932
+ "qkv_proj": [
933
+ "q_proj",
934
+ "k_proj",
935
+ "v_proj",
936
+ ],
937
+ "gate_up_proj": [
938
+ "gate_proj",
939
+ "up_proj",
940
+ ],
941
+ }
942
+
943
+ # LoRA specific attributes
944
+ # TODO Support LoRA for the visual encoder in the future.
945
+ supported_lora_modules = [
946
+ "qkv_proj",
947
+ "o_proj",
948
+ "gate_up_proj",
949
+ "down_proj",
950
+ ]
951
+ embedding_modules = {}
952
+ embedding_padding_modules = []
953
954
def __init__(self,
955
config: Qwen2VLConfig,
956
multimodal_config: MultiModalConfig,
957
cache_config: Optional[CacheConfig] = None,
936
- quant_config: Optional[QuantizationConfig] = None) -> None:
958
+ quant_config: Optional[QuantizationConfig] = None,
959
+ lora_config: Optional[LoRAConfig] = None) -> None:
960
+
961
super().__init__()
962
963
assert not cache_config.enable_prefix_caching, \