A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[Model][LoRA]LoRA support added for idefics3 (#10281)
Signed-off-by: B-201 <Joy25810@foxmail.com>
B-201 committed
Nov 13, 2024 at 17:25 UTC
d909acf9fe17b7db42d7de61903c0058c8b9b344
2 files changed
+47
-10
docs/source/models/supported_models.rst
+1
-1
@@ -450,7 +450,7 @@ Text Generation
450
- Idefics3
451
- T + I
452
- :code:`HuggingFaceM4/Idefics3-8B-Llama3` etc.
453
- -
453
+ - ✅︎
454
-
455
* - :code:`InternVLChatModel`
456
- InternVL2
vllm/model_executor/models/idefics3.py
+46
-9
@@ -33,6 +33,7 @@ from vllm.model_executor.layers.linear import ReplicatedLinear
33
from vllm.model_executor.layers.logits_processor import LogitsProcessor
34
from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
35
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
36
+from vllm.model_executor.models.module_mapping import MultiModelKeys
37
from vllm.model_executor.sampling_metadata import SamplingMetadata
38
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
39
from vllm.multimodal.image import cached_get_image_processor
@@ -44,7 +45,7 @@ from vllm.utils import is_list_of
45
from .idefics2_vision_model import (
46
Idefics2VisionTransformer as Idefics3VisionTransformer)
47
# yapf: enable
47
-from .interfaces import SupportsMultiModal
48
+from .interfaces import SupportsLoRA, SupportsMultiModal
49
from .llama import LlamaModel
50
from .utils import (AutoWeightsLoader, flatten_bn, maybe_prefix,
51
merge_multimodal_embeddings)
@@ -58,8 +59,6 @@ class Idefics3ImagePixelInputs(TypedDict):
59
"""
60
Shape: `(batch_size * num_images, num_channels, height, width)`
61
"""
61
- rows: List[int]
62
- cols: List[int]
62
pixel_attention_mask: Optional[torch.BoolTensor]
63
64
@@ -356,8 +355,15 @@ def dummy_data_for_idefics3(
355
image_seq_len = processor.image_seq_len
356
max_llm_image_tokens = max_num_image_patches * image_seq_len * num_images
357
358
+ if seq_len - max_llm_image_tokens < 0:
359
+ raise RuntimeError(
360
+ f"Idefics3 cannot process {num_images} images in a prompt, "
361
+ "please increase max_model_len or reduce image limit by "
362
+ "--limit-mm-per-prompt.")
363
+
364
seq_data = SequenceData.from_prompt_token_counts(
360
- (hf_config.image_token_id, max_llm_image_tokens), (0, seq_len))
365
+ (hf_config.image_token_id, max_llm_image_tokens),
366
+ (0, seq_len - max_llm_image_tokens))
367
368
width = height = hf_config.vision_config.image_size
369
image = Image.new("RGB", (width, height), color=0)
@@ -463,8 +469,6 @@ class Idefics3Model(nn.Module):
469
self, **kwargs: object) -> Optional[ImageInputs]:
470
pixel_values = kwargs.pop("pixel_values", None)
471
image_embeds = kwargs.pop("image_embeds", None)
466
- rows = kwargs.pop("rows", None)
467
- cols = kwargs.pop("cols", None)
472
pixel_attention_mask = kwargs.pop("pixel_attention_mask", None)
473
474
if pixel_values is None and image_embeds is None:
@@ -489,8 +493,6 @@ class Idefics3Model(nn.Module):
493
data=self._validate_pixel_values(
494
flatten_bn(pixel_values,
495
concat=True)),
492
- rows=rows,
493
- cols=cols,
496
pixel_attention_mask=flatten_bn(
497
pixel_attention_mask,
498
concat=True))
@@ -610,7 +612,33 @@ class Idefics3Model(nn.Module):
612
@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_idefics3_image_tokens)
613
@INPUT_REGISTRY.register_dummy_data(dummy_data_for_idefics3)
614
@INPUT_REGISTRY.register_input_processor(input_processor_for_idefics3)
613
-class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal):
615
+class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal,
616
+ SupportsLoRA):
617
+ packed_modules_mapping = {
618
+ "qkv_proj": [
619
+ "q_proj",
620
+ "k_proj",
621
+ "v_proj",
622
+ ],
623
+ "gate_up_proj": [
624
+ "gate_proj",
625
+ "up_proj",
626
+ ],
627
+ }
628
+ # LoRA specific attributes
629
+ supported_lora_modules = [
630
+ # vision_model
631
+ "fc1",
632
+ "fc2",
633
+ "out_proj",
634
+ # text_model
635
+ "qkv_proj", # same name with vision encoder
636
+ "o_proj",
637
+ "gate_up_proj",
638
+ "down_proj",
639
+ ]
640
+ embedding_modules = {}
641
+ embedding_padding_modules = []
642
643
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
644
super().__init__()
@@ -672,3 +700,12 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal):
700
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
701
loader = AutoWeightsLoader(self)
702
loader.load_weights(weights)
703
+
704
+ def get_mm_mapping(self) -> MultiModelKeys:
705
+ """
706
+ Get the module prefix in multimodal models
707
+ """
708
+ return MultiModelKeys.from_string_field(
709
+ language_model="model.text_model",
710
+ connector="model.connector",
711
+ tower_model="model.vision_model")