@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d909acf9fe
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Model][LoRA]LoRA support added for idefics3 (#10281)

Signed-off-by: B-201 <Joy25810@foxmail.com>

B-201 committed Nov 13, 2024 at 17:25 UTC d909acf9fe17b7db42d7de61903c0058c8b9b344
2 files changed +47 -10
docs/source/models/supported_models.rst
+1 -1
@@ -450,7 +450,7 @@ Text Generation
450 - Idefics3
451 - T + I
452 - :code:`HuggingFaceM4/Idefics3-8B-Llama3` etc.
453 - -
453 + - ✅︎
454 -
455 * - :code:`InternVLChatModel`
456 - InternVL2
vllm/model_executor/models/idefics3.py
+46 -9
@@ -33,6 +33,7 @@ from vllm.model_executor.layers.linear import ReplicatedLinear
33 from vllm.model_executor.layers.logits_processor import LogitsProcessor
34 from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
35 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
36 +from vllm.model_executor.models.module_mapping import MultiModelKeys
37 from vllm.model_executor.sampling_metadata import SamplingMetadata
38 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
39 from vllm.multimodal.image import cached_get_image_processor
@@ -44,7 +45,7 @@ from vllm.utils import is_list_of
45 from .idefics2_vision_model import (
46 Idefics2VisionTransformer as Idefics3VisionTransformer)
47 # yapf: enable
47 -from .interfaces import SupportsMultiModal
48 +from .interfaces import SupportsLoRA, SupportsMultiModal
49 from .llama import LlamaModel
50 from .utils import (AutoWeightsLoader, flatten_bn, maybe_prefix,
51 merge_multimodal_embeddings)
@@ -58,8 +59,6 @@ class Idefics3ImagePixelInputs(TypedDict):
59 """
60 Shape: `(batch_size * num_images, num_channels, height, width)`
61 """
61 - rows: List[int]
62 - cols: List[int]
62 pixel_attention_mask: Optional[torch.BoolTensor]
63
64
@@ -356,8 +355,15 @@ def dummy_data_for_idefics3(
355 image_seq_len = processor.image_seq_len
356 max_llm_image_tokens = max_num_image_patches * image_seq_len * num_images
357
358 + if seq_len - max_llm_image_tokens < 0:
359 + raise RuntimeError(
360 + f"Idefics3 cannot process {num_images} images in a prompt, "
361 + "please increase max_model_len or reduce image limit by "
362 + "--limit-mm-per-prompt.")
363 +
364 seq_data = SequenceData.from_prompt_token_counts(
360 - (hf_config.image_token_id, max_llm_image_tokens), (0, seq_len))
365 + (hf_config.image_token_id, max_llm_image_tokens),
366 + (0, seq_len - max_llm_image_tokens))
367
368 width = height = hf_config.vision_config.image_size
369 image = Image.new("RGB", (width, height), color=0)
@@ -463,8 +469,6 @@ class Idefics3Model(nn.Module):
469 self, **kwargs: object) -> Optional[ImageInputs]:
470 pixel_values = kwargs.pop("pixel_values", None)
471 image_embeds = kwargs.pop("image_embeds", None)
466 - rows = kwargs.pop("rows", None)
467 - cols = kwargs.pop("cols", None)
472 pixel_attention_mask = kwargs.pop("pixel_attention_mask", None)
473
474 if pixel_values is None and image_embeds is None:
@@ -489,8 +493,6 @@ class Idefics3Model(nn.Module):
493 data=self._validate_pixel_values(
494 flatten_bn(pixel_values,
495 concat=True)),
492 - rows=rows,
493 - cols=cols,
496 pixel_attention_mask=flatten_bn(
497 pixel_attention_mask,
498 concat=True))
@@ -610,7 +612,33 @@ class Idefics3Model(nn.Module):
612 @MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_idefics3_image_tokens)
613 @INPUT_REGISTRY.register_dummy_data(dummy_data_for_idefics3)
614 @INPUT_REGISTRY.register_input_processor(input_processor_for_idefics3)
613 -class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal):
615 +class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal,
616 + SupportsLoRA):
617 + packed_modules_mapping = {
618 + "qkv_proj": [
619 + "q_proj",
620 + "k_proj",
621 + "v_proj",
622 + ],
623 + "gate_up_proj": [
624 + "gate_proj",
625 + "up_proj",
626 + ],
627 + }
628 + # LoRA specific attributes
629 + supported_lora_modules = [
630 + # vision_model
631 + "fc1",
632 + "fc2",
633 + "out_proj",
634 + # text_model
635 + "qkv_proj", # same name with vision encoder
636 + "o_proj",
637 + "gate_up_proj",
638 + "down_proj",
639 + ]
640 + embedding_modules = {}
641 + embedding_padding_modules = []
642
643 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
644 super().__init__()
@@ -672,3 +700,12 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal):
700 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
701 loader = AutoWeightsLoader(self)
702 loader.load_weights(weights)
703 +
704 + def get_mm_mapping(self) -> MultiModelKeys:
705 + """
706 + Get the module prefix in multimodal models
707 + """
708 + return MultiModelKeys.from_string_field(
709 + language_model="model.text_model",
710 + connector="model.connector",
711 + tower_model="model.vision_model")