A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 19:00 UTC
[Model] Enable LoRA support for BLIP2 (#31620)
Signed-off-by: Qiping Pan <panqiping@outlook.com>
Qiping Pan committed
Jan 5, 2026 at 00:02 UTC
a2ad15c070199846a1bbaa72d81605992fbdd89e
2 files changed
+47
-4
docs/models/supported_models.md
+1
-1
@@ -673,7 +673,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
673
| `AyaVisionForConditionalGeneration` | Aya Vision | T + I<sup>+</sup> | `CohereLabs/aya-vision-8b`, `CohereLabs/aya-vision-32b`, etc. | | ✅︎ |
674
| `BagelForConditionalGeneration` | BAGEL | T + I<sup>+</sup> | `ByteDance-Seed/BAGEL-7B-MoT` | ✅︎ | ✅︎ |
675
| `BeeForConditionalGeneration` | Bee-8B | T + I<sup>E+</sup> | `Open-Bee/Bee-8B-RL`, `Open-Bee/Bee-8B-SFT` | | ✅︎ |
676
-| `Blip2ForConditionalGeneration` | BLIP-2 | T + I<sup>E</sup> | `Salesforce/blip2-opt-2.7b`, `Salesforce/blip2-opt-6.7b`, etc. | | ✅︎ |
676
+| `Blip2ForConditionalGeneration` | BLIP-2 | T + I<sup>E</sup> | `Salesforce/blip2-opt-2.7b`, `Salesforce/blip2-opt-6.7b`, etc. | ✅︎ | ✅︎ |
677
| `ChameleonForConditionalGeneration` | Chameleon | T + I | `facebook/chameleon-7b`, etc. | | ✅︎ |
678
| `Cohere2VisionForConditionalGeneration` | Command A Vision | T + I<sup>+</sup> | `CohereLabs/command-a-vision-07-2025`, etc. | | ✅︎ |
679
| `DeepseekVLV2ForCausalLM`<sup>^</sup> | DeepSeek-VL2 | T + I<sup>+</sup> | `deepseek-ai/deepseek-vl2-tiny`, `deepseek-ai/deepseek-vl2-small`, `deepseek-ai/deepseek-vl2`, etc. | | ✅︎ |
vllm/model_executor/models/blip2.py
+46
-3
@@ -35,13 +35,15 @@ from vllm.multimodal.profiling import BaseDummyInputsBuilder
35
from vllm.sequence import IntermediateTensors
36
from vllm.utils.tensor_schema import TensorSchema, TensorShape
37
38
-from .blip import BlipVisionModel
38
+from .blip import BlipVisionModel, get_blip_num_patches
39
from .interfaces import (
40
MultiModalEmbeddings,
41
+ SupportsLoRA,
42
SupportsMultiModal,
43
SupportsPP,
44
SupportsQuant,
45
)
46
+from .module_mapping import MultiModelKeys
47
from .utils import AutoWeightsLoader, init_vllm_registered_model, maybe_prefix
48
49
@@ -521,7 +523,7 @@ class Blip2MultiModalProcessor(BaseMultiModalProcessor[Blip2ProcessingInfo]):
523
dummy_inputs=Blip2DummyInputsBuilder,
524
)
525
class Blip2ForConditionalGeneration(
524
- nn.Module, SupportsMultiModal, SupportsPP, SupportsQuant
526
+ nn.Module, SupportsLoRA, SupportsMultiModal, SupportsPP, SupportsQuant
527
):
528
@classmethod
529
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
@@ -538,9 +540,17 @@ class Blip2ForConditionalGeneration(
540
multimodal_config = vllm_config.model_config.multimodal_config
541
self.config = config
542
self.multimodal_config = multimodal_config
543
+ vision_config = config.vision_config
544
+ self._vision_tokens_per_image = (
545
+ get_blip_num_patches(
546
+ image_size=vision_config.image_size,
547
+ patch_size=vision_config.patch_size,
548
+ )
549
+ + 1 # include class token
550
+ )
551
552
# TODO: Optionally initializes this for supporting embeddings.
543
- self.vision_model = BlipVisionModel(config.vision_config, quant_config)
553
+ self.vision_model = BlipVisionModel(vision_config, quant_config)
554
555
self.query_tokens = nn.Parameter(
556
torch.zeros(1, config.num_query_tokens, config.qformer_config.hidden_size)
@@ -691,3 +701,36 @@ class Blip2ForConditionalGeneration(
701
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
702
loader = AutoWeightsLoader(self)
703
return loader.load_weights(weights)
704
+
705
+ def get_mm_mapping(self) -> MultiModelKeys:
706
+ return MultiModelKeys.from_string_field(
707
+ language_model="language_model",
708
+ connector=["qformer", "language_projection"],
709
+ tower_model="vision_model",
710
+ )
711
+
712
+ def get_num_mm_encoder_tokens(
713
+ self,
714
+ num_image_tokens: int,
715
+ ) -> int:
716
+ if num_image_tokens <= 0:
717
+ return 0
718
+ assert num_image_tokens % self.config.num_query_tokens == 0, (
719
+ "The number of image tokens must be a multiple of "
720
+ "the number of query tokens."
721
+ )
722
+ num_images = num_image_tokens / self.config.num_query_tokens
723
+ return num_images * self._vision_tokens_per_image
724
+
725
+ def get_num_mm_connector_tokens(
726
+ self,
727
+ num_vision_tokens: int,
728
+ ) -> int:
729
+ if num_vision_tokens <= 0:
730
+ return 0
731
+ assert num_vision_tokens % self._vision_tokens_per_image == 0, (
732
+ "The number of vision tokens must be a multiple of "
733
+ "the number of tokens per image."
734
+ )
735
+ num_images = num_vision_tokens / self._vision_tokens_per_image
736
+ return num_images * self.config.num_query_tokens