A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 20:15 UTC
[Model] Enable LoRA support for tower and connector in LLaVA (#31513)
Signed-off-by: Jay Hemnani <jayhemnani9910@gmail.com> Co-authored-by: Jay Hemnani <jayhemnani9910@gmail.com> Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
Jay Hemnani committed
Jan 1, 2026 at 19:32 UTC
5ac55eb30ff1c15918af9471b277248f9afdbf3b
2 files changed
+37
-3
docs/models/supported_models.md
+1
-1
@@ -699,7 +699,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
699
| `LightOnOCRForConditionalGeneration` | LightOnOCR-1B | T + I<sup>+</sup> | `lightonai/LightOnOCR-1B`, etc | ✅︎ | ✅︎ |
700
| `Llama4ForConditionalGeneration` | Llama 4 | T + I<sup>+</sup> | `meta-llama/Llama-4-Scout-17B-16E-Instruct`, `meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8`, `meta-llama/Llama-4-Maverick-17B-128E-Instruct`, etc. | ✅︎ | ✅︎ |
701
| `Llama_Nemotron_Nano_VL` | Llama Nemotron Nano VL | T + I<sup>E+</sup> | `nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1` | ✅︎ | ✅︎ |
702
-| `LlavaForConditionalGeneration` | LLaVA-1.5, Pixtral (HF Transformers) | T + I<sup>E+</sup> | `llava-hf/llava-1.5-7b-hf`, `TIGER-Lab/Mantis-8B-siglip-llama3` (see note), `mistral-community/pixtral-12b`, etc. | | ✅︎ |
702
+| `LlavaForConditionalGeneration` | LLaVA-1.5, Pixtral (HF Transformers) | T + I<sup>E+</sup> | `llava-hf/llava-1.5-7b-hf`, `TIGER-Lab/Mantis-8B-siglip-llama3` (see note), `mistral-community/pixtral-12b`, etc. | ✅︎ | ✅︎ |
703
| `LlavaNextForConditionalGeneration` | LLaVA-NeXT | T + I<sup>E+</sup> | `llava-hf/llava-v1.6-mistral-7b-hf`, `llava-hf/llava-v1.6-vicuna-7b-hf`, etc. | | ✅︎ |
704
| `LlavaNextVideoForConditionalGeneration` | LLaVA-NeXT-Video | T + V | `llava-hf/LLaVA-NeXT-Video-7B-hf`, etc. | | ✅︎ |
705
| `LlavaOnevisionForConditionalGeneration` | LLaVA-Onevision | T + I<sup>+</sup> + V<sup>+</sup> | `llava-hf/llava-onevision-qwen2-7b-ov-hf`, `llava-hf/llava-onevision-qwen2-0.5b-ov-hf`, etc. | | ✅︎ |
vllm/model_executor/models/llava.py
+36
-2
@@ -51,7 +51,13 @@ from vllm.sequence import IntermediateTensors
51
from vllm.utils.tensor_schema import TensorSchema, TensorShape
52
53
from .clip import CLIPVisionModel
54
-from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
54
+from .interfaces import (
55
+ MultiModalEmbeddings,
56
+ SupportsLoRA,
57
+ SupportsMultiModal,
58
+ SupportsPP,
59
+)
60
+from .module_mapping import MultiModelKeys
61
from .pixtral import PixtralHFEncoderInfo, PixtralHFVisionModel
62
from .siglip import SiglipVisionModel
63
from .utils import (
@@ -505,7 +511,9 @@ def init_vision_tower_for_llava(
511
info=_build_llava_or_pixtral_hf_info,
512
dummy_inputs=LlavaDummyInputsBuilder,
513
)
508
-class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
514
+class LlavaForConditionalGeneration(
515
+ nn.Module, SupportsLoRA, SupportsMultiModal, SupportsPP
516
+):
517
packed_modules_mapping = {
518
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
519
"gate_up_proj": ["gate_proj", "up_proj"],
@@ -734,6 +742,32 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
742
loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
743
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
744
745
+ def get_mm_mapping(self) -> MultiModelKeys:
746
+ """
747
+ Get the module prefix in multimodal models
748
+ """
749
+ return MultiModelKeys.from_string_field(
750
+ language_model="language_model",
751
+ connector="multi_modal_projector",
752
+ tower_model="vision_tower",
753
+ )
754
+
755
+ def get_num_mm_encoder_tokens(
756
+ self,
757
+ num_image_tokens: int,
758
+ ) -> int:
759
+ # LLaVA's vision encoder outputs one token per patch without
760
+ # spatial merging or pixel shuffle
761
+ return num_image_tokens
762
+
763
+ def get_num_mm_connector_tokens(
764
+ self,
765
+ num_vision_tokens: int,
766
+ ) -> int:
767
+ # LLaVA's MLP projector outputs the same number of tokens
768
+ # as it receives from the vision encoder (1:1 mapping)
769
+ return num_vision_tokens
770
+
771
772
class MantisProcessingInfo(LlavaProcessingInfo):
773
def get_hf_processor(self, **kwargs: object):