A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 15:15 UTC
[Model] Enable LoRA support for PaliGemma (#31656)
Signed-off-by: 赵策 <alcor@mac.mynetworksettings.com> Signed-off-by: Alcor <alcor_zhao@outlook.com> Co-authored-by: 赵策 <alcor@mac.mynetworksettings.com>
Ce Zhao committed
Jan 6, 2026 at 21:09 UTC
a051525e071c2387641b17b95533fb51ed9363e1
2 files changed
+24
-3
docs/models/supported_models.md
+1
-1
@@ -716,7 +716,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
716
| `Ovis` | Ovis2, Ovis1.6 | T + I<sup>+</sup> | `AIDC-AI/Ovis2-1B`, `AIDC-AI/Ovis1.6-Llama3.2-3B`, etc. | | ✅︎ |
717
| `Ovis2_5` | Ovis2.5 | T + I<sup>+</sup> + V | `AIDC-AI/Ovis2.5-9B`, etc. | | |
718
| `PaddleOCRVLForConditionalGeneration` | Paddle-OCR | T + I<sup>+</sup> | `PaddlePaddle/PaddleOCR-VL`, etc. | | |
719
-| `PaliGemmaForConditionalGeneration` | PaliGemma, PaliGemma 2 | T + I<sup>E</sup> | `google/paligemma-3b-pt-224`, `google/paligemma-3b-mix-224`, `google/paligemma2-3b-ft-docci-448`, etc. | | ✅︎ |
719
+| `PaliGemmaForConditionalGeneration` | PaliGemma, PaliGemma 2 | T + I<sup>E</sup> | `google/paligemma-3b-pt-224`, `google/paligemma-3b-mix-224`, `google/paligemma2-3b-ft-docci-448`, etc. | ✅︎ | ✅︎ |
720
| `Phi3VForCausalLM` | Phi-3-Vision, Phi-3.5-Vision | T + I<sup>E+</sup> | `microsoft/Phi-3-vision-128k-instruct`, `microsoft/Phi-3.5-vision-instruct`, etc. | | ✅︎ |
721
| `Phi4MMForCausalLM` | Phi-4-multimodal | T + I<sup>+</sup> / T + A<sup>+</sup> / I<sup>+</sup> + A<sup>+</sup> | `microsoft/Phi-4-multimodal-instruct`, etc. | ✅︎ | ✅︎ |
722
| `PixtralForConditionalGeneration` | Ministral 3 (Mistral format), Mistral 3 (Mistral format), Mistral Large 3 (Mistral format), Pixtral (Mistral format) | T + I<sup>+</sup> | `mistralai/Ministral-3-3B-Instruct-2512`, `mistralai/Mistral-Small-3.1-24B-Instruct-2503`, `mistralai/Mistral-Large-3-675B-Instruct-2512` `mistralai/Pixtral-12B-2409` etc. | | ✅︎ |
vllm/model_executor/models/paligemma.py
+23
-2
@@ -35,7 +35,13 @@ from vllm.multimodal.profiling import BaseDummyInputsBuilder
35
from vllm.sequence import IntermediateTensors
36
from vllm.utils.tensor_schema import TensorSchema, TensorShape
37
38
-from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
38
+from .interfaces import (
39
+ MultiModalEmbeddings,
40
+ SupportsLoRA,
41
+ SupportsMultiModal,
42
+ SupportsPP,
43
+)
44
+from .module_mapping import MultiModelKeys
45
from .siglip import SiglipVisionModel
46
from .utils import (
47
AutoWeightsLoader,
@@ -250,7 +256,9 @@ class PaliGemmaMultiModalProcessor(BaseMultiModalProcessor[PaliGemmaProcessingIn
256
info=PaliGemmaProcessingInfo,
257
dummy_inputs=PaliGemmaDummyInputsBuilder,
258
)
253
-class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
259
+class PaliGemmaForConditionalGeneration(
260
+ nn.Module, SupportsLoRA, SupportsMultiModal, SupportsPP
261
+):
262
packed_modules_mapping = {
263
"qkv_proj": [
264
"q_proj",
@@ -406,3 +414,16 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsP
414
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
415
loader = AutoWeightsLoader(self)
416
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
417
+
418
+ def get_mm_mapping(self) -> MultiModelKeys:
419
+ return MultiModelKeys.from_string_field(
420
+ language_model="language_model",
421
+ connector="multi_modal_projector",
422
+ tower_model="vision_tower",
423
+ )
424
+
425
+ def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int:
426
+ return num_image_tokens
427
+
428
+ def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int:
429
+ return num_vision_tokens