A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 12:45 UTC
[Model] Enable LoRA support for Pixtral (#31724)
Signed-off-by: <> Signed-off-by: 赵策 <alcor@zhaocedeMacBook-Air.local> Signed-off-by: 赵策 <alcor@mac.mynetworksettings.com> Co-authored-by: 赵策 <alcor@mac.mynetworksettings.com>
Ce Zhao committed
Jan 8, 2026 at 08:00 UTC
1123a878920f3ef35f858558415d43b9ed22701e
2 files changed
+30
-3
docs/models/supported_models.md
+1
-1
@@ -724,7 +724,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
724
| `PaliGemmaForConditionalGeneration` | PaliGemma, PaliGemma 2 | T + I<sup>E</sup> | `google/paligemma-3b-pt-224`, `google/paligemma-3b-mix-224`, `google/paligemma2-3b-ft-docci-448`, etc. | ✅︎ | ✅︎ |
725
| `Phi3VForCausalLM` | Phi-3-Vision, Phi-3.5-Vision | T + I<sup>E+</sup> | `microsoft/Phi-3-vision-128k-instruct`, `microsoft/Phi-3.5-vision-instruct`, etc. | | ✅︎ |
726
| `Phi4MMForCausalLM` | Phi-4-multimodal | T + I<sup>+</sup> / T + A<sup>+</sup> / I<sup>+</sup> + A<sup>+</sup> | `microsoft/Phi-4-multimodal-instruct`, etc. | ✅︎ | ✅︎ |
727
-| `PixtralForConditionalGeneration` | Ministral 3 (Mistral format), Mistral 3 (Mistral format), Mistral Large 3 (Mistral format), Pixtral (Mistral format) | T + I<sup>+</sup> | `mistralai/Ministral-3-3B-Instruct-2512`, `mistralai/Mistral-Small-3.1-24B-Instruct-2503`, `mistralai/Mistral-Large-3-675B-Instruct-2512` `mistralai/Pixtral-12B-2409` etc. | | ✅︎ |
727
+| `PixtralForConditionalGeneration` | Ministral 3 (Mistral format), Mistral 3 (Mistral format), Mistral Large 3 (Mistral format), Pixtral (Mistral format) | T + I<sup>+</sup> | `mistralai/Ministral-3-3B-Instruct-2512`, `mistralai/Mistral-Small-3.1-24B-Instruct-2503`, `mistralai/Mistral-Large-3-675B-Instruct-2512` `mistralai/Pixtral-12B-2409` etc. | ✅︎ | ✅︎ |
728
| `QwenVLForConditionalGeneration`<sup>^</sup> | Qwen-VL | T + I<sup>E+</sup> | `Qwen/Qwen-VL`, `Qwen/Qwen-VL-Chat`, etc. | ✅︎ | ✅︎ |
729
| `Qwen2AudioForConditionalGeneration` | Qwen2-Audio | T + A<sup>+</sup> | `Qwen/Qwen2-Audio-7B-Instruct` | | ✅︎ |
730
| `Qwen2VLForConditionalGeneration` | QVQ, Qwen2-VL | T + I<sup>E+</sup> + V<sup>E+</sup> | `Qwen/QVQ-72B-Preview`, `Qwen/Qwen2-VL-7B-Instruct`, `Qwen/Qwen2-VL-72B-Instruct`, etc. | ✅︎ | ✅︎ |
vllm/model_executor/models/pixtral.py
+29
-2
@@ -63,7 +63,13 @@ from vllm.tokenizers import cached_tokenizer_from_config
63
from vllm.tokenizers.mistral import MistralTokenizer
64
from vllm.utils.tensor_schema import TensorSchema, TensorShape
65
66
-from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
66
+from .interfaces import (
67
+ MultiModalEmbeddings,
68
+ SupportsLoRA,
69
+ SupportsMultiModal,
70
+ SupportsPP,
71
+)
72
+from .module_mapping import MultiModelKeys
73
from .utils import init_vllm_registered_model, maybe_prefix
74
from .vision import (
75
VisionEncoderInfo,
@@ -365,7 +371,9 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo])
371
info=PixtralProcessingInfo,
372
dummy_inputs=PixtralDummyInputsBuilder,
373
)
368
-class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
374
+class PixtralForConditionalGeneration(
375
+ nn.Module, SupportsLoRA, SupportsMultiModal, SupportsPP
376
+):
377
@classmethod
378
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
379
if modality.startswith("image"):
@@ -581,6 +589,25 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP)
589
# Now we call the language model load with the generator
590
self.language_model.load_weights(llm_weights_generator())
591
592
+ def get_mm_mapping(self) -> MultiModelKeys:
593
+ return MultiModelKeys.from_string_field(
594
+ language_model="language_model",
595
+ connector="vision_language_adapter",
596
+ tower_model="vision_encoder",
597
+ )
598
+
599
+ def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int:
600
+ if getattr(self, "patch_merger", None) is None:
601
+ return num_image_tokens
602
+ merge_size = self.vision_args.spatial_merge_size
603
+ return num_image_tokens * (merge_size**2)
604
+
605
+ def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int:
606
+ if getattr(self, "patch_merger", None) is None:
607
+ return num_vision_tokens
608
+ merge_size = self.vision_args.spatial_merge_size
609
+ return num_vision_tokens // (merge_size**2)
610
+
611
612
# Vision encoder
613
@dataclass