A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 12:15 UTC
Support LoRA for Mistral3 (#17428)
Signed-off-by: mgoin <mgoin64@gmail.com>
Michael Goin committed
Apr 29, 2025 at 22:10 UTC
a44c4f1d2f7cb882e0045b0c7d7cbcf8e08ef9bd
2 files changed
+15
-4
docs/source/models/supported_models.md
+1
-1
@@ -990,7 +990,7 @@ See [this page](#generative-models) for more information on how to use generativ
990
* Mistral3
991
* T + I<sup>+</sup>
992
* `mistralai/Mistral-Small-3.1-24B-Instruct-2503`, etc.
993
- *
993
+ * ✅︎
994
* ✅︎
995
* ✅︎
996
- * `MllamaForConditionalGeneration`
vllm/model_executor/models/mistral3.py
+14
-3
@@ -18,6 +18,7 @@ from vllm.model_executor.layers.layernorm import RMSNorm
18
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
19
RowParallelLinear)
20
from vllm.model_executor.layers.quantization import QuantizationConfig
21
+from vllm.model_executor.models.module_mapping import MultiModelKeys
22
from vllm.model_executor.sampling_metadata import SamplingMetadata
23
from vllm.multimodal import MULTIMODAL_REGISTRY
24
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
@@ -31,7 +32,8 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor,
32
from vllm.multimodal.profiling import BaseDummyInputsBuilder
33
from vllm.sequence import IntermediateTensors
34
34
-from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
35
+from .interfaces import (MultiModalEmbeddings, SupportsLoRA,
36
+ SupportsMultiModal, SupportsPP)
37
from .pixtral import PixtralHFEncoderInfo, PixtralHFVisionModel
38
from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
39
maybe_prefix, merge_multimodal_embeddings)
@@ -382,8 +384,8 @@ def init_vision_tower_for_llava(
384
_build_mistral3_processor,
385
info=_build_mistral3_info,
386
dummy_inputs=Mistral3DummyInputsBuilder)
385
-class Mistral3ForConditionalGeneration(nn.Module, SupportsMultiModal,
386
- SupportsPP):
387
+class Mistral3ForConditionalGeneration(nn.Module, SupportsLoRA,
388
+ SupportsMultiModal, SupportsPP):
389
390
packed_modules_mapping = {
391
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
@@ -594,3 +596,12 @@ class Mistral3ForConditionalGeneration(nn.Module, SupportsMultiModal,
596
torch.Tensor]]) -> Set[str]:
597
loader = AutoWeightsLoader(self)
598
return loader.load_weights(weights)
599
+
600
+ def get_mm_mapping(self) -> MultiModelKeys:
601
+ """
602
+ Get the module prefix in multimodal models
603
+ """
604
+ return MultiModelKeys.from_string_field(
605
+ language_model="language_model",
606
+ connector="multi_modal_projector",
607
+ tower_model="vision_tower")