A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 16:00 UTC
[V1] Support Mistral3 in V1 (#15950)
Signed-off-by: mgoin <mgoin64@gmail.com>
Michael Goin committed
Apr 2, 2025 at 16:36 UTC
f021b9799386d0df3bb3befad351de11fcc36c5b
2 files changed
+10
-7
docs/source/models/supported_models.md
+1
-1
@@ -888,7 +888,7 @@ See [this page](#generative-models) for more information on how to use generativ
888
* `mistralai/Mistral-Small-3.1-24B-Instruct-2503`, etc.
889
*
890
* ✅︎
891
- *
891
+ * ✅︎
892
- * `MllamaForConditionalGeneration`
893
* Llama 3.2
894
* T + I<sup>+</sup>
vllm/model_executor/models/mistral3.py
+9
-6
@@ -31,12 +31,12 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor,
31
from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
32
from vllm.sequence import IntermediateTensors
33
34
-from .interfaces import (MultiModalEmbeddings, SupportsMultiModal, SupportsPP,
35
- SupportsV0Only)
34
+from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
35
from .pixtral import PixtralHFEncoderInfo, PixtralHFVisionModel
36
from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
37
maybe_prefix, merge_multimodal_embeddings)
39
-from .vision import get_vision_encoder_info, select_patch_features
38
+from .vision import (get_vision_encoder_info, scatter_patch_features,
39
+ select_patch_features)
40
41
42
class Mistral3ImagePixelInputs(TypedDict):
@@ -425,7 +425,7 @@ def init_vision_tower_for_llava(
425
info=_build_mistral3_info,
426
dummy_inputs=Mistral3DummyInputsBuilder)
427
class Mistral3ForConditionalGeneration(nn.Module, SupportsMultiModal,
428
- SupportsPP, SupportsV0Only):
428
+ SupportsPP):
429
430
packed_modules_mapping = {
431
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
@@ -518,7 +518,7 @@ class Mistral3ForConditionalGeneration(nn.Module, SupportsMultiModal,
518
return Mistral3ImagePixelInputs(
519
type="pixel_values_pixtral",
520
pixel_values=flatten_bn(pixel_values),
521
- embed_is_patch=embed_is_patch,
521
+ embed_is_patch=flatten_bn(embed_is_patch),
522
)
523
524
def _process_image_input(
@@ -557,7 +557,10 @@ class Mistral3ForConditionalGeneration(nn.Module, SupportsMultiModal,
557
558
vision_embeddings = self._process_image_input(image_input)
559
560
- return vision_embeddings
560
+ return scatter_patch_features(
561
+ vision_embeddings,
562
+ image_input["embed_is_patch"],
563
+ )
564
565
def get_input_embeddings(
566
self,