A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 21:00 UTC
[Transformers v5] fix missing pixtral/voxtral multimodal dispatch (#38410)
Signed-off-by: allgather <all2allops@gmail.com>
allgather committed
Mar 29, 2026 at 03:59 UTC
8c0b6267d7fa5c8a07e318809180fc021a0afbf2
4 files changed
+40
-22
vllm/model_executor/models/pixtral.py
+10
-5
@@ -61,7 +61,10 @@ from vllm.platforms import current_platform
61
from vllm.sequence import IntermediateTensors
62
from vllm.tokenizers import cached_tokenizer_from_config
63
from vllm.tokenizers.mistral import MistralTokenizer
64
-from vllm.transformers_utils.processors.pixtral import MistralCommonPixtralProcessor
64
+from vllm.transformers_utils.processors.pixtral import (
65
+ MistralCommonImageProcessor,
66
+ MistralCommonPixtralProcessor,
67
+)
68
from vllm.utils.collection_utils import is_list_of
69
from vllm.utils.tensor_schema import TensorSchema, TensorShape
70
@@ -128,18 +131,20 @@ class PixtralProcessingInfo(BaseProcessingInfo):
131
132
return tokenizer
133
134
+ def get_image_processor(self) -> MistralCommonImageProcessor:
135
+ return MistralCommonImageProcessor(self.get_tokenizer().instruct.mm_encoder)
136
+
137
def get_hf_processor(self, **kwargs) -> MistralCommonPixtralProcessor:
132
- return self.ctx.init_processor(
133
- MistralCommonPixtralProcessor,
138
+ return MistralCommonPixtralProcessor(
139
tokenizer=self.get_tokenizer(),
135
- **kwargs,
140
+ image_processor=self.get_image_processor(),
141
)
142
143
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
144
return {"image": None}
145
146
def get_image_size_with_most_features(self) -> ImageSize:
142
- image_processor = self.get_hf_processor().image_processor
147
+ image_processor = self.get_image_processor()
148
max_image_size = image_processor.mm_encoder.mm_config.max_image_size
149
150
return ImageSize(width=max_image_size, height=max_image_size)
vllm/model_executor/models/voxtral.py
+18
-9
@@ -55,7 +55,10 @@ from vllm.multimodal.processing.processor import (
55
from vllm.sequence import IntermediateTensors
56
from vllm.tokenizers import cached_tokenizer_from_config
57
from vllm.tokenizers.mistral import MistralTokenizer
58
-from vllm.transformers_utils.processors.voxtral import MistralCommonVoxtralProcessor
58
+from vllm.transformers_utils.processors.voxtral import (
59
+ MistralCommonFeatureExtractor,
60
+ MistralCommonVoxtralProcessor,
61
+)
62
from vllm.utils.collection_utils import is_list_of
63
64
from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsTranscription
@@ -84,15 +87,19 @@ class VoxtralProcessingInfo(BaseProcessingInfo):
87
88
return tokenizer
89
90
+ def get_feature_extractor(self) -> MistralCommonFeatureExtractor:
91
+ return MistralCommonFeatureExtractor(
92
+ self.get_tokenizer().instruct.audio_encoder
93
+ )
94
+
95
def get_hf_processor(self, **kwargs) -> MistralCommonVoxtralProcessor:
88
- return self.ctx.init_processor(
89
- MistralCommonVoxtralProcessor,
96
+ return MistralCommonVoxtralProcessor(
97
tokenizer=self.get_tokenizer(),
91
- **kwargs,
98
+ feature_extractor=self.get_feature_extractor(),
99
)
100
101
def get_data_parser(self):
95
- feature_extractor = self.get_hf_processor().feature_extractor
102
+ feature_extractor = self.get_feature_extractor()
103
104
return MultiModalDataParser(
105
target_sr=feature_extractor.sampling_rate,
@@ -114,7 +121,7 @@ class VoxtralProcessingInfo(BaseProcessingInfo):
121
return self.ctx.model_config.max_model_len
122
123
def get_max_audio_array_len(self) -> int:
117
- feature_extractor = self.get_hf_processor().feature_extractor
124
+ feature_extractor = self.get_feature_extractor()
125
126
return self.get_max_audio_tokens() * int(
127
feature_extractor.sampling_rate // feature_extractor.frame_rate
@@ -153,7 +160,7 @@ class VoxtralDummyInputsBuilder(BaseDummyInputsBuilder[VoxtralProcessingInfo]):
160
mm_data: MultiModalDataDict | None = None,
161
) -> ProcessorInputs:
162
tokenizer = self.info.get_tokenizer()
156
- feature_extractor = self.info.get_hf_processor().feature_extractor
163
+ feature_extractor = self.info.get_feature_extractor()
164
165
dummy_text = self.get_dummy_text(mm_counts)
166
dummy_mm_data = (
@@ -480,8 +487,10 @@ class VoxtralForConditionalGeneration(
487
This is used for estimating the amount of processing for this audio.
488
"""
489
tokenizer = cached_tokenizer_from_config(model_config)
483
- adapter = MistralCommonVoxtralProcessor(tokenizer)
484
- return adapter.feature_extractor.get_num_audio_tokens(
490
+ feature_extractor = MistralCommonFeatureExtractor(
491
+ tokenizer.instruct.audio_encoder
492
+ )
493
+ return feature_extractor.get_num_audio_tokens(
494
int(audio_duration_s * stt_config.sample_rate)
495
)
496
vllm/transformers_utils/processors/pixtral.py
+6
-4
@@ -50,16 +50,18 @@ class MistralCommonImageProcessor:
50
class MistralCommonPixtralProcessor(ProcessorMixin):
51
attributes = ["image_processor", "tokenizer"]
52
53
- def __init__(self, tokenizer: MistralTokenizer) -> None:
53
+ def __init__(
54
+ self,
55
+ tokenizer: MistralTokenizer,
56
+ image_processor: MistralCommonImageProcessor,
57
+ ) -> None:
58
self.tokenizer = tokenizer.transformers_tokenizer
59
60
# Back-compatibility for Transformers v4
61
if not hasattr(self.tokenizer, "init_kwargs"):
62
self.tokenizer.init_kwargs = {}
63
60
- self.image_processor = MistralCommonImageProcessor(
61
- tokenizer.instruct.mm_encoder
62
- )
64
+ self.image_processor = image_processor
65
66
image_special_ids = self.image_processor.mm_encoder.special_ids
67
self.image_break_id = image_special_ids.img_break
vllm/transformers_utils/processors/voxtral.py
+6
-4
@@ -57,16 +57,18 @@ class MistralCommonFeatureExtractor:
57
class MistralCommonVoxtralProcessor(ProcessorMixin):
58
attributes = ["feature_extractor", "tokenizer"]
59
60
- def __init__(self, tokenizer: MistralTokenizer) -> None:
60
+ def __init__(
61
+ self,
62
+ tokenizer: MistralTokenizer,
63
+ feature_extractor: MistralCommonFeatureExtractor,
64
+ ) -> None:
65
self.tokenizer = tokenizer.transformers_tokenizer
66
67
# Back-compatibility for Transformers v4
68
if not hasattr(self.tokenizer, "init_kwargs"):
69
self.tokenizer.init_kwargs = {}
70
67
- self.feature_extractor = MistralCommonFeatureExtractor(
68
- tokenizer.instruct.audio_encoder
69
- )
71
+ self.feature_extractor = feature_extractor
72
73
audio_special_ids = self.feature_extractor.audio_encoder.special_ids
74
self.audio_token_id = audio_special_ids.audio