Migrate Voxtral to mistral-common 1.11.5 audio API (#46705)
Signed-off-by: Julien Denize <40604584+juliendenize@users.noreply.github.com>
Julien Denize committed
Jun 26, 2026 at 20:06 UTC
3d3b96488f5f7d94b1ab63919e0f1e7922a9ded6
12 files changed
+22
-29
examples/generate/multimodal/audio_language_offline.py
+2
-5
@@ -463,16 +463,15 @@ def run_ultravox(question: str, audio_count: int) -> ModelRequestData:
463
# Voxtral
464
# Make sure to install mistral-common[audio].
465
def run_voxtral(question: str, audio_count: int) -> ModelRequestData:
466
- from mistral_common.audio import Audio
466
from mistral_common.protocol.instruct.chunk import (
467
AudioChunk,
469
- RawAudio,
468
TextChunk,
469
)
470
from mistral_common.protocol.instruct.messages import (
471
UserMessage,
472
)
473
from mistral_common.protocol.instruct.request import ChatCompletionRequest
474
+ from mistral_common.tokens.tokenizers.audio import Audio
475
from mistral_common.tokens.tokenizers.mistral import MistralTokenizer
476
477
model_name = "mistralai/Voxtral-Mini-3B-2507"
@@ -495,9 +494,7 @@ def run_voxtral(question: str, audio_count: int) -> ModelRequestData:
494
Audio.from_file(str(audio_assets[i].get_local_path()), strict=False)
495
for i in range(audio_count)
496
]
498
- audio_chunks = [
499
- AudioChunk(input_audio=RawAudio.from_audio(audio)) for audio in audios
500
- ]
497
+ audio_chunks = [AudioChunk.from_audio(audio) for audio in audios]
498
499
messages = [UserMessage(content=[*audio_chunks, text_chunk])]
500
requirements/common.txt
+1
-1
@@ -33,7 +33,7 @@ partial-json-parser # used for parsing partial JSON outputs
33
jsonschema >= 4.23.0 # required for MiniMax M3 tool schema validation
34
pyzmq >= 25.0.0
35
msgspec
36
-mistral_common[image] >= 1.11.3
36
+mistral_common[image] >= 1.11.5
37
opencv-python-headless >= 4.13.0 # required for video IO
38
pyyaml
39
six>=1.16.0; python_version > '3.11' # transitive dependency of pandas that needs to be the latest version for python 3.12
requirements/test/cuda.in
+1
-1
@@ -31,7 +31,7 @@ torchaudio==2.11.0
31
torchvision==0.26.0
32
transformers_stream_generator # required for qwen-vl test
33
matplotlib # required for qwen-vl test
34
-mistral_common[image,audio] >= 1.11.3 # required for voxtral test
34
+mistral_common[image,audio] >= 1.11.5 # required for voxtral test
35
num2words # required for smolvlm test
36
open_clip_torch==2.32.0 # Required for nemotron_vl test, Nemotron Parse in test_common.py
37
opencv-python-headless >= 4.13.0 # required for video test
requirements/test/cuda.txt
+1
-1
@@ -398,7 +398,7 @@ mbstrdecoder==1.1.3
398
# typepy
399
mdurl==0.1.2
400
# via markdown-it-py
401
-mistral-common==1.11.3
401
+mistral-common==1.11.5
402
# via
403
# -c requirements/common.txt
404
# -r requirements/test/cuda.in
requirements/test/nightly-torch.txt
+1
-1
@@ -23,7 +23,7 @@ jiwer # required for audio tests
23
timm # required for internvl test
24
transformers_stream_generator # required for qwen-vl test
25
matplotlib # required for qwen-vl test
26
-mistral_common[image,audio] >= 1.11.3 # required for voxtral test
26
+mistral_common[image,audio] >= 1.11.5 # required for voxtral test
27
num2words # required for smolvlm test
28
opencv-python-headless >= 4.13.0 # required for video test
29
datamodel_code_generator # required for minicpm3 test
requirements/test/rocm.in
+1
-1
@@ -30,7 +30,7 @@ tblib # for pickling test exceptions
30
timm>=1.0.17 # required for internvl and gemma3n-mm test
31
transformers_stream_generator # required for qwen-vl test
32
matplotlib # required for qwen-vl test
33
-mistral_common[image,audio]>=1.11.3 # required for voxtral test
33
+mistral_common[image,audio]>=1.11.5 # required for voxtral test
34
num2words # required for smolvlm test
35
open_clip_torch==2.32.0 # Required for nemotron_vl test, Nemotron Parse in test_common.py
36
opencv-python-headless>=4.13.0 # required for video test
requirements/test/rocm.txt
+1
-1
@@ -499,7 +499,7 @@ mcp==1.27.0
499
# via -r requirements/test/../common.txt
500
mdurl==0.1.2
501
# via markdown-it-py
502
-mistral-common==1.11.3
502
+mistral-common==1.11.5
503
# via
504
# -c requirements/common.txt
505
# -r requirements/test/../common.txt
requirements/test/xpu.txt
+1
-1
@@ -264,7 +264,7 @@ mbstrdecoder==1.1.4
264
# typepy
265
mdurl==0.1.2
266
# via markdown-it-py
267
-mistral-common==1.11.3
267
+mistral-common==1.11.5
268
# via
269
# -c requirements/common.txt
270
# -r requirements/test/xpu.in
tests/models/multimodal/generation/test_voxtral.py
+3
-5
@@ -4,9 +4,9 @@
4
import json
5
6
import pytest
7
-from mistral_common.audio import Audio
8
-from mistral_common.protocol.instruct.chunk import AudioChunk, RawAudio, TextChunk
7
+from mistral_common.protocol.instruct.chunk import AudioChunk, TextChunk
8
from mistral_common.protocol.instruct.messages import UserMessage
9
+from mistral_common.tokens.tokenizers.audio import Audio
10
from transformers import VoxtralForConditionalGeneration
11
12
from vllm.tokenizers.mistral import MistralTokenizer
@@ -36,9 +36,7 @@ def _get_prompt(audio_assets: AudioTestAssets, question: str) -> list[int]:
36
Audio.from_file(str(asset.get_local_path()), strict=False)
37
for asset in audio_assets
38
]
39
- audio_chunks = [
40
- AudioChunk(input_audio=RawAudio.from_audio(audio)) for audio in audios
41
- ]
39
+ audio_chunks = [AudioChunk.from_audio(audio) for audio in audios]
40
41
messages = [
42
UserMessage(content=[*audio_chunks, TextChunk(text=question)]).to_openai()
tests/models/multimodal/generation/test_voxtral_realtime.py
+3
-4
@@ -4,12 +4,11 @@ import contextlib
4
5
import pytest
6
import pytest_asyncio
7
-from mistral_common.audio import Audio
8
-from mistral_common.protocol.instruct.chunk import RawAudio
7
from mistral_common.protocol.transcription.request import (
8
StreamingMode,
9
TranscriptionRequest,
10
)
11
+from mistral_common.tokens.tokenizers.audio import Audio
12
from mistral_common.tokens.tokenizers.mistral import MistralTokenizer
13
from mistral_common.tokens.tokenizers.tekken import SpecialTokenPolicy
14
@@ -101,7 +100,7 @@ def test_voxtral_realtime_forward(audio_assets, tokenizer, engine):
100
def from_file(file_path: str):
101
audio = Audio.from_file(file_path, strict=False)
102
req = TranscriptionRequest(
104
- audio=RawAudio.from_audio(audio),
103
+ audio=audio.to_base64(audio.format),
104
streaming=StreamingMode.OFFLINE,
105
language=None,
106
)
@@ -156,7 +155,7 @@ async def test_voxtral_realtime_generator(audio_assets, tokenizer, async_engine)
155
156
req = TranscriptionRequest(
157
streaming=StreamingMode.OFFLINE,
159
- audio=RawAudio.from_audio(audio),
158
+ audio=audio.to_base64(audio.format),
159
language=None,
160
)
161
audio_enc = tokenizer.encode_transcription(req)
vllm/model_executor/models/voxtral.py
+5
-4
@@ -10,11 +10,12 @@ import numpy as np
10
import regex as re
11
import torch
12
import torch.nn as nn
13
-from mistral_common.audio import Audio, mel_filter_bank
14
-from mistral_common.protocol.instruct.chunk import AudioChunk, RawAudio, TextChunk
13
+from mistral_common.audio import mel_filter_bank
14
+from mistral_common.protocol.instruct.chunk import AudioChunk, TextChunk
15
from mistral_common.protocol.instruct.messages import UserMessage
16
from mistral_common.protocol.instruct.request import ChatCompletionRequest
17
from mistral_common.protocol.transcription.request import TranscriptionRequest
18
+from mistral_common.tokens.tokenizers.audio import Audio
19
from transformers import BatchFeature, WhisperConfig
20
21
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
@@ -182,7 +183,7 @@ class VoxtralDummyInputsBuilder(BaseDummyInputsBuilder[VoxtralProcessingInfo]):
183
sampling_rate=feature_extractor.sampling_rate,
184
format=format,
185
)
185
- chunk = AudioChunk(input_audio=RawAudio.from_audio(audio_item))
186
+ chunk = AudioChunk.from_audio(audio_item)
187
audio_chunks.append(chunk)
188
189
request = ChatCompletionRequest(
@@ -462,7 +463,7 @@ class VoxtralForConditionalGeneration(
463
audio = Audio(audio, int(stt_config.sample_rate), format="wav") # lossless
464
req = TranscriptionRequest(
465
model=model_config.model,
465
- audio=RawAudio.from_audio(audio),
466
+ audio=audio.to_base64(audio.format),
467
language=language,
468
)
469
vllm/model_executor/models/voxtral_realtime.py
+2
-4
@@ -7,13 +7,11 @@ from collections.abc import AsyncGenerator, Iterable, Iterator, Mapping
7
8
import numpy as np
9
import torch
10
-from mistral_common.audio import Audio
11
-from mistral_common.protocol.instruct.chunk import RawAudio
10
from mistral_common.protocol.transcription.request import (
11
StreamingMode,
12
TranscriptionRequest,
13
)
16
-from mistral_common.tokens.tokenizers.audio import AudioConfig
14
+from mistral_common.tokens.tokenizers.audio import Audio, AudioConfig
15
16
from vllm.compilation.decorators import support_torch_compile
17
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
@@ -477,7 +475,7 @@ class VoxtralRealtimeGeneration(VoxtralForConditionalGeneration, SupportsRealtim
475
476
req = TranscriptionRequest(
477
model=model_config.model,
480
- audio=RawAudio.from_audio(audio),
478
+ audio=audio.to_base64(audio.format),
479
language=language,
480
streaming=StreamingMode.OFFLINE,
481
)