A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 10:00 UTC
[Voxtral] Streaming example (#33042)
Signed-off-by: Patrick von Platen <patrick.v.platen@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io>
Patrick von Platen committed
Jan 29, 2026 at 12:22 UTC
40c35038d28891488f2a784cfeadd20fc259a215
5 files changed
+207
-46
requirements/common.txt
+2
-2
@@ -31,7 +31,7 @@ partial-json-parser # used for parsing partial JSON outputs
31
pyzmq >= 25.0.0
32
msgspec
33
gguf >= 0.17.0
34
-mistral_common[image] >= 1.8.8
34
+mistral_common[image] >= 1.9.0
35
opencv-python-headless >= 4.13.0 # required for video IO
36
pyyaml
37
six>=1.16.0; python_version > '3.11' # transitive dependency of pandas that needs to be the latest version for python 3.12
@@ -52,4 +52,4 @@ anthropic >= 0.71.0
52
model-hosting-container-standards >= 0.1.13, < 1.0.0
53
mcp
54
grpcio
55
-grpcio-reflection
\ No newline at end of file
55
+grpcio-reflection
requirements/nightly_torch_test.txt
+1
-1
@@ -23,7 +23,7 @@ jiwer # required for audio tests
23
timm # required for internvl test
24
transformers_stream_generator # required for qwen-vl test
25
matplotlib # required for qwen-vl test
26
-mistral_common[image,audio] >= 1.8.8 # required for voxtral test
26
+mistral_common[image,audio] >= 1.9.0 # required for voxtral test
27
num2words # required for smolvlm test
28
opencv-python-headless >= 4.13.0 # required for video test
29
datamodel_code_generator # required for minicpm3 test
requirements/test.in
+1
-1
@@ -30,7 +30,7 @@ torchaudio==2.9.1
30
torchvision==0.24.1
31
transformers_stream_generator # required for qwen-vl test
32
matplotlib # required for qwen-vl test
33
-mistral_common[image,audio] >= 1.8.8 # required for voxtral test
33
+mistral_common[image,audio] >= 1.9.0 # required for voxtral test
34
num2words # required for smolvlm test
35
open_clip_torch==2.32.0 # Required for nemotron_vl test, Nemotron Parse in test_common.py
36
opencv-python-headless >= 4.13.0 # required for video test
requirements/test.txt
+1
-1
@@ -491,7 +491,7 @@ mbstrdecoder==1.1.3
491
# typepy
492
mdurl==0.1.2
493
# via markdown-it-py
494
-mistral-common==1.8.8
494
+mistral-common==1.9.0
495
# via -r requirements/test.in
496
mlflow==2.22.0
497
# via terratorch
tests/models/multimodal/generation/test_voxtral_streaming.py
+202
-41
@@ -1,5 +1,6 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+import asyncio
4
from dataclasses import asdict
5
6
import pytest
@@ -9,33 +10,71 @@ from mistral_common.protocol.transcription.request import (
10
StreamingMode,
11
TranscriptionRequest,
12
)
13
+from mistral_common.tokens.tokenizers.audio import AudioConfig
14
from mistral_common.tokens.tokenizers.mistral import MistralTokenizer
15
16
from vllm import LLM, EngineArgs, SamplingParams
17
from vllm.assets.audio import AudioAsset
18
+from vllm.engine.arg_utils import AsyncEngineArgs
19
+from vllm.inputs.data import TokensPrompt
20
+from vllm.v1.engine.async_llm import AsyncLLM, StreamingInput
21
22
+MODEL_NAME = "mistralai/Voxtral-Mini-3B-Realtime-2602"
23
+ENGINE_CONFIG = dict(
24
+ model=MODEL_NAME,
25
+ max_model_len=8192,
26
+ max_num_seqs=4,
27
+ limit_mm_per_prompt={"audio": 1},
28
+ config_format="mistral",
29
+ load_format="mistral",
30
+ tokenizer_mode="mistral",
31
+ enforce_eager=True,
32
+ gpu_memory_utilization=0.4,
33
+)
34
18
-def _get_engine(path: str) -> LLM:
19
- engine_args = EngineArgs(
20
- model=path,
21
- max_model_len=8192,
22
- max_num_seqs=1,
23
- limit_mm_per_prompt={"audio": 1},
24
- config_format="mistral",
25
- load_format="mistral",
26
- tokenizer_mode="mistral",
27
- enforce_eager=True,
28
- gpu_memory_utilization=0.4,
29
- )
35
+
36
+EXPECTED_TEXT = [
37
+ (
38
+ " First words I spoke in the original phonograph. "
39
+ "A little piece of practical poetry. Mary had a little lamb,"
40
+ " it sleeps with quite a snow, and everywhere that Mary went, "
41
+ "the lamb was sure to go."
42
+ ),
43
+ (
44
+ " And the 0-1 pitch on the way to Edgar Martinez. Swung on"
45
+ " the line. Down the left field line for OBS. Here comes Joy. "
46
+ "Here is Junior to third base. They're going to wave him in. "
47
+ "The throw to the plate will be late. The Mariners are going"
48
+ " to play. For the American League Championship, "
49
+ "I don't believe it. It just continues. My, oh, my."
50
+ ),
51
+]
52
+
53
+
54
+@pytest.fixture
55
+def audio_assets() -> list[AudioAsset]:
56
+ return [AudioAsset("mary_had_lamb"), AudioAsset("winning_call")]
57
+
58
+
59
+@pytest.fixture
60
+def tokenizer() -> MistralTokenizer:
61
+ return MistralTokenizer.from_hf_hub(MODEL_NAME)
62
+
63
+
64
+@pytest.fixture
65
+def engine() -> LLM:
66
+ engine_args = EngineArgs(**ENGINE_CONFIG)
67
return LLM(**asdict(engine_args))
68
69
33
-@pytest.mark.skip(reason="Voxtral streaming is not yet public")
34
-def test_voxtral_streaming_forward():
35
- audio_assets = [AudioAsset("mary_had_lamb"), AudioAsset("winning_call")]
70
+@pytest.fixture
71
+def async_engine() -> AsyncLLM:
72
+ engine_args = AsyncEngineArgs(**ENGINE_CONFIG)
73
+ return AsyncLLM.from_engine_args(engine_args)
74
37
- model_name = "mistralai/Voxtral-Mini-3B-Realtime-2602"
38
- tokenizer = MistralTokenizer.from_hf_hub(model_name)
75
+
76
+@pytest.mark.skip(reason="Voxtral streaming is not yet public")
77
+def test_voxtral_streaming_forward(audio_assets, tokenizer, engine):
78
audio_config = tokenizer.instruct_tokenizer.tokenizer.audio
79
80
def from_file(file_path: str):
@@ -58,11 +97,7 @@ def test_voxtral_streaming_forward():
97
98
for tokens, audio_array in tokenized_list:
99
num_samples = audio_array.shape[0]
61
- max_tokens = (
62
- audio_config.num_audio_tokens(num_samples)
63
- - audio_config.num_delay_tokens
64
- - 1
65
- )
100
+ max_tokens = audio_config.num_audio_tokens(num_samples) - len(tokens) - 1
101
sampling_params.append(SamplingParams(temperature=0.0, max_tokens=max_tokens))
102
103
input_dict = {
@@ -71,27 +106,153 @@ def test_voxtral_streaming_forward():
106
}
107
inputs.append(input_dict)
108
74
- llm = _get_engine(model_name)
75
- outputs = llm.generate(
109
+ outputs = engine.generate(
110
inputs,
111
sampling_params=sampling_params,
112
)
113
114
texts = [out.outputs[0].text for out in outputs]
81
- expected = [
82
- (
83
- " First words I spoke in the original phonograph. "
84
- "A little piece of practical poetry. Mary had a little lamb,"
85
- " it sleeps with quite a snow, and everywhere that Mary went, "
86
- "the lamb was sure to go."
87
- ),
88
- (
89
- " And the 0-1 pitch on the way to Edgar Martinez. Swung on"
90
- " the line. Down the left field line for OBS. Here comes Joy. "
91
- "Here is Junior to third base. They're going to wave him in. "
92
- "The throw to the plate will be late. The Mariners are going"
93
- " to play. For the American League Championship, "
94
- "I don't believe it. It just continues. My oh, my."
95
- ),
96
- ]
97
- assert texts == expected
115
+ assert texts == EXPECTED_TEXT
116
+
117
+
118
+class RealTimeAudioInput:
119
+ """
120
+ This class is used to stream an audio file just as
121
+ if it would be streamed in real-time.
122
+ """
123
+
124
+ def __init__(self, tokenizer: MistralTokenizer) -> None:
125
+ self._tokenizer = tokenizer
126
+ self._config: AudioConfig = (
127
+ self._tokenizer.instruct_tokenizer.audio_encoder.audio_config
128
+ )
129
+
130
+ self._look_ahead_in_ms = self._config.streaming_look_ahead_ms
131
+ self._look_back_in_ms = self._config.streaming_look_back_ms
132
+
133
+ self._sampling_rate = self._config.sampling_rate
134
+
135
+ self._audio: Audio | None = None
136
+
137
+ # mutable objects
138
+ self._start = 0
139
+
140
+ n_left_pad_samples = (
141
+ self._config.raw_audio_length_per_tok * self._config.n_left_pad_tokens
142
+ )
143
+ self._end = self.streaming_delay + n_left_pad_samples + self.streaming_size
144
+ self._queue: asyncio.Queue[StreamingInput | None] = asyncio.Queue()
145
+
146
+ @classmethod
147
+ async def create(cls, audio: Audio, tokenizer: MistralTokenizer):
148
+ self = cls(tokenizer)
149
+
150
+ # we're doing "OFFLINE" encoding here to right & left pad the audio since
151
+ # we have access to the whole audio
152
+ # if we'd do an actual online realtime streaming application we
153
+ # should instead pass `StreamingMode.ONLINE`
154
+ req = TranscriptionRequest(
155
+ streaming=StreamingMode.OFFLINE,
156
+ audio=RawAudio.from_audio(audio),
157
+ language=None,
158
+ )
159
+ audio_enc = self._tokenizer.encode_transcription(req)
160
+ self._audio = audio_enc.audios[0]
161
+
162
+ # add first request
163
+ await self.add_tokens(audio_enc.tokens)
164
+
165
+ return self
166
+
167
+ @property
168
+ def look_ahead(self) -> int:
169
+ return self._get_len_in_samples(self._look_ahead_in_ms)
170
+
171
+ @property
172
+ def look_back(self) -> int:
173
+ return self._get_len_in_samples(self._look_back_in_ms)
174
+
175
+ @property
176
+ def streaming_delay(self) -> int:
177
+ return self._get_len_in_samples(self._config.transcription_delay_ms)
178
+
179
+ @property
180
+ def streaming_size(self) -> int:
181
+ stream_size_in_ms = 1000 / self._config.frame_rate
182
+ return self._get_len_in_samples(stream_size_in_ms)
183
+
184
+ def _get_len_in_samples(self, len_in_ms: float) -> int:
185
+ _len_in_s = self._sampling_rate * len_in_ms / 1000
186
+ assert _len_in_s.is_integer(), _len_in_s
187
+ len_in_s = int(_len_in_s)
188
+
189
+ return len_in_s
190
+
191
+ async def add_tokens(self, tokens: list[int]) -> None:
192
+ assert self._audio is not None
193
+ if self._start >= len(self._audio.audio_array):
194
+ self.stop()
195
+ return
196
+
197
+ _end = self._end + self.look_ahead
198
+ _start = max(0, self._start - self.look_back)
199
+
200
+ multi_modal_data = {"audio": (self._audio.audio_array[_start:_end], None)}
201
+
202
+ prompt = TokensPrompt(
203
+ prompt_token_ids=tokens, multi_modal_data=multi_modal_data
204
+ )
205
+
206
+ await self._queue.put(StreamingInput(prompt))
207
+
208
+ # increase
209
+ self._start = self._end
210
+ self._end = self._end + self.streaming_size
211
+
212
+ def stop(self):
213
+ self._queue.put_nowait(None)
214
+
215
+ async def generator(self):
216
+ while (item := await self._queue.get()) is not None:
217
+ yield item
218
+
219
+
220
+@pytest.mark.asyncio
221
+@pytest.mark.skip(reason="Voxtral streaming is not yet public")
222
+async def test_voxtral_streaming_generator(audio_assets, tokenizer, async_engine):
223
+ sampling_params = SamplingParams(temperature=0.0, max_tokens=1)
224
+
225
+ output_tokens_list = []
226
+ for i, audio_asset in enumerate(audio_assets):
227
+ output_tokens = []
228
+ audio = Audio.from_file(audio_asset.get_local_path(), strict=False)
229
+ streaming_input = await RealTimeAudioInput.create(
230
+ audio=audio, tokenizer=tokenizer
231
+ )
232
+
233
+ request_id = f"session-{i}"
234
+
235
+ async for resp in async_engine.generate(
236
+ prompt=streaming_input.generator(),
237
+ sampling_params=sampling_params,
238
+ request_id=request_id,
239
+ ):
240
+ tokens = resp.outputs[0].token_ids[-1:]
241
+
242
+ output_tokens.extend(tokens)
243
+ await streaming_input.add_tokens(tokens)
244
+
245
+ output_tokens_list.append(output_tokens)
246
+
247
+ texts = [tokenizer.decode(output_tokens) for output_tokens in output_tokens_list]
248
+
249
+ # 'true' streaming and 'offline' streaming differ a bit because log-mels are
250
+ # differently noramalized
251
+ texts[0] = (
252
+ texts[0]
253
+ .replace("He has f", "F")
254
+ .replace("its fleece was quite a slow", "it sleeps with quite a snow")
255
+ )
256
+ texts[1] = texts[1].replace("a base hit", "OBS").replace("oh my", "oh, my")
257
+
258
+ assert texts == EXPECTED_TEXT