A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 06:15 UTC
[Streaming -> Realtime] Rename all voxtral related classes, fn, files (#33415)
Signed-off-by: Patrick von Platen <patrick.v.platen@gmail.com>
Patrick von Platen committed
Jan 31, 2026 at 05:49 UTC
15e0bb9c428e08ef74e067da2b8b3f4765e3cd52
6 files changed
+18
-65
docs/serving/openai_compatible_server.md
+3
-47
@@ -610,54 +610,10 @@ Audio must be sent as base64-encoded PCM16 audio at 16kHz sample rate, mono chan
610
| `transcription.done` | Final transcription with usage stats |
611
| `error` | Error notification with message and optional code |
612
613
-#### Python WebSocket Example
613
+#### Example Clients
614
615
-??? code
616
-
617
- ```python
618
- import asyncio
619
- import base64
620
- import json
621
- import websockets
622
-
623
- async def realtime_transcribe():
624
- uri = "ws://localhost:8000/v1/realtime"
625
-
626
- async with websockets.connect(uri) as ws:
627
- # Wait for session.created
628
- response = await ws.recv()
629
- print(f"Session: {response}")
630
-
631
- # Commit buffer
632
- await ws.send(json.dumps({
633
- "type": "input_audio_buffer.commit"
634
- }))
635
-
636
- # Send audio chunks (example with file)
637
- with open("audio.raw", "rb") as f:
638
- while chunk := f.read(4096):
639
- await ws.send(json.dumps({
640
- "type": "input_audio_buffer.append",
641
- "audio": base64.b64encode(chunk).decode()
642
- }))
643
-
644
- # Signal all audio is sent
645
- await ws.send(json.dumps({
646
- "type": "input_audio_buffer.commit",
647
- "final": True,
648
- }))
649
-
650
- # Receive transcription
651
- while True:
652
- response = json.loads(await ws.recv())
653
- if response["type"] == "transcription.delta":
654
- print(response["delta"], end="", flush=True)
655
- elif response["type"] == "transcription.done":
656
- print(f"\nFinal: {response['text']}")
657
- break
658
-
659
- asyncio.run(realtime_transcribe())
660
- ```
615
+- [openai_realtime_client.py](https://github.com/vllm-project/vllm/tree/main/examples/online_serving/openai_realtime_client.py) - Upload and transcribe an audio file
616
+- [openai_realtime_microphone_client.py](https://github.com/vllm-project/vllm/tree/main/examples/online_serving/openai_realtime_microphone_client.py) - Gradio demo for live microphone transcription
617
618
### Tokenizer API
619
tests/models/multimodal/generation/test_voxtral_realtime.py
renamed
+2
-2
@@ -74,7 +74,7 @@ def async_engine() -> AsyncLLM:
74
75
76
@pytest.mark.skip(reason="Voxtral streaming is not yet public")
77
-def test_voxtral_streaming_forward(audio_assets, tokenizer, engine):
77
+def test_voxtral_realtime_forward(audio_assets, tokenizer, engine):
78
audio_config = tokenizer.instruct_tokenizer.tokenizer.audio
79
80
def from_file(file_path: str):
@@ -219,7 +219,7 @@ class RealTimeAudioInput:
219
220
@pytest.mark.asyncio
221
@pytest.mark.skip(reason="Voxtral streaming is not yet public")
222
-async def test_voxtral_streaming_generator(audio_assets, tokenizer, async_engine):
222
+async def test_voxtral_realtime_generator(audio_assets, tokenizer, async_engine):
223
sampling_params = SamplingParams(temperature=0.0, max_tokens=1)
224
225
output_tokens_list = []
tests/models/registry.py
+1
-1
@@ -989,7 +989,7 @@ _MULTIMODAL_EXAMPLE_MODELS = {
989
# disable this temporarily until we support HF format
990
is_available_online=False,
991
),
992
- "VoxtralStreamingGeneration": _HfExamplesInfo(
992
+ "VoxtralRealtimeGeneration": _HfExamplesInfo(
993
"<place-holder>",
994
# disable this temporarily until we support HF format
995
is_available_online=False,
vllm/model_executor/models/registry.py
+1
-1
@@ -462,7 +462,7 @@ _MULTIMODAL_MODELS = {
462
),
463
"UltravoxModel": ("ultravox", "UltravoxModel"),
464
"VoxtralForConditionalGeneration": ("voxtral", "VoxtralForConditionalGeneration"), # noqa: E501
465
- "VoxtralStreamingGeneration": ("voxtral_streaming", "VoxtralStreamingGeneration"), # noqa: E501
465
+ "VoxtralRealtimeGeneration": ("voxtral_realtime", "VoxtralRealtimeGeneration"), # noqa: E501
466
# [Encoder-decoder]
467
"NemotronParseForConditionalGeneration": (
468
"nemotron_parse",
vllm/model_executor/models/voxtral_realtime.py
renamed
+10
-13
@@ -50,7 +50,7 @@ logger = init_logger(__name__)
50
_PRE_ALLOCATE_BUFFER_SIZE_IN_S = 30
51
52
53
-class VoxtralStreamingMultiModalProcessor(VoxtralMultiModalProcessor):
53
+class VoxtralRealtimeMultiModalProcessor(VoxtralMultiModalProcessor):
54
def __init__(
55
self,
56
info: _I,
@@ -58,7 +58,7 @@ class VoxtralStreamingMultiModalProcessor(VoxtralMultiModalProcessor):
58
*,
59
cache: BaseMultiModalProcessorCache | None = None,
60
) -> None:
61
- # streaming can't make use of a cache yet
61
+ # realtime can't make use of a cache yet
62
super().__init__(info, dummy_inputs, cache=None)
63
64
def _maybe_apply_prompt_updates(
@@ -72,10 +72,10 @@ class VoxtralStreamingMultiModalProcessor(VoxtralMultiModalProcessor):
72
# there are no placeholder audio tokens for streaming
73
# so we need to build the place placeholder positions manually
74
75
- # in streaming there is always only one audio input
75
+ # in realtime there is always only one audio input
76
audios = mm_kwargs.get("audio", [])
77
assert len(audios) == 1, (
78
- f"Expected only one audio input for streaming, got {mm_kwargs=}"
78
+ f"Expected only one audio input for realtime, got {mm_kwargs=}"
79
)
80
tokenizer = self.info.get_tokenizer()
81
audio_config = tokenizer.instruct.audio_encoder.audio_config
@@ -211,12 +211,12 @@ class VoxtralRealtimeBuffer:
211
212
213
@MULTIMODAL_REGISTRY.register_processor(
214
- VoxtralStreamingMultiModalProcessor,
214
+ VoxtralRealtimeMultiModalProcessor,
215
info=VoxtralProcessingInfo,
216
dummy_inputs=VoxtralDummyInputsBuilder,
217
)
218
@support_torch_compile
219
-class VoxtralStreamingGeneration(VoxtralForConditionalGeneration, SupportsRealtime):
219
+class VoxtralRealtimeGeneration(VoxtralForConditionalGeneration, SupportsRealtime):
220
requires_raw_input_tokens = True
221
222
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
@@ -224,10 +224,7 @@ class VoxtralStreamingGeneration(VoxtralForConditionalGeneration, SupportsRealti
224
225
assert (
226
not vllm_config.compilation_config.cudagraph_mode.has_full_cudagraphs()
227
- ), (
228
- "Voxtral streaming doesn't support full cudagraphs yet. "
229
- "Please use PIECEWISE."
230
- )
227
+ ), "Voxtral realtime doesn't support full cudagraphs yet. Please use PIECEWISE."
228
229
self.time_embedding: TimeEmbedding = TimeEmbedding(
230
dim=self.config.text_config.hidden_size
@@ -302,11 +299,11 @@ class VoxtralStreamingGeneration(VoxtralForConditionalGeneration, SupportsRealti
299
handle_oov_mm_token: bool = True,
300
) -> torch.Tensor:
301
"""Pass post-conv embeddings directly as input"""
305
- # for streaming we simply flatten the multimodal embeddings
302
+ # for realtime we simply flatten the multimodal embeddings
303
# to be in tensor format, we treat the input ids later
304
assert multimodal_embeddings is not None
305
assert len(multimodal_embeddings) > 0, (
309
- "For streaming you must provide a multimodal_embedding at every step."
306
+ "For realtime you must provide a multimodal_embedding at every step."
307
)
308
mm_embeds_flat = _flatten_embeddings(multimodal_embeddings)
309
return mm_embeds_flat
@@ -370,7 +367,7 @@ class VoxtralStreamingGeneration(VoxtralForConditionalGeneration, SupportsRealti
367
audio_inputs = self._parse_and_validate_audio_arrays(**kwargs)
368
369
assert audio_inputs is not None, (
373
- "For streaming you must provide an audio input at every step."
370
+ "For realtime you must provide an audio input at every step."
371
)
372
373
def _truncate_left(
vllm/transformers_utils/configs/mistral.py
+1
-1
@@ -204,7 +204,7 @@ def _remap_mistral_audio_args(config: dict) -> dict:
204
raise NotImplementedError(f"Unsupported: {_maybe_sliding_window=}")
205
206
architecture = (
207
- "VoxtralStreamingGeneration"
207
+ "VoxtralRealtimeGeneration"
208
if encoder_args.get("causal")
209
else "VoxtralForConditionalGeneration"
210
)