A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 08:45 UTC
[Realtime API] Adds minimal realtime API based on websockets (#33187)
Signed-off-by: Patrick von Platen <patrick.v.platen@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
Patrick von Platen committed
Jan 30, 2026 at 11:41 UTC
10152d2194c32f6e312c99ed51fd62b01f55598e
21 files changed
+1316
-48
docs/serving/openai_compatible_server.md
+92
@@ -59,6 +59,8 @@ We currently support the following OpenAI APIs:
59
- Only applicable to [Automatic Speech Recognition (ASR) models](../models/supported_models.md#transcription).
60
- [Translation API](#translations-api) (`/v1/audio/translations`)
61
- Only applicable to [Automatic Speech Recognition (ASR) models](../models/supported_models.md#transcription).
62
+- [Realtime API](#realtime-api) (`/v1/realtime`)
63
+ - Only applicable to [Automatic Speech Recognition (ASR) models](../models/supported_models.md#transcription).
64
65
In addition, we have the following custom APIs:
66
@@ -567,6 +569,96 @@ The following extra parameters are supported:
569
--8<-- "vllm/entrypoints/openai/protocol.py:translation-extra-params"
570
```
571
572
+### Realtime API
573
+
574
+The Realtime API provides WebSocket-based streaming audio transcription, allowing real-time speech-to-text as audio is being recorded.
575
+
576
+!!! note
577
+ To use the Realtime API, please install with extra audio dependencies using `uv pip install vllm[audio]`.
578
+
579
+#### Audio Format
580
+
581
+Audio must be sent as base64-encoded PCM16 audio at 16kHz sample rate, mono channel.
582
+
583
+#### Protocol Overview
584
+
585
+1. Client connects to `ws://host/v1/realtime`
586
+2. Server sends `session.created` event
587
+3. Client optionally sends `session.update` with model/params
588
+4. Client sends `input_audio_buffer.commit` when ready
589
+5. Client sends `input_audio_buffer.append` events with base64 PCM16 chunks
590
+6. Server sends `transcription.delta` events with incremental text
591
+7. Server sends `transcription.done` with final text + usage
592
+8. Repeat from step 5 for next utterance
593
+9. Optionally, client sends input_audio_buffer.commit with final=True
594
+ to signal audio input is finished. Useful when streaming audio files
595
+
596
+#### Client → Server Events
597
+
598
+| Event | Description |
599
+|-------|-------------|
600
+| `input_audio_buffer.append` | Send base64-encoded audio chunk: `{"type": "input_audio_buffer.append", "audio": "<base64>"}` |
601
+| `input_audio_buffer.commit` | Trigger transcription processing or end: `{"type": "input_audio_buffer.commit", "final": bool}` |
602
+| `session.update` | Configure session: `{"type": "session.update", "model": "model-name"}` |
603
+
604
+#### Server → Client Events
605
+
606
+| Event | Description |
607
+|-------|-------------|
608
+| `session.created` | Connection established with session ID and timestamp |
609
+| `transcription.delta` | Incremental transcription text: `{"type": "transcription.delta", "delta": "text"}` |
610
+| `transcription.done` | Final transcription with usage stats |
611
+| `error` | Error notification with message and optional code |
612
+
613
+#### Python WebSocket Example
614
+
615
+??? code
616
+
617
+ ```python
618
+ import asyncio
619
+ import base64
620
+ import json
621
+ import websockets
622
+
623
+ async def realtime_transcribe():
624
+ uri = "ws://localhost:8000/v1/realtime"
625
+
626
+ async with websockets.connect(uri) as ws:
627
+ # Wait for session.created
628
+ response = await ws.recv()
629
+ print(f"Session: {response}")
630
+
631
+ # Commit buffer
632
+ await ws.send(json.dumps({
633
+ "type": "input_audio_buffer.commit"
634
+ }))
635
+
636
+ # Send audio chunks (example with file)
637
+ with open("audio.raw", "rb") as f:
638
+ while chunk := f.read(4096):
639
+ await ws.send(json.dumps({
640
+ "type": "input_audio_buffer.append",
641
+ "audio": base64.b64encode(chunk).decode()
642
+ }))
643
+
644
+ # Signal all audio is sent
645
+ await ws.send(json.dumps({
646
+ "type": "input_audio_buffer.commit",
647
+ "final": True,
648
+ }))
649
+
650
+ # Receive transcription
651
+ while True:
652
+ response = json.loads(await ws.recv())
653
+ if response["type"] == "transcription.delta":
654
+ print(response["delta"], end="", flush=True)
655
+ elif response["type"] == "transcription.done":
656
+ print(f"\nFinal: {response['text']}")
657
+ break
658
+
659
+ asyncio.run(realtime_transcribe())
660
+ ```
661
+
662
### Tokenizer API
663
664
Our Tokenizer API is a simple wrapper over [HuggingFace-style tokenizers](https://huggingface.co/docs/transformers/en/main_classes/tokenizer).
examples/online_serving/openai_realtime_client.py
new
+151
@@ -0,0 +1,151 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""
4
+This script demonstrates how to use the vLLM Realtime WebSocket API to perform
5
+audio transcription by uploading an audio file.
6
+
7
+Before running this script, you must start the vLLM server with a realtime-capable
8
+model, for example:
9
+
10
+ vllm serve mistralai/Voxtral-Mini-3B-Realtime-2602 --enforce-eager
11
+
12
+Requirements:
13
+- vllm with audio support
14
+- websockets
15
+- librosa
16
+- numpy
17
+
18
+The script:
19
+1. Connects to the Realtime WebSocket endpoint
20
+2. Converts an audio file to PCM16 @ 16kHz
21
+3. Sends audio chunks to the server
22
+4. Receives and prints transcription as it streams
23
+"""
24
+
25
+import argparse
26
+import asyncio
27
+import base64
28
+import json
29
+
30
+import librosa
31
+import numpy as np
32
+import websockets
33
+
34
+from vllm.assets.audio import AudioAsset
35
+
36
+
37
+def audio_to_pcm16_base64(audio_path: str) -> str:
38
+ """
39
+ Load an audio file and convert it to base64-encoded PCM16 @ 16kHz.
40
+ """
41
+ # Load audio and resample to 16kHz mono
42
+ audio, _ = librosa.load(audio_path, sr=16000, mono=True)
43
+ # Convert to PCM16
44
+ pcm16 = (audio * 32767).astype(np.int16)
45
+ # Encode as base64
46
+ return base64.b64encode(pcm16.tobytes()).decode("utf-8")
47
+
48
+
49
+async def realtime_transcribe(audio_path: str, host: str, port: int, model: str):
50
+ """
51
+ Connect to the Realtime API and transcribe an audio file.
52
+ """
53
+ uri = f"ws://{host}:{port}/v1/realtime"
54
+
55
+ async with websockets.connect(uri) as ws:
56
+ # Wait for session.created
57
+ response = json.loads(await ws.recv())
58
+ if response["type"] == "session.created":
59
+ print(f"Session created: {response['id']}")
60
+ else:
61
+ print(f"Unexpected response: {response}")
62
+ return
63
+
64
+ # Validate model
65
+ await ws.send(json.dumps({"type": "session.update", "model": model}))
66
+
67
+ # Signal ready to start
68
+ await ws.send(json.dumps({"type": "input_audio_buffer.commit"}))
69
+
70
+ # Convert audio file to base64 PCM16
71
+ print(f"Loading audio from: {audio_path}")
72
+ audio_base64 = audio_to_pcm16_base64(audio_path)
73
+
74
+ # Send audio in chunks (4KB of raw audio = ~8KB base64)
75
+ chunk_size = 4096
76
+ audio_bytes = base64.b64decode(audio_base64)
77
+ total_chunks = (len(audio_bytes) + chunk_size - 1) // chunk_size
78
+
79
+ print(f"Sending {total_chunks} audio chunks...")
80
+ for i in range(0, len(audio_bytes), chunk_size):
81
+ chunk = audio_bytes[i : i + chunk_size]
82
+ await ws.send(
83
+ json.dumps(
84
+ {
85
+ "type": "input_audio_buffer.append",
86
+ "audio": base64.b64encode(chunk).decode("utf-8"),
87
+ }
88
+ )
89
+ )
90
+
91
+ # Signal all audio is sent
92
+ await ws.send(json.dumps({"type": "input_audio_buffer.commit", "final": True}))
93
+ print("Audio sent. Waiting for transcription...\n")
94
+
95
+ # Receive transcription
96
+ print("Transcription: ", end="", flush=True)
97
+ while True:
98
+ response = json.loads(await ws.recv())
99
+ if response["type"] == "transcription.delta":
100
+ print(response["delta"], end="", flush=True)
101
+ elif response["type"] == "transcription.done":
102
+ print(f"\n\nFinal transcription: {response['text']}")
103
+ if response.get("usage"):
104
+ print(f"Usage: {response['usage']}")
105
+ break
106
+ elif response["type"] == "error":
107
+ print(f"\nError: {response['error']}")
108
+ break
109
+
110
+
111
+def main(args):
112
+ if args.audio_path:
113
+ audio_path = args.audio_path
114
+ else:
115
+ # Use default audio asset
116
+ audio_path = str(AudioAsset("mary_had_lamb").get_local_path())
117
+ print(f"No audio path provided, using default: {audio_path}")
118
+
119
+ asyncio.run(realtime_transcribe(audio_path, args.host, args.port, args.model))
120
+
121
+
122
+if __name__ == "__main__":
123
+ parser = argparse.ArgumentParser(
124
+ description="Realtime WebSocket Transcription Client"
125
+ )
126
+ parser.add_argument(
127
+ "--model",
128
+ type=str,
129
+ default="mistralai/Voxtral-Mini-3B-Realtime-2602",
130
+ help="Model that is served and should be pinged.",
131
+ )
132
+ parser.add_argument(
133
+ "--audio_path",
134
+ type=str,
135
+ default=None,
136
+ help="Path to the audio file to transcribe.",
137
+ )
138
+ parser.add_argument(
139
+ "--host",
140
+ type=str,
141
+ default="localhost",
142
+ help="vLLM server host (default: localhost)",
143
+ )
144
+ parser.add_argument(
145
+ "--port",
146
+ type=int,
147
+ default=8000,
148
+ help="vLLM server port (default: 8000)",
149
+ )
150
+ args = parser.parse_args()
151
+ main(args)
examples/online_serving/openai_realtime_microphone_client.py
new
+183
@@ -0,0 +1,183 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""
4
+Minimal Gradio demo for real-time speech transcription using the vLLM Realtime API.
5
+
6
+Start the vLLM server first:
7
+
8
+ vllm serve mistralai/Voxtral-Mini-3B-Realtime-2602 --enforce-eager
9
+
10
+Then run this script:
11
+
12
+ python openai_realtime_microphone_client.py --host localhost --port 8000
13
+
14
+Use --share to create a public Gradio link.
15
+
16
+Requirements: websockets, numpy, gradio
17
+"""
18
+
19
+import argparse
20
+import asyncio
21
+import base64
22
+import json
23
+import queue
24
+import threading
25
+
26
+import gradio as gr
27
+import numpy as np
28
+import websockets
29
+
30
+SAMPLE_RATE = 16_000
31
+
32
+# Global state
33
+audio_queue: queue.Queue = queue.Queue()
34
+transcription_text = ""
35
+is_running = False
36
+ws_url = ""
37
+model = ""
38
+
39
+
40
+async def websocket_handler():
41
+ """Connect to WebSocket and handle audio streaming + transcription."""
42
+ global transcription_text, is_running
43
+
44
+ async with websockets.connect(ws_url) as ws:
45
+ # Wait for session.created
46
+ await ws.recv()
47
+
48
+ # Validate model
49
+ await ws.send(json.dumps({"type": "session.update", "model": model}))
50
+
51
+ # Signal ready
52
+ await ws.send(json.dumps({"type": "input_audio_buffer.commit"}))
53
+
54
+ async def send_audio():
55
+ while is_running:
56
+ try:
57
+ chunk = await asyncio.get_event_loop().run_in_executor(
58
+ None, lambda: audio_queue.get(timeout=0.1)
59
+ )
60
+ await ws.send(
61
+ json.dumps(
62
+ {"type": "input_audio_buffer.append", "audio": chunk}
63
+ )
64
+ )
65
+ except queue.Empty:
66
+ continue
67
+
68
+ async def receive_transcription():
69
+ global transcription_text
70
+ async for message in ws:
71
+ data = json.loads(message)
72
+ if data.get("type") == "transcription.delta":
73
+ transcription_text += data["delta"]
74
+
75
+ await asyncio.gather(send_audio(), receive_transcription())
76
+
77
+
78
+def start_websocket():
79
+ """Start WebSocket connection in background thread."""
80
+ global is_running
81
+ is_running = True
82
+ loop = asyncio.new_event_loop()
83
+ asyncio.set_event_loop(loop)
84
+ try:
85
+ loop.run_until_complete(websocket_handler())
86
+ except Exception as e:
87
+ print(f"WebSocket error: {e}")
88
+
89
+
90
+def start_recording():
91
+ """Start the transcription service."""
92
+ global transcription_text
93
+ transcription_text = ""
94
+ thread = threading.Thread(target=start_websocket, daemon=True)
95
+ thread.start()
96
+ return gr.update(interactive=False), gr.update(interactive=True), ""
97
+
98
+
99
+def stop_recording():
100
+ """Stop the transcription service."""
101
+ global is_running
102
+ is_running = False
103
+ return gr.update(interactive=True), gr.update(interactive=False), transcription_text
104
+
105
+
106
+def process_audio(audio):
107
+ """Process incoming audio and queue for streaming."""
108
+ global transcription_text
109
+
110
+ if audio is None or not is_running:
111
+ return transcription_text
112
+
113
+ sample_rate, audio_data = audio
114
+
115
+ # Convert to mono if stereo
116
+ if len(audio_data.shape) > 1:
117
+ audio_data = audio_data.mean(axis=1)
118
+
119
+ # Normalize to float
120
+ if audio_data.dtype == np.int16:
121
+ audio_float = audio_data.astype(np.float32) / 32767.0
122
+ else:
123
+ audio_float = audio_data.astype(np.float32)
124
+
125
+ # Resample to 16kHz if needed
126
+ if sample_rate != SAMPLE_RATE:
127
+ num_samples = int(len(audio_float) * SAMPLE_RATE / sample_rate)
128
+ audio_float = np.interp(
129
+ np.linspace(0, len(audio_float) - 1, num_samples),
130
+ np.arange(len(audio_float)),
131
+ audio_float,
132
+ )
133
+
134
+ # Convert to PCM16 and base64 encode
135
+ pcm16 = (audio_float * 32767).astype(np.int16)
136
+ b64_chunk = base64.b64encode(pcm16.tobytes()).decode("utf-8")
137
+ audio_queue.put(b64_chunk)
138
+
139
+ return transcription_text
140
+
141
+
142
+# Gradio interface
143
+with gr.Blocks(title="Real-time Speech Transcription") as demo:
144
+ gr.Markdown("# Real-time Speech Transcription")
145
+ gr.Markdown("Click **Start** and speak into your microphone.")
146
+
147
+ with gr.Row():
148
+ start_btn = gr.Button("Start", variant="primary")
149
+ stop_btn = gr.Button("Stop", variant="stop", interactive=False)
150
+
151
+ audio_input = gr.Audio(sources=["microphone"], streaming=True, type="numpy")
152
+ transcription_output = gr.Textbox(label="Transcription", lines=5)
153
+
154
+ start_btn.click(
155
+ start_recording, outputs=[start_btn, stop_btn, transcription_output]
156
+ )
157
+ stop_btn.click(stop_recording, outputs=[start_btn, stop_btn, transcription_output])
158
+ audio_input.stream(
159
+ process_audio, inputs=[audio_input], outputs=[transcription_output]
160
+ )
161
+
162
+if __name__ == "__main__":
163
+ parser = argparse.ArgumentParser(
164
+ description="Realtime WebSocket Transcription with Gradio"
165
+ )
166
+ parser.add_argument(
167
+ "--model",
168
+ type=str,
169
+ default="mistralai/Voxtral-Mini-3B-Realtime-2602",
170
+ help="Model that is served and should be pinged.",
171
+ )
172
+ parser.add_argument(
173
+ "--host", type=str, default="localhost", help="vLLM server host"
174
+ )
175
+ parser.add_argument("--port", type=int, default=8000, help="vLLM server port")
176
+ parser.add_argument(
177
+ "--share", action="store_true", help="Create public Gradio link"
178
+ )
179
+ args = parser.parse_args()
180
+
181
+ ws_url = f"ws://{args.host}:{args.port}/v1/realtime"
182
+ model = args.model
183
+ demo.launch(share=args.share)
tests/entrypoints/openai/test_realtime_validation.py
new
+133
@@ -0,0 +1,133 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+import asyncio
5
+import base64
6
+import json
7
+
8
+import librosa
9
+import numpy as np
10
+import pytest
11
+import websockets
12
+
13
+from vllm.assets.audio import AudioAsset
14
+
15
+from ...utils import RemoteOpenAIServer
16
+from .conftest import add_attention_backend
17
+
18
+MISTRAL_FORMAT_ARGS = [
19
+ "--tokenizer_mode",
20
+ "mistral",
21
+ "--config_format",
22
+ "mistral",
23
+ "--load_format",
24
+ "mistral",
25
+]
26
+
27
+MODEL_NAME = "mistralai/Voxtral-Mini-3B-Realtime-2602"
28
+
29
+
30
+def _audio_to_base64_pcm16(path: str, target_sr: int = 16000) -> str:
31
+ """Load audio file, convert to PCM16 @ target sample rate, base64 encode."""
32
+ audio, _ = librosa.load(path, sr=target_sr, mono=True)
33
+ # Convert float32 [-1, 1] to int16 [-32768, 32767]
34
+ audio_int16 = (audio * 32767).astype(np.int16)
35
+ audio_bytes = audio_int16.tobytes()
36
+ return base64.b64encode(audio_bytes).decode("utf-8")
37
+
38
+
39
+def _get_websocket_url(server: RemoteOpenAIServer) -> str:
40
+ """Convert HTTP URL to WebSocket URL for realtime endpoint."""
41
+ http_url = server.url_root
42
+ ws_url = http_url.replace("http://", "ws://")
43
+ return f"{ws_url}/v1/realtime"
44
+
45
+
46
+async def receive_event(ws, timeout: float = 60.0) -> dict:
47
+ """Receive and parse JSON event from WebSocket."""
48
+ message = await asyncio.wait_for(ws.recv(), timeout=timeout)
49
+ return json.loads(message)
50
+
51
+
52
+async def send_event(ws, event: dict) -> None:
53
+ """Send JSON event to WebSocket."""
54
+ await ws.send(json.dumps(event))
55
+
56
+
57
+@pytest.fixture
58
+def mary_had_lamb_audio_chunks() -> list[str]:
59
+ """Audio split into ~1 second chunks for streaming."""
60
+ path = AudioAsset("mary_had_lamb").get_local_path()
61
+ audio, _ = librosa.load(str(path), sr=16000, mono=True)
62
+
63
+ # Split into ~0.1 second chunks (1600 samples at 16kHz)
64
+ chunk_size = 1600
65
+ chunks = []
66
+ for i in range(0, len(audio), chunk_size):
67
+ chunk = audio[i : i + chunk_size]
68
+ chunk_int16 = (chunk * 32767).astype(np.int16)
69
+ chunk_bytes = chunk_int16.tobytes()
70
+ chunks.append(base64.b64encode(chunk_bytes).decode("utf-8"))
71
+
72
+ return chunks
73
+
74
+
75
+@pytest.mark.asyncio
76
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
77
+@pytest.mark.skip(reason="Voxtral streaming is not yet public")
78
+async def test_multi_chunk_streaming(
79
+ model_name, mary_had_lamb_audio_chunks, rocm_aiter_fa_attention
80
+):
81
+ """Test streaming multiple audio chunks before committing."""
82
+ server_args = ["--enforce-eager"]
83
+
84
+ if model_name.startswith("mistralai"):
85
+ server_args += MISTRAL_FORMAT_ARGS
86
+
87
+ add_attention_backend(server_args, rocm_aiter_fa_attention)
88
+
89
+ with RemoteOpenAIServer(model_name, server_args) as remote_server:
90
+ ws_url = _get_websocket_url(remote_server)
91
+ async with websockets.connect(ws_url) as ws:
92
+ # Receive session.created
93
+ event = await receive_event(ws, timeout=30.0)
94
+ assert event["type"] == "session.created"
95
+
96
+ await send_event(ws, {"type": "session.update", "model": model_name})
97
+
98
+ # Send commit to start transcription
99
+ await send_event(ws, {"type": "input_audio_buffer.commit"})
100
+
101
+ # Send multiple audio chunks
102
+ for chunk in mary_had_lamb_audio_chunks:
103
+ await send_event(
104
+ ws, {"type": "input_audio_buffer.append", "audio": chunk}
105
+ )
106
+
107
+ # Send commit to end
108
+ await send_event(ws, {"type": "input_audio_buffer.commit", "final": True})
109
+
110
+ # Collect transcription deltas
111
+ full_text = ""
112
+ done_received = False
113
+
114
+ while not done_received:
115
+ event = await receive_event(ws, timeout=60.0)
116
+
117
+ if event["type"] == "transcription.delta":
118
+ full_text += event["delta"]
119
+ elif event["type"] == "transcription.done":
120
+ done_received = True
121
+ assert "text" in event
122
+ elif event["type"] == "error":
123
+ pytest.fail(f"Received error: {event}")
124
+
125
+ # Verify transcription contains expected content
126
+ assert event["type"] == "transcription.done"
127
+ assert event["text"] == full_text
128
+ assert full_text == (
129
+ " He has first words I spoke in the original phonograph."
130
+ " A little piece of practical poetry. Mary had a little lamb,"
131
+ " it squeaked with quite a flow, and everywhere that Mary went,"
132
+ " the lamb was sure to go"
133
+ )
tests/v1/e2e/test_streaming_input.py
+2
-1
@@ -19,11 +19,12 @@ import pytest
19
import pytest_asyncio
20
21
from vllm import SamplingParams
22
+from vllm.inputs.data import StreamingInput
23
from vllm.outputs import RequestOutput
24
from vllm.platforms import current_platform
25
from vllm.sampling_params import RequestOutputKind
26
from vllm.utils.torch_utils import set_default_torch_num_threads
26
-from vllm.v1.engine.async_llm import AsyncLLM, StreamingInput
27
+from vllm.v1.engine.async_llm import AsyncLLM
28
29
if not current_platform.is_cuda():
30
pytest.skip(reason="V1 currently only supported on CUDA.", allow_module_level=True)
tests/v1/streaming_input/test_async_llm_streaming.py
+2
-1
@@ -7,9 +7,10 @@ from unittest.mock import AsyncMock, MagicMock
7
8
import pytest
9
10
+from vllm.inputs.data import StreamingInput
11
from vllm.outputs import RequestOutput
12
from vllm.sampling_params import RequestOutputKind, SamplingParams
12
-from vllm.v1.engine.async_llm import AsyncLLM, StreamingInput
13
+from vllm.v1.engine.async_llm import AsyncLLM
14
from vllm.v1.engine.output_processor import RequestOutputCollector
15
16
vllm/engine/protocol.py
+2
-2
@@ -6,7 +6,7 @@ from collections.abc import AsyncGenerator, Iterable, Mapping
6
from typing import Any
7
8
from vllm.config import ModelConfig, VllmConfig
9
-from vllm.inputs.data import PromptType
9
+from vllm.inputs.data import PromptType, StreamingInput
10
from vllm.lora.request import LoRARequest
11
from vllm.outputs import PoolingRequestOutput, RequestOutput
12
from vllm.plugins.io_processors import IOProcessor
@@ -49,7 +49,7 @@ class EngineClient(ABC):
49
@abstractmethod
50
def generate(
51
self,
52
- prompt: EngineCoreRequest | PromptType,
52
+ prompt: EngineCoreRequest | PromptType | AsyncGenerator[StreamingInput, None],
53
sampling_params: SamplingParams,
54
request_id: str,
55
*,
vllm/entrypoints/launcher.py
+12
@@ -36,6 +36,7 @@ async def serve_http(
36
h11_max_header_count.
37
"""
38
logger.info("Available routes are:")
39
+ # post endpoints
40
for route in app.routes:
41
methods = getattr(route, "methods", None)
42
path = getattr(route, "path", None)
@@ -45,6 +46,17 @@ async def serve_http(
46
47
logger.info("Route: %s, Methods: %s", path, ", ".join(methods))
48
49
+ # other endpoints
50
+ for route in app.routes:
51
+ endpoint = getattr(route, "endpoint", None)
52
+ methods = getattr(route, "methods", None)
53
+ path = getattr(route, "path", None)
54
+
55
+ if endpoint is None or path is None or methods is not None:
56
+ continue
57
+
58
+ logger.info("Route: %s, Endpoint: %s", path, endpoint.__name__)
59
+
60
# Extract header limit options if present
61
h11_max_incomplete_event_size = uvicorn_kwargs.pop(
62
"h11_max_incomplete_event_size", None
vllm/entrypoints/openai/api_server.py
+12
@@ -196,6 +196,13 @@ def build_app(args: Namespace, supported_tasks: tuple["SupportedTask", ...]) ->
196
197
register_translations_api_router(app)
198
199
+ if "realtime" in supported_tasks:
200
+ from vllm.entrypoints.openai.realtime.api_router import (
201
+ attach_router as register_realtime_api_router,
202
+ )
203
+
204
+ register_realtime_api_router(app)
205
+
206
if any(task in POOLING_TASKS for task in supported_tasks):
207
from vllm.entrypoints.pooling import register_pooling_api_routers
208
@@ -319,6 +326,11 @@ async def init_app_state(
326
engine_client, state, args, request_logger, supported_tasks
327
)
328
329
+ if "realtime" in supported_tasks:
330
+ from vllm.entrypoints.openai.realtime.api_router import init_realtime_state
331
+
332
+ init_realtime_state(engine_client, state, args, request_logger, supported_tasks)
333
+
334
if any(task in POOLING_TASKS for task in supported_tasks):
335
from vllm.entrypoints.pooling import init_pooling_state
336
vllm/entrypoints/openai/realtime/__init__.py
new
+2
@@ -0,0 +1,2 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
vllm/entrypoints/openai/realtime/api_router.py
new
+75
@@ -0,0 +1,75 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+from typing import TYPE_CHECKING
5
+
6
+from fastapi import APIRouter, FastAPI, WebSocket
7
+
8
+from vllm.entrypoints.openai.realtime.connection import RealtimeConnection
9
+from vllm.entrypoints.openai.realtime.serving import OpenAIServingRealtime
10
+from vllm.logger import init_logger
11
+
12
+logger = init_logger(__name__)
13
+
14
+if TYPE_CHECKING:
15
+ from argparse import Namespace
16
+
17
+ from starlette.datastructures import State
18
+
19
+ from vllm.engine.protocol import EngineClient
20
+ from vllm.entrypoints.logger import RequestLogger
21
+ from vllm.tasks import SupportedTask
22
+else:
23
+ RequestLogger = object
24
+
25
+router = APIRouter()
26
+
27
+
28
+@router.websocket("/v1/realtime")
29
+async def realtime_endpoint(websocket: WebSocket):
30
+ """WebSocket endpoint for realtime audio transcription.
31
+
32
+ Protocol:
33
+ 1. Client connects to ws://host/v1/realtime
34
+ 2. Server sends session.created event
35
+ 3. Client optionally sends session.update with model/params
36
+ 4. Client sends input_audio_buffer.commit when ready
37
+ 5. Client sends input_audio_buffer.append events with base64 PCM16 chunks
38
+ 6. Server processes and sends transcription.delta events
39
+ 7. Server sends transcription.done with final text + usage
40
+ 8. Repeat from step 5 for next utterance
41
+ 9. Optionally, client sends input_audio_buffer.commit with final=True
42
+ to signal audio input is finished. Useful when streaming audio files
43
+
44
+ Audio format: PCM16, 16kHz, mono, base64-encoded
45
+ """
46
+ app = websocket.app
47
+ serving = app.state.openai_serving_realtime
48
+
49
+ connection = RealtimeConnection(websocket, serving)
50
+ await connection.handle_connection()
51
+
52
+
53
+def attach_router(app: FastAPI):
54
+ """Attach the realtime router to the FastAPI app."""
55
+ app.include_router(router)
56
+ logger.info("Realtime API router attached")
57
+
58
+
59
+def init_realtime_state(
60
+ engine_client: "EngineClient",
61
+ state: "State",
62
+ args: "Namespace",
63
+ request_logger: RequestLogger | None,
64
+ supported_tasks: tuple["SupportedTask", ...],
65
+):
66
+ state.openai_serving_realtime = (
67
+ OpenAIServingRealtime(
68
+ engine_client,
69
+ state.openai_serving_models,
70
+ request_logger=request_logger,
71
+ log_error_stack=args.log_error_stack,
72
+ )
73
+ if "realtime" in supported_tasks
74
+ else None
75
+ )
vllm/entrypoints/openai/realtime/connection.py
new
+285
@@ -0,0 +1,285 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+import asyncio
5
+import base64
6
+import json
7
+from collections.abc import AsyncGenerator
8
+from http import HTTPStatus
9
+from uuid import uuid4
10
+
11
+import numpy as np
12
+from fastapi import WebSocket
13
+from starlette.websockets import WebSocketDisconnect
14
+
15
+from vllm import envs
16
+from vllm.entrypoints.openai.engine.protocol import ErrorResponse, UsageInfo
17
+from vllm.entrypoints.openai.realtime.protocol import (
18
+ ErrorEvent,
19
+ InputAudioBufferAppend,
20
+ InputAudioBufferCommit,
21
+ SessionCreated,
22
+ TranscriptionDelta,
23
+ TranscriptionDone,
24
+)
25
+from vllm.entrypoints.openai.realtime.serving import OpenAIServingRealtime
26
+from vllm.exceptions import VLLMValidationError
27
+from vllm.logger import init_logger
28
+
29
+logger = init_logger(__name__)
30
+
31
+
32
+class RealtimeConnection:
33
+ """Manages WebSocket lifecycle and state for realtime transcription.
34
+
35
+ This class handles:
36
+ - WebSocket connection lifecycle (accept, receive, send, close)
37
+ - Event routing (session.update, append, commit)
38
+ - Audio buffering via asyncio.Queue
39
+ - Generation task management
40
+ - Error handling and cleanup
41
+ """
42
+
43
+ def __init__(self, websocket: WebSocket, serving: OpenAIServingRealtime):
44
+ self.websocket = websocket
45
+ self.connection_id = f"ws-{uuid4()}"
46
+ self.serving = serving
47
+ self.audio_queue: asyncio.Queue[np.ndarray | None] = asyncio.Queue()
48
+ self.generation_task: asyncio.Task | None = None
49
+
50
+ self._is_connected = False
51
+ self._is_input_finished = False
52
+ self._is_model_validated = False
53
+
54
+ self._max_audio_filesize_mb = envs.VLLM_MAX_AUDIO_CLIP_FILESIZE_MB
55
+
56
+ async def handle_connection(self):
57
+ """Main connection loop."""
58
+ await self.websocket.accept()
59
+ logger.debug("WebSocket connection accepted: %s", self.connection_id)
60
+ self._is_connected = True
61
+
62
+ # Send session created event
63
+ await self.send(SessionCreated())
64
+
65
+ try:
66
+ while True:
67
+ message = await self.websocket.receive_text()
68
+ try:
69
+ event = json.loads(message)
70
+ await self.handle_event(event)
71
+ except json.JSONDecodeError:
72
+ await self.send_error("Invalid JSON", "invalid_json")
73
+ except Exception as e:
74
+ logger.exception("Error handling event: %s", e)
75
+ await self.send_error(str(e), "processing_error")
76
+ except WebSocketDisconnect:
77
+ logger.debug("WebSocket disconnected: %s", self.connection_id)
78
+ self._is_connected = False
79
+ except Exception as e:
80
+ logger.exception("Unexpected error in connection: %s", e)
81
+ finally:
82
+ await self.cleanup()
83
+
84
+ def _check_model(self, model: str | None) -> None | ErrorResponse:
85
+ if self.serving._is_model_supported(model):
86
+ return None
87
+
88
+ return self.serving.create_error_response(
89
+ message=f"The model `{model}` does not exist.",
90
+ err_type="NotFoundError",
91
+ status_code=HTTPStatus.NOT_FOUND,
92
+ param="model",
93
+ )
94
+
95
+ async def handle_event(self, event: dict):
96
+ """Route events to handlers.
97
+
98
+ Supported event types:
99
+ - session.update: Configure model
100
+ - input_audio_buffer.append: Add audio chunk to queue
101
+ - input_audio_buffer.commit: Start transcription generation
102
+ """
103
+ event_type = event.get("type")
104
+ if event_type == "session.update":
105
+ logger.debug("Session updated: %s", event)
106
+ self._check_model(event["model"])
107
+ self._is_model_validated = True
108
+ elif event_type == "input_audio_buffer.append":
109
+ append_event = InputAudioBufferAppend(**event)
110
+ try:
111
+ audio_bytes = base64.b64decode(append_event.audio)
112
+ # Convert PCM16 bytes to float32 numpy array
113
+ audio_array = (
114
+ np.frombuffer(audio_bytes, dtype=np.int16).astype(np.float32)
115
+ / 32768.0
116
+ )
117
+
118
+ if len(audio_array) / 1024**2 > self._max_audio_filesize_mb:
119
+ raise VLLMValidationError(
120
+ "Maximum file size exceeded",
121
+ parameter="audio_filesize_mb",
122
+ value=len(audio_array) / 1024**2,
123
+ )
124
+ if len(audio_array) == 0:
125
+ raise VLLMValidationError("Can't process empty audio.")
126
+
127
+ # Put audio chunk in queue
128
+ self.audio_queue.put_nowait(audio_array)
129
+
130
+ except Exception as e:
131
+ logger.error("Failed to decode audio: %s", e)
132
+ await self.send_error("Invalid audio data", "invalid_audio")
133
+
134
+ elif event_type == "input_audio_buffer.commit":
135
+ if not self._is_model_validated:
136
+ err_msg = (
137
+ "Model not validated. Make sure to validate the"
138
+ " model by sending a session.update event."
139
+ )
140
+ await self.send_error(
141
+ err_msg,
142
+ "model_not_validated",
143
+ )
144
+
145
+ commit_event = InputAudioBufferCommit(**event)
146
+ # final signals that the audio is finished
147
+ if commit_event.final:
148
+ self._is_input_finished = True
149
+ else:
150
+ await self.start_generation()
151
+ else:
152
+ await self.send_error(f"Unknown event type: {event_type}", "unknown_event")
153
+
154
+ async def audio_stream_generator(self) -> AsyncGenerator[np.ndarray, None]:
155
+ """Generator that yields audio chunks from the queue."""
156
+ while True:
157
+ audio_chunk = await self.audio_queue.get()
158
+ if audio_chunk is None: # Sentinel value to stop
159
+ break
160
+ yield audio_chunk
161
+
162
+ async def start_generation(self):
163
+ """Start the transcription generation task."""
164
+ if self.generation_task is not None and not self.generation_task.done():
165
+ logger.warning("Generation already in progress, ignoring commit")
166
+ return
167
+
168
+ # Create audio stream generator
169
+ audio_stream = self.audio_stream_generator()
170
+ input_stream = asyncio.Queue[list[int]]()
171
+
172
+ # Transform to StreamingInput generator
173
+ streaming_input_gen = self.serving.transcribe_realtime(
174
+ audio_stream, input_stream
175
+ )
176
+
177
+ # Start generation task
178
+ self.generation_task = asyncio.create_task(
179
+ self._run_generation(streaming_input_gen, input_stream)
180
+ )
181
+
182
+ async def _run_generation(
183
+ self,
184
+ streaming_input_gen: AsyncGenerator,
185
+ input_stream: asyncio.Queue[list[int]],
186
+ ):
187
+ """Run the generation and stream results back to the client.
188
+
189
+ This method:
190
+ 1. Creates sampling parameters from session config
191
+ 2. Passes the streaming input generator to engine.generate()
192
+ 3. Streams transcription.delta events as text is generated
193
+ 4. Sends final transcription.done event with usage stats
194
+ 5. Feeds generated token IDs back to input_stream for next iteration
195
+ 6. Cleans up the audio queue
196
+ """
197
+ request_id = f"rt-{self.connection_id}-{uuid4()}"
198
+ full_text = ""
199
+
200
+ prompt_token_ids_len: int = 0
201
+ completion_tokens_len: int = 0
202
+
203
+ try:
204
+ # Create sampling params
205
+ from vllm.sampling_params import RequestOutputKind, SamplingParams
206
+
207
+ sampling_params = SamplingParams.from_optional(
208
+ temperature=0.0,
209
+ max_tokens=1,
210
+ output_kind=RequestOutputKind.DELTA,
211
+ skip_clone=True,
212
+ )
213
+
214
+ # Pass the streaming input generator to the engine
215
+ # The engine will consume audio chunks as they arrive and
216
+ # stream back transcription results incrementally
217
+ result_gen = self.serving.engine_client.generate(
218
+ prompt=streaming_input_gen,
219
+ sampling_params=sampling_params,
220
+ request_id=request_id,
221
+ )
222
+
223
+ # Stream results back to client as they're generated
224
+ async for output in result_gen:
225
+ if output.outputs and len(output.outputs) > 0:
226
+ if not prompt_token_ids_len and output.prompt_token_ids:
227
+ prompt_token_ids_len = len(output.prompt_token_ids)
228
+
229
+ delta = output.outputs[0].text
230
+ full_text += delta
231
+
232
+ # append output to input
233
+ input_stream.put_nowait(list(output.outputs[0].token_ids))
234
+ await self.send(TranscriptionDelta(delta=delta))
235
+
236
+ completion_tokens_len += len(output.outputs[0].token_ids)
237
+
238
+ if not self._is_connected:
239
+ # finish because websocket connection was killed
240
+ break
241
+
242
+ if self.audio_queue.empty() and self._is_input_finished:
243
+ # finish because client signals that audio input
244
+ # is finished
245
+ break
246
+
247
+ usage = UsageInfo(
248
+ prompt_tokens=prompt_token_ids_len,
249
+ completion_tokens=completion_tokens_len,
250
+ total_tokens=prompt_token_ids_len + completion_tokens_len,
251
+ )
252
+
253
+ # Send final completion event
254
+ await self.send(TranscriptionDone(text=full_text, usage=usage))
255
+
256
+ # Clear queue for next utterance
257
+ while not self.audio_queue.empty():
258
+ self.audio_queue.get_nowait()
259
+
260
+ except Exception as e:
261
+ logger.exception("Error in generation: %s", e)
262
+ await self.send_error(str(e), "processing_error")
263
+
264
+ async def send(
265
+ self, event: SessionCreated | TranscriptionDelta | TranscriptionDone
266
+ ):
267
+ """Send event to client."""
268
+ data = event.model_dump_json()
269
+ await self.websocket.send_text(data)
270
+
271
+ async def send_error(self, message: str, code: str | None = None):
272
+ """Send error event to client."""
273
+ error_event = ErrorEvent(error=message, code=code)
274
+ await self.websocket.send_text(error_event.model_dump_json())
275
+
276
+ async def cleanup(self):
277
+ """Cleanup resources."""
278
+ # Signal audio stream to stop
279
+ self.audio_queue.put_nowait(None)
280
+
281
+ # Cancel generation task if running
282
+ if self.generation_task and not self.generation_task.done():
283
+ self.generation_task.cancel()
284
+
285
+ logger.debug("Connection cleanup complete: %s", self.connection_id)
vllm/entrypoints/openai/realtime/protocol.py
new
+68
@@ -0,0 +1,68 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+import time
5
+from typing import Literal
6
+
7
+from pydantic import Field
8
+
9
+from vllm.entrypoints.openai.engine.protocol import (
10
+ OpenAIBaseModel,
11
+ UsageInfo,
12
+)
13
+from vllm.utils import random_uuid
14
+
15
+# Client -> Server Events
16
+
17
+
18
+class InputAudioBufferAppend(OpenAIBaseModel):
19
+ """Append audio chunk to buffer"""
20
+
21
+ type: Literal["input_audio_buffer.append"] = "input_audio_buffer.append"
22
+ audio: str # base64-encoded PCM16 @ 16kHz
23
+
24
+
25
+class InputAudioBufferCommit(OpenAIBaseModel):
26
+ """Process accumulated audio buffer"""
27
+
28
+ type: Literal["input_audio_buffer.commit"] = "input_audio_buffer.commit"
29
+ final: bool = False
30
+
31
+
32
+# Server -> Client Events
33
+class SessionUpdate(OpenAIBaseModel):
34
+ """Configure session parameters"""
35
+
36
+ type: Literal["session.update"] = "session.update"
37
+ model: str | None = None
38
+
39
+
40
+class SessionCreated(OpenAIBaseModel):
41
+ """Connection established notification"""
42
+
43
+ type: Literal["session.created"] = "session.created"
44
+ id: str = Field(default_factory=lambda: f"sess-{random_uuid()}")
45
+ created: int = Field(default_factory=lambda: int(time.time()))
46
+
47
+
48
+class TranscriptionDelta(OpenAIBaseModel):
49
+ """Incremental transcription text"""
50
+
51
+ type: Literal["transcription.delta"] = "transcription.delta"
52
+ delta: str # Incremental text
53
+
54
+
55
+class TranscriptionDone(OpenAIBaseModel):
56
+ """Final transcription with usage stats"""
57
+
58
+ type: Literal["transcription.done"] = "transcription.done"
59
+ text: str # Complete transcription
60
+ usage: UsageInfo | None = None
61
+
62
+
63
+class ErrorEvent(OpenAIBaseModel):
64
+ """Error notification"""
65
+
66
+ type: Literal["error"] = "error"
67
+ error: str
68
+ code: str | None = None
vllm/entrypoints/openai/realtime/serving.py
new
+84
@@ -0,0 +1,84 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+import asyncio
5
+from collections.abc import AsyncGenerator
6
+from functools import cached_property
7
+from typing import Literal, cast
8
+
9
+import numpy as np
10
+
11
+from vllm.engine.protocol import EngineClient
12
+from vllm.entrypoints.logger import RequestLogger
13
+from vllm.entrypoints.openai.engine.serving import OpenAIServing
14
+from vllm.entrypoints.openai.models.serving import OpenAIServingModels
15
+from vllm.inputs.data import PromptType, StreamingInput
16
+from vllm.logger import init_logger
17
+from vllm.model_executor.models.interfaces import SupportsRealtime
18
+
19
+logger = init_logger(__name__)
20
+
21
+
22
+class OpenAIServingRealtime(OpenAIServing):
23
+ """Realtime audio transcription service via WebSocket streaming.
24
+
25
+ Provides streaming audio-to-text transcription by transforming audio chunks
26
+ into StreamingInput objects that can be consumed by the engine.
27
+ """
28
+
29
+ def __init__(
30
+ self,
31
+ engine_client: EngineClient,
32
+ models: OpenAIServingModels,
33
+ *,
34
+ request_logger: RequestLogger | None,
35
+ log_error_stack: bool = False,
36
+ ):
37
+ super().__init__(
38
+ engine_client=engine_client,
39
+ models=models,
40
+ request_logger=request_logger,
41
+ log_error_stack=log_error_stack,
42
+ )
43
+
44
+ self.task_type: Literal["realtime"] = "realtime"
45
+
46
+ logger.info("OpenAIServingRealtime initialized for task: %s", self.task_type)
47
+
48
+ @cached_property
49
+ def model_cls(self) -> type[SupportsRealtime]:
50
+ """Get the model class that supports transcription."""
51
+ from vllm.model_executor.model_loader import get_model_cls
52
+
53
+ model_cls = get_model_cls(self.model_config)
54
+ return cast(type[SupportsRealtime], model_cls)
55
+
56
+ async def transcribe_realtime(
57
+ self,
58
+ audio_stream: AsyncGenerator[np.ndarray, None],
59
+ input_stream: asyncio.Queue[list[int]],
60
+ ) -> AsyncGenerator[StreamingInput, None]:
61
+ """Transform audio stream into StreamingInput for engine.generate().
62
+
63
+ Args:
64
+ audio_stream: Async generator yielding float32 numpy audio arrays
65
+ input_stream: Queue containing context token IDs from previous
66
+ generation outputs. Used for autoregressive multi-turn
67
+ processing where each generation's output becomes the context
68
+ for the next iteration.
69
+
70
+ Yields:
71
+ StreamingInput objects containing audio prompts for the engine
72
+ """
73
+
74
+ # mypy is being stupid
75
+ # TODO(Patrick) - fix this
76
+ stream_input_iter = cast(
77
+ AsyncGenerator[PromptType, None],
78
+ self.model_cls.buffer_realtime_audio(
79
+ audio_stream, input_stream, self.model_config
80
+ ),
81
+ )
82
+
83
+ async for prompt in stream_input_iter:
84
+ yield StreamingInput(prompt=prompt)
vllm/inputs/data.py
+15
@@ -1,11 +1,14 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
from collections.abc import Iterable
4
+from dataclasses import dataclass
5
from typing import TYPE_CHECKING, Any, Generic, Literal, TypeAlias, cast
6
7
import torch
8
from typing_extensions import NotRequired, TypedDict, TypeIs, TypeVar
9
10
+from vllm.sampling_params import SamplingParams
11
+
12
if TYPE_CHECKING:
13
from vllm.multimodal.inputs import (
14
MultiModalDataDict,
@@ -357,3 +360,15 @@ def to_enc_dec_tuple_list(
360
(enc_dec_prompt["encoder_prompt"], enc_dec_prompt["decoder_prompt"])
361
for enc_dec_prompt in enc_dec_prompts
362
]
363
+
364
+
365
+@dataclass
366
+class StreamingInput:
367
+ """Input data for a streaming generation request.
368
+
369
+ This is used with generate() to support multi-turn streaming sessions
370
+ where inputs are provided via an async generator.
371
+ """
372
+
373
+ prompt: PromptType
374
+ sampling_params: SamplingParams | None = None
vllm/model_executor/models/interfaces.py
+33
-1
@@ -1,7 +1,8 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
-from collections.abc import Callable, Iterable, Mapping, MutableSequence
4
+import asyncio
5
+from collections.abc import AsyncGenerator, Callable, Iterable, Mapping, MutableSequence
6
from contextlib import ExitStack, contextmanager, nullcontext
7
from typing import (
8
TYPE_CHECKING,
@@ -1015,6 +1016,37 @@ class SupportsQuant:
1016
return None
1017
1018
1019
+@runtime_checkable
1020
+class SupportsRealtime(Protocol):
1021
+ """The interface required for all models that support transcription."""
1022
+
1023
+ supports_realtime: ClassVar[Literal[True]] = True
1024
+
1025
+ @classmethod
1026
+ async def buffer_realtime_audio(
1027
+ cls,
1028
+ audio_stream: AsyncGenerator[np.ndarray, None],
1029
+ input_stream: asyncio.Queue[list[int]],
1030
+ model_config: ModelConfig,
1031
+ ) -> AsyncGenerator[PromptType, None]: ...
1032
+
1033
+
1034
+@overload
1035
+def supports_realtime(
1036
+ model: type[object],
1037
+) -> TypeIs[type[SupportsRealtime]]: ...
1038
+
1039
+
1040
+@overload
1041
+def supports_realtime(model: object) -> TypeIs[SupportsRealtime]: ...
1042
+
1043
+
1044
+def supports_realtime(
1045
+ model: type[object] | object,
1046
+) -> TypeIs[type[SupportsRealtime]] | TypeIs[SupportsRealtime]:
1047
+ return getattr(model, "supports_realtime", False)
1048
+
1049
+
1050
@runtime_checkable
1051
class SupportsTranscription(Protocol):
1052
"""The interface required for all models that support transcription."""
vllm/model_executor/models/voxtral.py
+4
-15
@@ -1,7 +1,6 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
-import inspect
4
import math
5
from collections.abc import Iterable, Mapping, Sequence
6
from functools import cached_property, partial
@@ -20,7 +19,6 @@ from mistral_common.protocol.transcription.request import TranscriptionRequest
19
from mistral_common.tokens.tokenizers.audio import (
20
Audio,
21
AudioEncoder,
23
- TranscriptionFormat,
22
)
23
from transformers import BatchFeature, TensorType, WhisperConfig
24
from transformers.tokenization_utils_base import TextInput
@@ -163,19 +161,10 @@ class VoxtralProcessorAdapter:
161
assert isinstance(audio, np.ndarray)
162
assert audio.ndim == 1
163
166
- # pad if necessary
167
- # TODO(Patrick) - remove once mistral-common is bumped
168
- if (
169
- self._audio_processor.audio_config.transcription_format
170
- != TranscriptionFormat.STREAMING
171
- ):
172
- sig = inspect.signature(self._audio_processor.pad)
173
- if "is_online_streaming" in sig.parameters:
174
- audio = self._audio_processor.pad(
175
- audio, self.sampling_rate, is_online_streaming=False
176
- )
177
- else:
178
- audio = self._audio_processor.pad(audio, self.sampling_rate)
164
+ if not self._audio_processor.audio_config.is_streaming:
165
+ audio = self._audio_processor.pad(
166
+ audio, self.sampling_rate, is_online_streaming=False
167
+ )
168
169
audio_tokens = [self.begin_audio_token_id] + [
170
self.audio_token_id
vllm/model_executor/models/voxtral_streaming.py
+155
-14
@@ -1,8 +1,9 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
+import asyncio
5
import math
5
-from collections.abc import Mapping
6
+from collections.abc import AsyncGenerator, Mapping
7
from typing import Literal, cast
8
9
import numpy as np
@@ -12,12 +13,14 @@ from mistral_common.protocol.transcription.request import (
13
StreamingMode,
14
TranscriptionRequest,
15
)
15
-from mistral_common.tokens.tokenizers.audio import Audio
16
+from mistral_common.tokens.tokenizers.audio import Audio, AudioConfig
17
18
+from vllm.compilation.decorators import support_torch_compile
19
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
18
-from vllm.inputs.data import PromptType
20
+from vllm.envs import VLLM_ENGINE_ITERATION_TIMEOUT_S
21
+from vllm.inputs.data import PromptType, TokensPrompt
22
from vllm.logger import init_logger
20
-from vllm.model_executor.models.interfaces import MultiModalEmbeddings
23
+from vllm.model_executor.models.interfaces import MultiModalEmbeddings, SupportsRealtime
24
from vllm.model_executor.models.voxtral import (
25
VoxtralDummyInputsBuilder,
26
VoxtralForConditionalGeneration,
@@ -44,6 +47,8 @@ from .utils import (
47
48
logger = init_logger(__name__)
49
50
+_PRE_ALLOCATE_BUFFER_SIZE_IN_S = 30
51
+
52
53
class VoxtralStreamingMultiModalProcessor(VoxtralMultiModalProcessor):
54
def __init__(
@@ -124,29 +129,164 @@ def _expand_tensor(input_tensor: torch.Tensor, scaling: int) -> torch.Tensor:
129
return (base.unsqueeze(1) + offsets).view(-1)
130
131
132
+class VoxtralRealtimeBuffer:
133
+ def __init__(self, config: AudioConfig) -> None:
134
+ self._config = config
135
+
136
+ self._look_ahead_in_ms = config.streaming_look_ahead_ms
137
+ self._look_back_in_ms = config.streaming_look_back_ms
138
+
139
+ self._sampling_rate = self._config.sampling_rate
140
+
141
+ self._look_ahead = self._get_len_in_samples(self._look_ahead_in_ms)
142
+ self._look_back = self._get_len_in_samples(self._look_back_in_ms)
143
+ self._streaming_size = self._get_len_in_samples(1000 / self._config.frame_rate)
144
+
145
+ # mutable objects
146
+ streaming_delay = self._get_len_in_samples(self._config.transcription_delay_ms)
147
+ self._start = 0
148
+ self._end = streaming_delay + self._streaming_size
149
+
150
+ # always pre-allocate 30 second buffers
151
+ self._buffer_size = _PRE_ALLOCATE_BUFFER_SIZE_IN_S * self._sampling_rate
152
+ self._buffer: np.ndarray = np.empty(self._buffer_size, dtype=np.float32)
153
+ self._filled_buffer_len = 0
154
+
155
+ @property
156
+ def start_idx(self):
157
+ return max(self._start - self._look_back, 0)
158
+
159
+ @property
160
+ def end_idx(self):
161
+ return self._end + self._look_ahead
162
+
163
+ @property
164
+ def is_audio_complete(self) -> bool:
165
+ return self._filled_buffer_len >= self.end_idx
166
+
167
+ def _get_len_in_samples(self, len_in_ms: float) -> int:
168
+ _len_in_s = self._sampling_rate * len_in_ms / 1000
169
+ assert _len_in_s.is_integer(), _len_in_s
170
+ len_in_s = int(_len_in_s)
171
+
172
+ return len_in_s
173
+
174
+ def _allocate_new_buffer(self) -> None:
175
+ # allocate new buffer
176
+ new_buffer = np.empty(self._buffer_size, dtype=np.float32)
177
+ left_to_copy = max(self._filled_buffer_len - self.start_idx, 0)
178
+
179
+ if left_to_copy > 0:
180
+ new_buffer[:left_to_copy] = self._buffer[
181
+ self.start_idx : self._filled_buffer_len
182
+ ]
183
+
184
+ del self._buffer
185
+ self._buffer = new_buffer
186
+
187
+ self._filled_buffer_len = left_to_copy
188
+ self._start = self._look_back
189
+ self._end = self._start + self._streaming_size
190
+
191
+ def write_audio(self, audio: np.ndarray) -> None:
192
+ put_end_idx = self._filled_buffer_len + len(audio)
193
+
194
+ if put_end_idx > self._buffer_size:
195
+ self._allocate_new_buffer()
196
+
197
+ self._buffer[self._filled_buffer_len : self._filled_buffer_len + len(audio)] = (
198
+ audio
199
+ )
200
+ self._filled_buffer_len += len(audio)
201
+
202
+ def read_audio(self) -> np.ndarray | None:
203
+ if not self.is_audio_complete:
204
+ return None
205
+
206
+ audio = self._buffer[self.start_idx : self.end_idx]
207
+ self._start = self._end
208
+ self._end += self._streaming_size
209
+
210
+ return audio
211
+
212
+
213
@MULTIMODAL_REGISTRY.register_processor(
214
VoxtralStreamingMultiModalProcessor,
215
info=VoxtralProcessingInfo,
216
dummy_inputs=VoxtralDummyInputsBuilder,
217
)
132
-class VoxtralStreamingGeneration(VoxtralForConditionalGeneration):
218
+@support_torch_compile
219
+class VoxtralStreamingGeneration(VoxtralForConditionalGeneration, SupportsRealtime):
220
requires_raw_input_tokens = True
221
222
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
223
super().__init__(vllm_config=vllm_config, prefix=prefix)
224
+
225
+ assert (
226
+ not vllm_config.compilation_config.cudagraph_mode.has_full_cudagraphs()
227
+ ), (
228
+ "Voxtral streaming doesn't support full cudagraphs yet. "
229
+ "Please use PIECEWISE."
230
+ )
231
+
232
self.time_embedding: TimeEmbedding = TimeEmbedding(
233
dim=self.config.text_config.hidden_size
234
)
235
236
audio_config = self.tokenizer.instruct.audio_encoder.audio_config
142
- _n_delay_tokens = (
143
- audio_config.frame_rate * audio_config.transcription_delay_ms / 1000
144
- )
145
- assert _n_delay_tokens.is_integer(), (
146
- f"n_delay_tokens must be integer, got {_n_delay_tokens}"
147
- )
237
+ self.n_delay_tokens = audio_config.num_delay_tokens
238
149
- self.n_delay_tokens = int(_n_delay_tokens)
239
+ # for realtime transcription
240
+ @classmethod
241
+ async def buffer_realtime_audio(
242
+ cls,
243
+ audio_stream: AsyncGenerator[np.ndarray, None],
244
+ input_stream: asyncio.Queue[list[int]],
245
+ model_config: ModelConfig,
246
+ ) -> AsyncGenerator[PromptType, None]:
247
+ tokenizer = cached_tokenizer_from_config(model_config)
248
+ audio_encoder = tokenizer.instruct.audio_encoder
249
+ config = audio_encoder.audio_config
250
+
251
+ buffer = VoxtralRealtimeBuffer(config)
252
+ is_first_yield = True
253
+
254
+ async for audio in audio_stream:
255
+ buffer.write_audio(audio)
256
+
257
+ while (new_audio := buffer.read_audio()) is not None:
258
+ if is_first_yield:
259
+ # make sure that input_stream is empty
260
+ assert input_stream.empty()
261
+
262
+ audio = Audio(new_audio, config.sampling_rate, format="wav")
263
+
264
+ request = TranscriptionRequest(
265
+ streaming=StreamingMode.ONLINE,
266
+ audio=RawAudio.from_audio(audio),
267
+ language=None,
268
+ )
269
+ # mistral tokenizer takes care
270
+ # of preparing the first prompt inputs
271
+ # and does some left-silence padding
272
+ # for improved performance
273
+ audio_enc = tokenizer.mistral.encode_transcription(request)
274
+
275
+ token_ids = audio_enc.tokens
276
+ new_audio = audio_enc.audios[0].audio_array
277
+
278
+ is_first_yield = False
279
+ else:
280
+ # pop last element from input_stream
281
+ all_outputs = await asyncio.wait_for(
282
+ input_stream.get(), timeout=VLLM_ENGINE_ITERATION_TIMEOUT_S
283
+ )
284
+ token_ids = all_outputs[-1:]
285
+
286
+ multi_modal_data = {"audio": (new_audio, None)}
287
+ yield TokensPrompt(
288
+ prompt_token_ids=token_ids, multi_modal_data=multi_modal_data
289
+ )
290
291
@property
292
def audio_config(self):
@@ -205,8 +345,9 @@ class VoxtralStreamingGeneration(VoxtralForConditionalGeneration):
345
# sum pool text and audio embeddings
346
inputs_embeds = audio_text_embeds + text_embeds
347
208
- time_tensor = torch.tensor(
209
- [self.n_delay_tokens],
348
+ time_tensor = torch.full(
349
+ (1,),
350
+ fill_value=self.n_delay_tokens,
351
device=inputs_embeds.device,
352
dtype=inputs_embeds.dtype,
353
)
vllm/tasks.py
+1
-1
@@ -2,7 +2,7 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
from typing import Literal, get_args
4
5
-GenerationTask = Literal["generate", "transcription"]
5
+GenerationTask = Literal["generate", "transcription", "realtime"]
6
GENERATION_TASKS: tuple[GenerationTask, ...] = get_args(GenerationTask)
7
8
PoolingTask = Literal[
vllm/v1/engine/async_llm.py
+1
-13
@@ -7,7 +7,6 @@ import time
7
import warnings
8
from collections.abc import AsyncGenerator, Iterable, Mapping
9
from copy import copy
10
-from dataclasses import dataclass
10
from typing import Any
11
12
import torch
@@ -19,6 +18,7 @@ from vllm.engine.arg_utils import AsyncEngineArgs
18
from vllm.engine.protocol import EngineClient
19
from vllm.entrypoints.utils import _validate_truncation_size
20
from vllm.inputs import PromptType
21
+from vllm.inputs.data import StreamingInput
22
from vllm.logger import init_logger
23
from vllm.lora.request import LoRARequest
24
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
@@ -53,18 +53,6 @@ from vllm.v1.metrics.stats import IterationStats
53
logger = init_logger(__name__)
54
55
56
-@dataclass
57
-class StreamingInput:
58
- """Input data for a streaming generation request.
59
-
60
- This is used with generate() to support multi-turn streaming sessions
61
- where inputs are provided via an async generator.
62
- """
63
-
64
- prompt: PromptType
65
- sampling_params: SamplingParams | None = None
66
-
67
-
56
class InputStreamError(Exception):
57
"""Wrapper for errors from the input stream generator.
58
vllm/v1/worker/gpu_model_runner.py
+4
@@ -68,6 +68,7 @@ from vllm.model_executor.models.interfaces import (
68
supports_eagle3,
69
supports_mrope,
70
supports_multimodal_pruning,
71
+ supports_realtime,
72
supports_transcription,
73
supports_xdrope,
74
)
@@ -2541,6 +2542,9 @@ class GPUModelRunner(
2542
2543
supported_tasks.append("transcription")
2544
2545
+ if supports_realtime(model):
2546
+ supported_tasks.append("realtime")
2547
+
2548
return supported_tasks
2549
2550
def get_supported_pooling_tasks(self) -> list[PoolingTask]: