A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 01:15 UTC
[Voxtral Realtime] Fix engine crash on empty multimodal embeddings (#34862)
Signed-off-by: Tal Nir <tal@nervexneurotech.com> Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
Tal Nir committed
Feb 19, 2026 at 02:21 UTC
f75b61a9e9dfc15d7821b35b1f88f9482805202a
2 files changed
+101
-10
tests/entrypoints/openai/test_realtime_validation.py
+72
@@ -121,3 +121,75 @@ async def test_multi_chunk_streaming(
121
" it sleeps with quite a flow, and everywhere that Mary went,"
122
" the lamb was sure to go."
123
)
124
+
125
+
126
+@pytest.mark.asyncio
127
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
128
+async def test_empty_commit_does_not_crash_engine(
129
+ model_name, mary_had_lamb_audio_chunks, rocm_aiter_fa_attention
130
+):
131
+ """Test that committing without audio does not crash the engine.
132
+
133
+ Regression test for https://github.com/vllm-project/vllm/issues/34532.
134
+ An empty commit (no prior input_audio_buffer.append) used to trigger
135
+ ``AssertionError: For realtime you must provide a multimodal_embedding
136
+ at every step`` which killed the entire engine process, disconnecting
137
+ every connected client.
138
+ """
139
+ server_args = ["--enforce-eager", "--max-model-len", "2048"]
140
+
141
+ if model_name.startswith("mistralai"):
142
+ server_args += MISTRAL_FORMAT_ARGS
143
+
144
+ add_attention_backend(server_args, rocm_aiter_fa_attention)
145
+
146
+ with RemoteOpenAIServer(model_name, server_args) as remote_server:
147
+ ws_url = _get_websocket_url(remote_server)
148
+
149
+ # --- First connection: empty commit (no audio appended) ----------
150
+ async with websockets.connect(ws_url) as ws:
151
+ event = await receive_event(ws, timeout=30.0)
152
+ assert event["type"] == "session.created"
153
+
154
+ await send_event(ws, {"type": "session.update", "model": model_name})
155
+
156
+ # Start generation without sending any audio
157
+ await send_event(ws, {"type": "input_audio_buffer.commit"})
158
+
159
+ # Immediately signal end-of-audio
160
+ await send_event(ws, {"type": "input_audio_buffer.commit", "final": True})
161
+
162
+ # We should get *some* response (error or empty transcription),
163
+ # but the engine must NOT crash.
164
+ event = await receive_event(ws, timeout=30.0)
165
+ assert event["type"] in (
166
+ "error",
167
+ "transcription.done",
168
+ "transcription.delta",
169
+ )
170
+
171
+ # --- Second connection: normal transcription ---------------------
172
+ # Verifies the engine is still alive after the empty commit above.
173
+ async with websockets.connect(ws_url) as ws:
174
+ event = await receive_event(ws, timeout=30.0)
175
+ assert event["type"] == "session.created"
176
+
177
+ await send_event(ws, {"type": "session.update", "model": model_name})
178
+
179
+ await send_event(ws, {"type": "input_audio_buffer.commit"})
180
+
181
+ for chunk in mary_had_lamb_audio_chunks:
182
+ await send_event(
183
+ ws, {"type": "input_audio_buffer.append", "audio": chunk}
184
+ )
185
+
186
+ await send_event(ws, {"type": "input_audio_buffer.commit", "final": True})
187
+
188
+ done_received = False
189
+ while not done_received:
190
+ event = await receive_event(ws, timeout=60.0)
191
+ if event["type"] == "transcription.done":
192
+ done_received = True
193
+ elif event["type"] == "error":
194
+ pytest.fail(f"Engine error after empty commit: {event}")
195
+ assert done_received
vllm/model_executor/models/voxtral_realtime.py
+29
-10
@@ -299,13 +299,29 @@ class VoxtralRealtimeGeneration(VoxtralForConditionalGeneration, SupportsRealtim
299
# Multi-modal token ID may exceed vocab size
300
handle_oov_mm_token: bool = True,
301
) -> torch.Tensor:
302
- """Pass post-conv embeddings directly as input"""
303
- # for realtime we simply flatten the multimodal embeddings
304
- # to be in tensor format, we treat the input ids later
305
- assert multimodal_embeddings is not None
306
- assert len(multimodal_embeddings) > 0, (
307
- "For realtime you must provide a multimodal_embedding at every step."
308
- )
302
+ """Pass post-conv embeddings directly as input.
303
+
304
+ For realtime models, multimodal embeddings are required at every
305
+ decode step. If they are missing (e.g. due to an empty audio
306
+ commit, encoder-cache eviction under GPU memory pressure, or a
307
+ client disconnect), return zero embeddings instead of crashing
308
+ the engine so that all other in-flight requests stay alive.
309
+ """
310
+ if multimodal_embeddings is None or len(multimodal_embeddings) == 0:
311
+ logger.warning(
312
+ "Realtime model received empty multimodal embeddings "
313
+ "for %d input tokens. Returning zero embeddings to "
314
+ "avoid engine crash.",
315
+ input_ids.shape[0],
316
+ )
317
+ pool_size = self.config.audio_config.block_pool_size
318
+ embed_dim = self.config.audio_config.d_model * pool_size
319
+ return torch.zeros(
320
+ input_ids.shape[0],
321
+ embed_dim,
322
+ dtype=self.whisper_encoder.dtype,
323
+ device=input_ids.device,
324
+ )
325
mm_embeds_flat = _flatten_embeddings(multimodal_embeddings)
326
return mm_embeds_flat
327
@@ -367,9 +383,12 @@ class VoxtralRealtimeGeneration(VoxtralForConditionalGeneration, SupportsRealtim
383
"""Transform audio waveforms -> initial whisper post-conv embeddings"""
384
audio_inputs = self._parse_and_validate_audio_arrays(**kwargs)
385
370
- assert audio_inputs is not None, (
371
- "For realtime you must provide an audio input at every step."
372
- )
386
+ if audio_inputs is None:
387
+ logger.warning(
388
+ "Realtime model received no audio inputs in "
389
+ "embed_multimodal. Returning empty embeddings."
390
+ )
391
+ return []
392
393
def _truncate_left(
394
sample: torch.Tensor, mult_of: int, pos: int