[Misc] Clarify mono audio requirement (#50141)
Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
Nicolò Lucchesi committed
Jul 31, 2026 at 12:46 UTC
17beffd55f6fcc28f2e777d47d3e8aab6ea34c9a
4 files changed
+31
-5
docs/features/multimodal_inputs.md
+1
@@ -421,6 +421,7 @@ full_transcription = " ".join(transcriptions)
421
422
The `split_audio` function:
423
424
+- Expects 1D mono audio (`load_audio` downmixes by default)
425
- Splits audio at quiet points to avoid cutting through speech
426
- Uses RMS energy to find low-amplitude regions within the overlap window
427
- Preserves all audio samples (no data loss)
tests/multimodal/test_audio.py
+17
@@ -744,6 +744,23 @@ class TestAudioChunking:
744
# from start_idx, so the quietest scanned window starts at 19200.
745
assert split_idx == 19200
746
747
+ def test_split_audio_rejects_multi_channel(self):
748
+ """Chunking is mono-only; stereo must fail loudly rather than silently.
749
+
750
+ find_split_point searches axis 0, so (channels, time) input would skip
751
+ the energy search entirely and split at fixed boundaries instead.
752
+ """
753
+ stereo = np.ones((2, 16000 * 65), dtype=np.float32)
754
+
755
+ with pytest.raises(ValueError, match="expects mono audio"):
756
+ split_audio(
757
+ audio_data=stereo,
758
+ sample_rate=16000,
759
+ max_clip_duration_s=30.0,
760
+ overlap_duration_s=1.0,
761
+ min_energy_window_size=1600,
762
+ )
763
+
764
def test_split_audio_preserves_boundaries(self):
765
"""Verify first and last samples are preserved when chunking."""
766
vllm/entrypoints/speech_to_text/base/serving.py
+1
@@ -178,6 +178,7 @@ class SpeechToTextBaseServing(GenerateBaseServing):
178
y, sr = load_audio(
179
buf,
180
sr=self.asr_config.sample_rate,
181
+ mono=True,
182
max_duration_s=self.max_audio_decode_duration_s,
183
)
184
except ValueError:
vllm/multimodal/audio.py
+12
-5
@@ -336,8 +336,8 @@ def split_audio(
336
for splitting.
337
338
Args:
339
- audio_data: Audio array to split. Can be 1D (mono) or multi-dimensional.
340
- Splits along the last dimension (time axis).
339
+ audio_data: 1D mono audio array to split. ASR models consume mono, so
340
+ callers must downmix before chunking.
341
sample_rate: Sample rate of the audio in Hz.
342
max_clip_duration_s: Maximum duration of each chunk in seconds.
343
overlap_duration_s: Overlap duration in seconds between consecutive chunks.
@@ -345,8 +345,10 @@ def split_audio(
345
min_energy_window_size: Window size in samples for finding low-energy regions.
346
347
Returns:
348
- List of audio chunks. Each chunk is a numpy array with the same shape
349
- as the input except for the last (time) dimension.
348
+ List of 1D audio chunks.
349
+
350
+ Raises:
351
+ AssertionError: If ``audio_data`` is not 1D.
352
353
Example:
354
>>> audio = np.random.randn(1040000) # 65 seconds at 16kHz
@@ -360,6 +362,11 @@ def split_audio(
362
>>> len(chunks)
363
3
364
"""
365
+ if audio_data.ndim > 1:
366
+ raise ValueError(
367
+ f"split_audio expects mono audio, got shape {audio_data.shape}"
368
+ )
369
+
370
chunk_size = int(sample_rate * max_clip_duration_s)
371
overlap_size = int(sample_rate * overlap_duration_s)
372
chunks = []
@@ -402,7 +409,7 @@ def find_split_point(
409
RMS energy in sliding windows.
410
411
Args:
405
- wav: Audio array. Can be 1D or multi-dimensional.
412
+ wav: 1D mono audio array.
413
start_idx: Start index of search region (inclusive).
414
end_idx: End index of search region (exclusive).
415
min_energy_window: Window size in samples for energy calculation.