A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 20:15 UTC
[MM] Allow audio chunking for offline LLM (#34628)
Signed-off-by: NickLucche <nlucches@redhat.com>
Nicolò Lucchesi committed
Feb 24, 2026 at 06:04 UTC
f91808ae0ddf750acfdeb351fa072c91d4d678fc
5 files changed
+366
-54
docs/features/multimodal_inputs.md
+45
@@ -295,6 +295,51 @@ You can pass a tuple `(array, sampling_rate)` to the `'audio'` field of the mult
295
296
Full example: [examples/offline_inference/audio_language.py](../../examples/offline_inference/audio_language.py)
297
298
+#### Chunking Long Audio for Transcription
299
+
300
+Speech-to-text models like Whisper have a maximum audio length they can process (typically 30 seconds). For longer audio files, vLLM provides a utility to intelligently split audio into chunks at quiet points to minimize cutting through speech.
301
+
302
+```python
303
+import librosa
304
+from vllm import LLM, SamplingParams
305
+from vllm.multimodal.audio import split_audio
306
+
307
+# Load long audio file
308
+audio, sr = librosa.load("long_audio.wav", sr=16000)
309
+
310
+# Split into chunks at low-energy (quiet) regions
311
+chunks = split_audio(
312
+ audio_data=audio,
313
+ sample_rate=sr,
314
+ max_clip_duration_s=30.0, # Maximum chunk length in seconds
315
+ overlap_duration_s=1.0, # Search window for finding quiet split points
316
+ min_energy_window_size=1600, # Window size for energy calculation (~100ms at 16kHz)
317
+)
318
+
319
+# Initialize Whisper model
320
+llm = LLM(model="openai/whisper-large-v3-turbo")
321
+sampling_params = SamplingParams(temperature=0, max_tokens=256)
322
+
323
+# Transcribe each chunk
324
+transcriptions = []
325
+for chunk in chunks:
326
+ outputs = llm.generate({
327
+ "prompt": "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>",
328
+ "multi_modal_data": {"audio": (chunk, sr)},
329
+ }, sampling_params)
330
+ transcriptions.append(outputs[0].outputs[0].text)
331
+
332
+# Combine results
333
+full_transcription = " ".join(transcriptions)
334
+```
335
+
336
+The `split_audio` function:
337
+
338
+- Splits audio at quiet points to avoid cutting through speech
339
+- Uses RMS energy to find low-amplitude regions within the overlap window
340
+- Preserves all audio samples (no data loss)
341
+- Supports any sample rate
342
+
343
#### Automatic Audio Channel Normalization
344
345
vLLM automatically normalizes audio channels for models that require specific audio formats. When loading audio with libraries like `torchaudio`, stereo files return shape `[channels, time]`, but many audio models (particularly Whisper-based models) expect mono audio with shape `[time]`.
tests/multimodal/test_audio.py
+184
@@ -16,6 +16,7 @@ from vllm.multimodal.audio import (
16
normalize_audio,
17
resample_audio_librosa,
18
resample_audio_scipy,
19
+ split_audio,
20
)
21
22
@@ -584,3 +585,186 @@ class TestAudioPipelineE2E:
585
assert audio_output.ndim == 1
586
assert audio_output.shape == (10,)
587
np.testing.assert_array_almost_equal(audio_output, np.zeros(10))
588
+
589
+
590
+# ============================================================
591
+# Tests for Audio Chunking Utilities
592
+# ============================================================
593
+
594
+
595
+class TestAudioChunking:
596
+ """Tests for split_audio and find_split_point utilities in vllm.multimodal.audio."""
597
+
598
+ def test_split_audio_short_clip(self):
599
+ """Audio shorter than max_clip_duration_s should not be split."""
600
+
601
+ # 10 seconds of audio at 16kHz
602
+ audio = np.linspace(-1.0, 1.0, 160000, dtype=np.float32)
603
+
604
+ chunks = split_audio(
605
+ audio_data=audio,
606
+ sample_rate=16000,
607
+ max_clip_duration_s=30.0,
608
+ overlap_duration_s=1.0,
609
+ min_energy_window_size=1600,
610
+ )
611
+
612
+ assert len(chunks) == 1
613
+ np.testing.assert_array_equal(chunks[0], audio)
614
+
615
+ def test_split_audio_exact_length(self):
616
+ """Audio exactly at max_clip_duration_s should not be split."""
617
+
618
+ # Exactly 30 seconds at 16kHz
619
+ audio = np.linspace(-1.0, 1.0, 480000, dtype=np.float32)
620
+
621
+ chunks = split_audio(
622
+ audio_data=audio,
623
+ sample_rate=16000,
624
+ max_clip_duration_s=30.0,
625
+ overlap_duration_s=1.0,
626
+ min_energy_window_size=1600,
627
+ )
628
+
629
+ assert len(chunks) == 1
630
+ np.testing.assert_array_equal(chunks[0], audio)
631
+
632
+ def test_split_audio_long_clip(self):
633
+ """Long audio should be split into multiple chunks."""
634
+
635
+ # 65 seconds of audio at 16kHz
636
+ audio = np.linspace(-1.0, 1.0, 1040000, dtype=np.float32)
637
+
638
+ chunks = split_audio(
639
+ audio_data=audio,
640
+ sample_rate=16000,
641
+ max_clip_duration_s=30.0,
642
+ overlap_duration_s=1.0,
643
+ min_energy_window_size=1600,
644
+ )
645
+
646
+ assert len(chunks) > 1
647
+ # First sample preserved
648
+ assert chunks[0][0] == audio[0]
649
+ # Last sample preserved
650
+ assert chunks[-1][-1] == audio[-1]
651
+
652
+ def test_split_audio_chunks_have_correct_length(self):
653
+ """Each chunk (except last) should be approximately max_clip_duration_s."""
654
+
655
+ # 65 seconds of audio at 16kHz
656
+ audio = np.linspace(-1.0, 1.0, 1040000, dtype=np.float32)
657
+
658
+ chunks = split_audio(
659
+ audio_data=audio,
660
+ sample_rate=16000,
661
+ max_clip_duration_s=30.0,
662
+ overlap_duration_s=1.0,
663
+ min_energy_window_size=1600,
664
+ )
665
+
666
+ max_samples = int(30.0 * 16000)
667
+ overlap_samples = int(1.0 * 16000)
668
+
669
+ for chunk in chunks[:-1]:
670
+ assert chunk.shape[0] >= max_samples - overlap_samples
671
+ assert chunk.shape[0] <= max_samples
672
+
673
+ def test_find_split_point_finds_quiet_region(self):
674
+ """find_split_point should identify low-energy regions."""
675
+ from vllm.multimodal.audio import find_split_point
676
+
677
+ # Create audio with a quiet section in the middle
678
+ segment = np.ones(32000, dtype=np.float32)
679
+ # Insert quiet region at sample 16000-17600 (100ms)
680
+ segment[16000:17600] = 0.01
681
+
682
+ split_idx = find_split_point(
683
+ wav=segment,
684
+ start_idx=0,
685
+ end_idx=32000,
686
+ min_energy_window=1600,
687
+ )
688
+
689
+ # Split should be in or near the quiet region
690
+ assert 16000 <= split_idx <= 17600
691
+
692
+ def test_find_split_point_handles_uniform_audio(self):
693
+ """find_split_point should handle uniform energy audio gracefully."""
694
+ from vllm.multimodal.audio import find_split_point
695
+
696
+ segment = np.ones(32000, dtype=np.float32) * 0.5
697
+
698
+ split_idx = find_split_point(
699
+ wav=segment,
700
+ start_idx=0,
701
+ end_idx=32000,
702
+ min_energy_window=1600,
703
+ )
704
+
705
+ assert 0 <= split_idx <= 32000
706
+
707
+ def test_find_split_point_silence(self):
708
+ """find_split_point should prefer the quietest scanned window."""
709
+ from vllm.multimodal.audio import find_split_point
710
+
711
+ # Deterministic signal: constant energy everywhere except silence.
712
+ segment = np.ones(32000, dtype=np.float32)
713
+ # Complete silence at 20000-21600.
714
+ segment[20000:21600] = 0.0
715
+
716
+ split_idx = find_split_point(
717
+ wav=segment,
718
+ start_idx=16000,
719
+ end_idx=28000,
720
+ min_energy_window=1600,
721
+ )
722
+
723
+ # Current implementation evaluates non-overlapping 1600-sample windows
724
+ # from start_idx, so the quietest scanned window starts at 19200.
725
+ assert split_idx == 19200
726
+
727
+ def test_split_audio_preserves_boundaries(self):
728
+ """Verify first and last samples are preserved when chunking."""
729
+
730
+ audio = np.arange(1120000, dtype=np.float32) # 70s at 16kHz
731
+
732
+ chunks = split_audio(
733
+ audio_data=audio,
734
+ sample_rate=16000,
735
+ max_clip_duration_s=30.0,
736
+ overlap_duration_s=1.0,
737
+ min_energy_window_size=1600,
738
+ )
739
+
740
+ assert chunks[0][0] == audio[0]
741
+ assert chunks[-1][-1] == audio[-1]
742
+
743
+ def test_split_audio_with_different_sample_rates(self):
744
+ """Test chunking works with different sample rates."""
745
+
746
+ # 40 seconds at 8kHz
747
+ audio_8k = np.linspace(-1.0, 1.0, 320000, dtype=np.float32)
748
+
749
+ chunks = split_audio(
750
+ audio_data=audio_8k,
751
+ sample_rate=8000,
752
+ max_clip_duration_s=30.0,
753
+ overlap_duration_s=1.0,
754
+ min_energy_window_size=800,
755
+ )
756
+
757
+ assert len(chunks) >= 2
758
+
759
+ # 40 seconds at 48kHz
760
+ audio_48k = np.linspace(-1.0, 1.0, 1920000, dtype=np.float32)
761
+
762
+ chunks_48k = split_audio(
763
+ audio_data=audio_48k,
764
+ sample_rate=48000,
765
+ max_clip_duration_s=30.0,
766
+ overlap_duration_s=1.0,
767
+ min_energy_window_size=4800,
768
+ )
769
+
770
+ assert len(chunks_48k) >= 2
vllm/config/speech_to_text.py
+4
-1
@@ -33,4 +33,7 @@ class SpeechToTextConfig:
33
34
@property
35
def allow_audio_chunking(self) -> bool:
36
- return self.min_energy_split_window_size is not None
36
+ return (
37
+ self.min_energy_split_window_size is not None
38
+ and self.max_audio_clip_s is not None
39
+ )
vllm/entrypoints/openai/speech_to_text/speech_to_text.py
+15
-53
@@ -45,6 +45,7 @@ from vllm.model_executor.models import (
45
SupportsTranscription,
46
supports_transcription,
47
)
48
+from vllm.multimodal.audio import split_audio
49
from vllm.outputs import RequestOutput
50
from vllm.renderers.inputs import DictPrompt, EncoderDecoderDictPrompt
51
from vllm.renderers.inputs.preprocess import parse_enc_dec_prompt, parse_model_prompt
@@ -323,11 +324,24 @@ class OpenAISpeechToText(OpenAIServing):
324
self.asr_config.allow_audio_chunking
325
and duration > self.asr_config.max_audio_clip_s
326
)
326
- chunks = [y] if not do_split_audio else self._split_audio(y, int(sr))
327
+
328
+ if not do_split_audio:
329
+ chunks = [y]
330
+ else:
331
+ assert self.asr_config.max_audio_clip_s is not None
332
+ assert self.asr_config.min_energy_split_window_size is not None
333
+ chunks = split_audio(
334
+ audio_data=y,
335
+ sample_rate=int(sr),
336
+ max_clip_duration_s=self.asr_config.max_audio_clip_s,
337
+ overlap_duration_s=self.asr_config.overlap_chunk_second,
338
+ min_energy_window_size=self.asr_config.min_energy_split_window_size,
339
+ )
340
341
if language is None and getattr(
342
self.model_cls, "supports_explicit_language_detection", False
343
):
344
+ # Auto-detect language from the first chunk.
345
language = await self._detect_language(
346
chunks[0], f"{request_id}-lang_detect"
347
)
@@ -754,55 +768,3 @@ class OpenAISpeechToText(OpenAIServing):
768
yield f"data: {data}\n\n"
769
# Send the final done message after all response.n are finished
770
yield "data: [DONE]\n\n"
757
-
758
- def _split_audio(
759
- self, audio_data: np.ndarray, sample_rate: int
760
- ) -> list[np.ndarray]:
761
- assert self.asr_config.max_audio_clip_s is not None, (
762
- f"{self.asr_config.max_audio_clip_s=} cannot be None to"
763
- " split audio into chunks."
764
- )
765
- chunk_size = sample_rate * self.asr_config.max_audio_clip_s
766
- overlap_size = sample_rate * self.asr_config.overlap_chunk_second
767
- chunks = []
768
- i = 0
769
- while i < audio_data.shape[-1]:
770
- if i + chunk_size >= audio_data.shape[-1]:
771
- # handle last chunk
772
- chunks.append(audio_data[..., i:])
773
- break
774
-
775
- # Find the best split point in the overlap region
776
- search_start = i + chunk_size - overlap_size
777
- search_end = min(i + chunk_size, audio_data.shape[-1])
778
- split_point = self._find_split_point(audio_data, search_start, search_end)
779
-
780
- # Extract chunk up to the split point
781
- chunks.append(audio_data[..., i:split_point])
782
- i = split_point
783
- return chunks
784
-
785
- def _find_split_point(self, wav: np.ndarray, start_idx: int, end_idx: int) -> int:
786
- """Find the best point to split audio by
787
- looking for silence or low amplitude.
788
- Args:
789
- wav: Audio tensor [1, T]
790
- start_idx: Start index of search region
791
- end_idx: End index of search region
792
- Returns:
793
- Index of best splitting point
794
- """
795
- segment = wav[start_idx:end_idx]
796
-
797
- # Calculate RMS energy in small windows
798
- min_energy = math.inf
799
- quietest_idx = 0
800
- min_energy_window = self.asr_config.min_energy_split_window_size
801
- assert min_energy_window is not None
802
- for i in range(0, len(segment) - min_energy_window, min_energy_window):
803
- window = segment[i : i + min_energy_window]
804
- energy = (window**2).mean() ** 0.5
805
- if energy < min_energy:
806
- quietest_idx = i + start_idx
807
- min_energy = energy
808
- return quietest_idx
vllm/multimodal/audio.py
+118
@@ -216,3 +216,121 @@ class AudioResampler:
216
f"Invalid resampling method: {self.method}. "
217
"Supported methods are 'librosa' and 'scipy'."
218
)
219
+
220
+
221
+# ============================================================
222
+# Audio Chunking / Splitting
223
+# ============================================================
224
+
225
+
226
+def split_audio(
227
+ audio_data: np.ndarray,
228
+ sample_rate: int,
229
+ max_clip_duration_s: float,
230
+ overlap_duration_s: float,
231
+ min_energy_window_size: int,
232
+) -> list[np.ndarray]:
233
+ """Split audio into chunks with intelligent split points.
234
+
235
+ Splits long audio into smaller chunks at low-energy regions to minimize
236
+ cutting through speech. Uses overlapping windows to find quiet moments
237
+ for splitting.
238
+
239
+ Args:
240
+ audio_data: Audio array to split. Can be 1D (mono) or multi-dimensional.
241
+ Splits along the last dimension (time axis).
242
+ sample_rate: Sample rate of the audio in Hz.
243
+ max_clip_duration_s: Maximum duration of each chunk in seconds.
244
+ overlap_duration_s: Overlap duration in seconds between consecutive chunks.
245
+ Used to search for optimal split points.
246
+ min_energy_window_size: Window size in samples for finding low-energy regions.
247
+
248
+ Returns:
249
+ List of audio chunks. Each chunk is a numpy array with the same shape
250
+ as the input except for the last (time) dimension.
251
+
252
+ Example:
253
+ >>> audio = np.random.randn(1040000) # 65 seconds at 16kHz
254
+ >>> chunks = split_audio(
255
+ ... audio_data=audio,
256
+ ... sample_rate=16000,
257
+ ... max_clip_duration_s=30.0,
258
+ ... overlap_duration_s=1.0,
259
+ ... min_energy_window_size=1600,
260
+ ... )
261
+ >>> len(chunks)
262
+ 3
263
+ """
264
+ chunk_size = int(sample_rate * max_clip_duration_s)
265
+ overlap_size = int(sample_rate * overlap_duration_s)
266
+ chunks = []
267
+ i = 0
268
+
269
+ while i < audio_data.shape[-1]:
270
+ if i + chunk_size >= audio_data.shape[-1]:
271
+ # Handle last chunk - take everything remaining
272
+ chunks.append(audio_data[..., i:])
273
+ break
274
+
275
+ # Find the best split point in the overlap region
276
+ search_start = i + chunk_size - overlap_size
277
+ search_end = min(i + chunk_size, audio_data.shape[-1])
278
+ split_point = find_split_point(
279
+ audio_data, search_start, search_end, min_energy_window_size
280
+ )
281
+
282
+ # Extract chunk up to the split point
283
+ chunks.append(audio_data[..., i:split_point])
284
+ i = split_point
285
+
286
+ return chunks
287
+
288
+
289
+def find_split_point(
290
+ wav: np.ndarray,
291
+ start_idx: int,
292
+ end_idx: int,
293
+ min_energy_window: int,
294
+) -> int:
295
+ """Find the best point to split audio by looking for silence or low amplitude.
296
+
297
+ Searches for the quietest region within a specified range by calculating
298
+ RMS energy in sliding windows.
299
+
300
+ Args:
301
+ wav: Audio array. Can be 1D or multi-dimensional.
302
+ start_idx: Start index of search region (inclusive).
303
+ end_idx: End index of search region (exclusive).
304
+ min_energy_window: Window size in samples for energy calculation.
305
+
306
+ Returns:
307
+ Index of the quietest point within the search region. This is the
308
+ recommended split point to minimize audio artifacts.
309
+
310
+ Example:
311
+ >>> audio = np.random.randn(32000)
312
+ >>> # Insert quiet region
313
+ >>> audio[16000:17600] = 0.01
314
+ >>> split_idx = find_split_point(
315
+ ... wav=audio,
316
+ ... start_idx=0,
317
+ ... end_idx=32000,
318
+ ... min_energy_window=1600,
319
+ ... )
320
+ >>> 16000 <= split_idx <= 17600
321
+ True
322
+ """
323
+ segment = wav[start_idx:end_idx]
324
+
325
+ # Calculate RMS energy in small windows
326
+ min_energy = math.inf
327
+ quietest_idx = 0
328
+
329
+ for i in range(0, len(segment) - min_energy_window, min_energy_window):
330
+ window = segment[i : i + min_energy_window]
331
+ energy = (window**2).mean() ** 0.5
332
+ if energy < min_energy:
333
+ quietest_idx = i + start_idx
334
+ min_energy = energy
335
+
336
+ return quietest_idx