[Bugfix] Changed speech to text chunk timestamp to cumulative approach (#41131)
Signed-off-by: Toby Bell <toby.bell1702@hotmail.co.uk>
TobyJBell committed
Jul 27, 2026 at 13:06 UTC
15d65f86694ac647b139179f88c9d7aea0820108
5 files changed
+101
-11
docs/contributing/model/transcription.md
+1
-1
@@ -195,7 +195,7 @@ Provide a fast duration→token estimate to improve streaming usage statistics:
195
The API server takes care of basic audio I/O and optional chunking before building prompts:
196
197
- Resampling: Input audio is resampled to `SpeechToTextConfig.sample_rate` using `AudioResampler`.
198
-- Chunking: If `SpeechToTextConfig.allow_audio_chunking` is True and the duration exceeds `max_audio_clip_s`, the server splits the audio into overlapping chunks and generates a prompt per chunk. Overlap is controlled by `overlap_chunk_second`.
198
+- Chunking: If `SpeechToTextConfig.allow_audio_chunking` is True and the duration exceeds `max_audio_clip_s`, the server splits the audio into chunks and generates a prompt per chunk. There is no overlap between chunks, overlap_chunk_second controls the size of the search window used to find the split point.
199
- Energy-aware splitting: When `min_energy_split_window_size` is set, the server finds low-energy regions to minimize cutting within words.
200
201
Relevant server logic:
tests/entrypoints/speech_to_text/test_speech_to_text_cancellation.py
+10
-2
@@ -53,7 +53,13 @@ async def test_non_streaming_cancel_aborts_engine_requests(
53
server.asr_config = SimpleNamespace(max_audio_clip_s=30)
54
server._check_model = AsyncMock(return_value=None)
55
server._maybe_get_adapters = Mock(return_value=None)
56
- server._preprocess_speech_to_text = AsyncMock(return_value=(engine_inputs, 40.0))
56
+ server._preprocess_speech_to_text = AsyncMock(
57
+ return_value=(
58
+ engine_inputs,
59
+ 40.0,
60
+ [30.0 * i for i in range(len(engine_inputs))],
61
+ )
62
+ )
63
server._log_inputs = Mock()
64
65
request = SimpleNamespace(
@@ -122,7 +128,9 @@ async def test_non_streaming_cancel_advances_all_chunk_generators():
128
server.asr_config = SimpleNamespace(max_audio_clip_s=30)
129
server._check_model = AsyncMock(return_value=None)
130
server._maybe_get_adapters = Mock(return_value=None)
125
- server._preprocess_speech_to_text = AsyncMock(return_value=(engine_inputs, 90.0))
131
+ server._preprocess_speech_to_text = AsyncMock(
132
+ return_value=(engine_inputs, 90.0, [0.0, 29.5, 29.5 + 29.7])
133
+ )
134
server._log_inputs = Mock()
135
136
request = SimpleNamespace(
tests/entrypoints/speech_to_text/transcription/test_chunk_timestamp_offset.py
new
+70
@@ -0,0 +1,70 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""Regression test for chunk timestamp offset drift in _preprocess_speech_to_text."""
4
+
5
+from __future__ import annotations
6
+
7
+from unittest.mock import AsyncMock, MagicMock, patch
8
+
9
+import numpy as np
10
+import pytest
11
+
12
+from vllm.config.speech_to_text import SpeechToTextConfig
13
+from vllm.entrypoints.speech_to_text.base.serving import SpeechToTextBaseServing
14
+
15
+SR = 16_000
16
+_PATCH = "vllm.entrypoints.speech_to_text.base.serving"
17
+
18
+
19
+@pytest.mark.asyncio
20
+async def test_chunk_offsets_are_cumulative_not_nominal():
21
+ """
22
+ chunk_start_offsets must be cumulative actual chunk lengths, not the old approach of
23
+ 'idx * max_audio_clip_s'. When split_audio places a boundary before the
24
+ nominal 30 s mark, the old formula drifts; the fixed formula stays exact.
25
+ """
26
+ # Chunks shorter than exactly 30 s, as split_audio produces when a quiet
27
+ # region falls inside the 1 s overlap window before the nominal boundary.
28
+ chunk_lengths = [int(29.5 * SR), int(29.7 * SR), int(5.0 * SR)]
29
+ chunks = [np.zeros(n, dtype=np.float32) for n in chunk_lengths]
30
+
31
+ duration = sum(chunk_lengths) / SR
32
+
33
+ expected_offsets = [0.0, 29.5, 29.5 + 29.7] # cumulative seconds
34
+ wrong_offsets = [0.0, 30.0, 60.0] # what the old bug produced
35
+
36
+ serving = SpeechToTextBaseServing.__new__(SpeechToTextBaseServing)
37
+ serving._decode_and_chunk_speech_async = AsyncMock(return_value=(chunks, duration))
38
+ serving.asr_config = SpeechToTextConfig(
39
+ sample_rate=float(SR),
40
+ max_audio_clip_s=30,
41
+ overlap_chunk_second=1,
42
+ min_energy_split_window_size=1600,
43
+ )
44
+ serving.max_audio_filesize_mb = 100.0
45
+ serving.model_cls = MagicMock()
46
+ serving.model_cls.validate_language.side_effect = lambda lang: lang
47
+ serving.model_cls.supports_explicit_language_detection = False
48
+ serving.model_cls.get_generation_prompt.return_value = {}
49
+ serving.model_config = MagicMock()
50
+ serving.task_type = "transcribe"
51
+ serving.renderer = MagicMock()
52
+ serving.renderer.render_cmpl_async = AsyncMock(
53
+ return_value=[MagicMock()] * len(chunks)
54
+ )
55
+
56
+ request = MagicMock()
57
+ request.language = "en"
58
+ request.to_language = None
59
+ request.response_format = "json"
60
+ request.build_stt_params.return_value = MagicMock()
61
+
62
+ with patch(f"{_PATCH}.parse_model_prompt", return_value=MagicMock()):
63
+ _, _, offsets = await serving._preprocess_speech_to_text(
64
+ request=request,
65
+ audio_data=b"\x00",
66
+ request_id="test",
67
+ )
68
+
69
+ assert offsets == pytest.approx(expected_offsets, abs=1e-6)
70
+ assert offsets != pytest.approx(wrong_offsets, abs=1e-6)
tests/entrypoints/speech_to_text/transcription/test_transcription_inter_chunk_spacing.py
+3
-1
@@ -342,7 +342,9 @@ async def test_create_transcription_non_streaming_joins_chunks_by_language():
342
models.lora_requests = {}
343
models.is_base_model.return_value = True
344
345
- preprocess_mock = AsyncMock(return_value=([MagicMock(), MagicMock()], 1.0))
345
+ preprocess_mock = AsyncMock(
346
+ return_value=([MagicMock(), MagicMock()], 1.0, [0.0, 29.5])
347
+ )
348
349
with (
350
patch(
vllm/entrypoints/speech_to_text/base/serving.py
+17
-7
@@ -258,7 +258,7 @@ class SpeechToTextBaseServing(GenerateBaseServing):
258
request: SpeechToTextRequest,
259
audio_data: bytes,
260
request_id: str,
261
- ) -> tuple[list[EngineInput], float]:
261
+ ) -> tuple[list[EngineInput], float, list[float]]:
262
# Validate request
263
request.language = self.model_cls.validate_language(request.language)
264
request.to_language = (
@@ -277,6 +277,13 @@ class SpeechToTextBaseServing(GenerateBaseServing):
277
# Run cpu intensive preprocess step in a separate thread pool executor.
278
chunks, duration = await self._decode_and_chunk_speech_async(audio_data)
279
280
+ chunk_start_offsets: list[float] = [0.0]
281
+
282
+ for chunk in chunks[:-1]:
283
+ chunk_start_offsets.append(
284
+ chunk_start_offsets[-1] + chunk.shape[-1] / self.asr_config.sample_rate
285
+ )
286
+
287
if request.language is None and getattr(
288
self.model_cls, "supports_explicit_language_detection", False
289
):
@@ -306,7 +313,7 @@ class SpeechToTextBaseServing(GenerateBaseServing):
313
314
engine_inputs = await self.renderer.render_cmpl_async(parsed_prompts)
315
309
- return engine_inputs, duration
316
+ return engine_inputs, duration, chunk_start_offsets
317
318
def _preprocess_verbose_prompt(self, prompt: EncoderDecoderDictPrompt):
319
dec_prompt = prompt["decoder_prompt"]
@@ -391,7 +398,7 @@ class SpeechToTextBaseServing(GenerateBaseServing):
398
SpeechToTextSegment,
399
segment_class(
400
id=len(segments),
394
- seek=start_time,
401
+ seek=int(start_time),
402
start=start_time + BASE_OFFSET * start_timestamp,
403
end=start_time + BASE_OFFSET * end_timestamp,
404
temperature=request.temperature,
@@ -467,7 +474,11 @@ class SpeechToTextBaseServing(GenerateBaseServing):
474
475
lora_request = self._maybe_get_adapters(request)
476
470
- engine_inputs, duration_s = await self._preprocess_speech_to_text(
477
+ (
478
+ engine_inputs,
479
+ duration_s,
480
+ chunk_start_offsets,
481
+ ) = await self._preprocess_speech_to_text(
482
request=request,
483
audio_data=audio_data,
484
request_id=request_id,
@@ -587,11 +598,10 @@ class SpeechToTextBaseServing(GenerateBaseServing):
598
assert len(list_result_generator) == 1, (
599
"`max_audio_clip_s` is set to None, audio cannot be chunked"
600
)
601
+ assert len(chunk_start_offsets) == len(list_result_generator)
602
result_generator = merge_async_iterators(*list_result_generator)
603
async for idx, op in result_generator:
592
- start_time = (
593
- float(idx * chunk_size_in_s) if chunk_size_in_s is not None else 0.0
594
- )
604
+ start_time = chunk_start_offsets[idx]
605
if request.response_format == "verbose_json":
606
assert op.outputs[0].logprobs
607
segments: list[SpeechToTextSegment] = self._get_verbose_segments(