A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 09:45 UTC
[ASR] Fix spacing bw chunks in multi chunk audio transcription (#39116)
Signed-off-by: Ekagra Ranjan <3116519+ekagra-ranjan@users.noreply.github.com>
Ekagra Ranjan committed
Apr 9, 2026 at 15:46 UTC
f7cad674124922e4945386ee9ba312792f625961
5 files changed
+315
-4
tests/entrypoints/openai/speech_to_text/test_transcription_inter_chunk_spacing.py
new
+271
@@ -0,0 +1,271 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""ASR inter-chunk spacing: ``asr_inter_chunk_separator`` and transcription
4
+serving (mocked).
5
+
6
+Unit tests cover the helper and ``SupportsTranscription.no_space_languages``.
7
+Integration-style tests exercise ``OpenAIServingTranscription`` streaming and
8
+``create_transcription`` without loading a model.
9
+"""
10
+
11
+from __future__ import annotations
12
+
13
+import json
14
+from collections.abc import AsyncGenerator
15
+from types import SimpleNamespace
16
+from unittest.mock import AsyncMock, MagicMock, patch
17
+
18
+import pytest
19
+
20
+from vllm.config import ModelConfig
21
+from vllm.config.speech_to_text import SpeechToTextConfig
22
+from vllm.entrypoints.openai.engine.protocol import (
23
+ ErrorResponse,
24
+ RequestResponseMetadata,
25
+)
26
+from vllm.entrypoints.openai.models.serving import OpenAIServingModels
27
+from vllm.entrypoints.openai.speech_to_text.protocol import TranscriptionRequest
28
+from vllm.entrypoints.openai.speech_to_text.serving import OpenAIServingTranscription
29
+from vllm.entrypoints.openai.speech_to_text.speech_to_text import (
30
+ OpenAISpeechToText,
31
+ asr_inter_chunk_separator,
32
+)
33
+from vllm.model_executor.models.interfaces import SupportsTranscription
34
+from vllm.outputs import CompletionOutput, RequestOutput
35
+
36
+# --- Unit: helper + protocol -------------------------------------------------
37
+
38
+
39
+def test_default_no_space_languages_includes_zh_and_ja():
40
+ assert SupportsTranscription.no_space_languages == {"ja", "zh"}
41
+
42
+
43
+@pytest.mark.parametrize(
44
+ ("language", "expected_sep"),
45
+ [
46
+ ("en", " "),
47
+ ("EN", " "),
48
+ ("zh", ""),
49
+ ("ZH", ""),
50
+ ("ja", ""),
51
+ (None, " "),
52
+ ],
53
+)
54
+def test_asr_inter_chunk_separator_matches_protocol(language, expected_sep):
55
+ sep = asr_inter_chunk_separator(language, SupportsTranscription.no_space_languages)
56
+ assert sep == expected_sep
57
+
58
+
59
+def test_joined_chunks_english_has_space_between():
60
+ sep = asr_inter_chunk_separator("en", SupportsTranscription.no_space_languages)
61
+ assert sep.join(["hello", "world"]) == "hello world"
62
+
63
+
64
+def test_joined_chunks_chinese_has_no_space_between():
65
+ sep = asr_inter_chunk_separator("zh", SupportsTranscription.no_space_languages)
66
+ assert sep.join(["你好", "世界"]) == "你好世界"
67
+
68
+
69
+# --- Integration: serving (no model) -----------------------------------------
70
+
71
+
72
+class _StubTranscriptionModel:
73
+ """Minimal stand-in for a SupportsTranscription implementation (no torch)."""
74
+
75
+ no_space_languages: set[str] = {"ja", "zh"}
76
+ supports_segment_timestamp = False
77
+
78
+ @classmethod
79
+ def get_speech_to_text_config(
80
+ cls, model_config: ModelConfig, task_type: str
81
+ ) -> SpeechToTextConfig:
82
+ return SpeechToTextConfig(
83
+ sample_rate=16000.0,
84
+ max_audio_clip_s=5.0,
85
+ )
86
+
87
+ @classmethod
88
+ def post_process_output(cls, text: str) -> str:
89
+ return text
90
+
91
+
92
+def _request_output(text: str) -> RequestOutput:
93
+ return RequestOutput(
94
+ request_id="rid",
95
+ prompt=None,
96
+ prompt_token_ids=None,
97
+ prompt_logprobs=None,
98
+ outputs=[
99
+ CompletionOutput(
100
+ index=0,
101
+ text=text,
102
+ token_ids=(1, 2, 3),
103
+ cumulative_logprob=None,
104
+ logprobs=None,
105
+ finish_reason="stop",
106
+ )
107
+ ],
108
+ finished=True,
109
+ )
110
+
111
+
112
+def _sse_delta_contents(sse_body: str) -> list[str]:
113
+ """Extract ``choices[0].delta.content`` from each ``data:`` line (streaming API)."""
114
+ contents: list[str] = []
115
+ for line in sse_body.splitlines():
116
+ if not line.startswith("data: "):
117
+ continue
118
+ payload = line.removeprefix("data: ").strip()
119
+ if payload == "[DONE]":
120
+ continue
121
+ obj = json.loads(payload)
122
+ for choice in obj.get("choices") or []:
123
+ delta = choice.get("delta") or {}
124
+ if "content" in delta:
125
+ contents.append(delta["content"])
126
+ return contents
127
+
128
+
129
+@pytest.mark.asyncio
130
+async def test_transcription_stream_generator_english_inserts_space_between_chunks():
131
+ """Online streaming: first output per audio chunk is prefixed with *separator*."""
132
+
133
+ async def gen_hello() -> AsyncGenerator[RequestOutput, None]:
134
+ yield _request_output("hello")
135
+
136
+ async def gen_world() -> AsyncGenerator[RequestOutput, None]:
137
+ yield _request_output("world")
138
+
139
+ serving = OpenAIServingTranscription.__new__(OpenAIServingTranscription)
140
+ serving.enable_force_include_usage = False
141
+ serving.model_cls = _StubTranscriptionModel
142
+ serving.task_type = "transcribe"
143
+ request = SimpleNamespace(
144
+ model="stub-model",
145
+ stream_include_usage=False,
146
+ stream_continuous_usage_stats=False,
147
+ )
148
+ sep = asr_inter_chunk_separator("en", _StubTranscriptionModel.no_space_languages)
149
+ assert sep == " "
150
+
151
+ out_lines: list[str] = []
152
+ agen = OpenAIServingTranscription.transcription_stream_generator(
153
+ serving,
154
+ request=request,
155
+ result_generator=[gen_hello(), gen_world()],
156
+ request_id="test-req",
157
+ request_metadata=RequestResponseMetadata(request_id="test-req"),
158
+ audio_duration_s=1.0,
159
+ separator=sep,
160
+ )
161
+ async for line in agen:
162
+ out_lines.append(line)
163
+ sse = "".join(out_lines)
164
+ combined = "".join(_sse_delta_contents(sse))
165
+ assert combined.strip() == "hello world"
166
+
167
+
168
+@pytest.mark.asyncio
169
+async def test_transcription_stream_generator_chinese_no_space_between_chunks():
170
+ async def gen_a() -> AsyncGenerator[RequestOutput, None]:
171
+ yield _request_output("你好")
172
+
173
+ async def gen_b() -> AsyncGenerator[RequestOutput, None]:
174
+ yield _request_output("世界")
175
+
176
+ serving = OpenAIServingTranscription.__new__(OpenAIServingTranscription)
177
+ serving.enable_force_include_usage = False
178
+ serving.model_cls = _StubTranscriptionModel
179
+ serving.task_type = "transcribe"
180
+ request = SimpleNamespace(
181
+ model="stub-model",
182
+ stream_include_usage=False,
183
+ stream_continuous_usage_stats=False,
184
+ )
185
+ sep = asr_inter_chunk_separator("zh", _StubTranscriptionModel.no_space_languages)
186
+ assert sep == ""
187
+
188
+ out_lines: list[str] = []
189
+ agen = OpenAIServingTranscription.transcription_stream_generator(
190
+ serving,
191
+ request=request,
192
+ result_generator=[gen_a(), gen_b()],
193
+ request_id="test-req-zh",
194
+ request_metadata=RequestResponseMetadata(request_id="test-req-zh"),
195
+ audio_duration_s=1.0,
196
+ separator=sep,
197
+ )
198
+ async for line in agen:
199
+ out_lines.append(line)
200
+ combined = "".join(_sse_delta_contents("".join(out_lines)))
201
+ assert combined == "你好世界"
202
+
203
+
204
+@pytest.mark.asyncio
205
+async def test_create_transcription_non_streaming_joins_chunks_by_language():
206
+ """``create_transcription`` uses the same separator logic as the helper."""
207
+
208
+ async def gen_hello() -> AsyncGenerator[RequestOutput, None]:
209
+ yield _request_output("hello")
210
+
211
+ async def gen_world() -> AsyncGenerator[RequestOutput, None]:
212
+ yield _request_output("world")
213
+
214
+ engine_client = MagicMock()
215
+ engine_client.model_config = MagicMock()
216
+ engine_client.model_config.get_diff_sampling_param.return_value = {
217
+ "max_tokens": 256,
218
+ "temperature": 0.0,
219
+ }
220
+ engine_client.model_config.max_model_len = 8192
221
+ engine_client.errored = False
222
+ engine_client.generate.side_effect = [gen_hello(), gen_world()]
223
+
224
+ models = MagicMock(spec=OpenAIServingModels)
225
+ models.lora_requests = {}
226
+ models.is_base_model.return_value = True
227
+
228
+ preprocess_mock = AsyncMock(return_value=([MagicMock(), MagicMock()], 1.0))
229
+
230
+ with (
231
+ patch(
232
+ "vllm.model_executor.model_loader.get_model_cls",
233
+ return_value=_StubTranscriptionModel,
234
+ ),
235
+ patch.object(OpenAISpeechToText, "_preprocess_speech_to_text", preprocess_mock),
236
+ ):
237
+ serving = OpenAIServingTranscription(engine_client, models, request_logger=None)
238
+
239
+ req_en = TranscriptionRequest.model_construct(
240
+ file=MagicMock(),
241
+ model="stub-model",
242
+ language="en",
243
+ stream=False,
244
+ response_format="json",
245
+ )
246
+ out_en = await serving.create_transcription(
247
+ b"\x00\x00", req_en, raw_request=None
248
+ )
249
+ assert not isinstance(out_en, ErrorResponse)
250
+ assert out_en.text == "hello world"
251
+
252
+ async def gen_nihao() -> AsyncGenerator[RequestOutput, None]:
253
+ yield _request_output("你好")
254
+
255
+ async def gen_shijie() -> AsyncGenerator[RequestOutput, None]:
256
+ yield _request_output("世界")
257
+
258
+ engine_client.generate.side_effect = [gen_nihao(), gen_shijie()]
259
+
260
+ req_zh = TranscriptionRequest.model_construct(
261
+ file=MagicMock(),
262
+ model="stub-model",
263
+ language="zh",
264
+ stream=False,
265
+ response_format="json",
266
+ )
267
+ out_zh = await serving.create_transcription(
268
+ b"\x00\x00", req_zh, raw_request=None
269
+ )
270
+ assert not isinstance(out_zh, ErrorResponse)
271
+ assert out_zh.text == "你好世界"
vllm/entrypoints/openai/speech_to_text/serving.py
+4
@@ -86,6 +86,7 @@ class OpenAIServingTranscription(OpenAISpeechToText):
86
request_id: str,
87
request_metadata: RequestResponseMetadata,
88
audio_duration_s: float,
89
+ separator: str,
90
) -> AsyncGenerator[str, None]:
91
generator = self._speech_to_text_stream_generator(
92
request=request,
@@ -96,6 +97,7 @@ class OpenAIServingTranscription(OpenAISpeechToText):
97
chunk_object_type="transcription.chunk",
98
response_stream_choice_class=TranscriptionResponseStreamChoice,
99
stream_response_class=TranscriptionStreamResponse,
100
+ separator=separator,
101
)
102
async for chunk in generator:
103
yield chunk
@@ -157,6 +159,7 @@ class OpenAIServingTranslation(OpenAISpeechToText):
159
request_id: str,
160
request_metadata: RequestResponseMetadata,
161
audio_duration_s: float,
162
+ separator: str,
163
) -> AsyncGenerator[str, None]:
164
generator = self._speech_to_text_stream_generator(
165
request=request,
@@ -167,6 +170,7 @@ class OpenAIServingTranslation(OpenAISpeechToText):
170
chunk_object_type="translation.chunk",
171
response_stream_choice_class=TranslationResponseStreamChoice,
172
stream_response_class=TranslationStreamResponse,
173
+ separator=separator,
174
)
175
async for chunk in generator:
176
yield chunk
vllm/entrypoints/openai/speech_to_text/speech_to_text.py
+33
-4
@@ -5,7 +5,7 @@ import io
5
import math
6
import time
7
import zlib
8
-from collections.abc import AsyncGenerator, Callable
8
+from collections.abc import AsyncGenerator, Callable, Set
9
from functools import cached_property
10
from typing import Final, Literal, TypeAlias, TypeVar, cast
11
@@ -69,6 +69,17 @@ ResponseType: TypeAlias = (
69
logger = init_logger(__name__)
70
71
72
+def asr_inter_chunk_separator(
73
+ language: str | None, no_space_languages: Set[str]
74
+) -> str:
75
+ """Space to insert between ASR text chunks for streaming and non-streaming join.
76
+
77
+ Languages in ``no_space_languages`` (e.g. Chinese, Japanese) use an empty
78
+ separator; others use a single ASCII space.
79
+ """
80
+ return "" if language and language.lower() in no_space_languages else " "
81
+
82
+
83
class OpenAISpeechToText(OpenAIServing):
84
"""Base class for speech-to-text operations like transcription and
85
translation."""
@@ -486,9 +497,18 @@ class OpenAISpeechToText(OpenAIServing):
497
498
list_result_generator.append(generator)
499
500
+ separator = asr_inter_chunk_separator(
501
+ request.language, self.model_cls.no_space_languages
502
+ )
503
+
504
if request.stream:
505
return stream_generator_method(
491
- request, list_result_generator, request_id, request_metadata, duration_s
506
+ request,
507
+ list_result_generator,
508
+ request_id,
509
+ request_metadata,
510
+ duration_s,
511
+ separator,
512
)
513
# Non-streaming response.
514
total_segments = []
@@ -500,7 +520,6 @@ class OpenAISpeechToText(OpenAIServing):
520
"translate": TranslationSegment,
521
}
522
segment_class: type[SpeechToTextSegment] = segments_types[self.task_type]
503
- text = ""
523
chunk_size_in_s = self.asr_config.max_audio_clip_s
524
if chunk_size_in_s is None:
525
assert len(list_result_generator) == 1, (
@@ -528,7 +547,7 @@ class OpenAISpeechToText(OpenAIServing):
547
else:
548
raw_text = op.outputs[0].text
549
text_parts.append(self.model_cls.post_process_output(raw_text))
531
- text = "".join(text_parts)
550
+ text = separator.join(text_parts)
551
if self.task_type == "transcribe":
552
final_response: ResponseType
553
# add usage in TranscriptionResponse.
@@ -581,6 +600,7 @@ class OpenAISpeechToText(OpenAIServing):
600
| type[TranslationResponseStreamChoice],
601
stream_response_class: type[TranscriptionStreamResponse]
602
| type[TranslationStreamResponse],
603
+ separator: str,
604
) -> AsyncGenerator[str, None]:
605
created_time = int(time.time())
606
model_name = request.model
@@ -597,6 +617,7 @@ class OpenAISpeechToText(OpenAIServing):
617
618
try:
619
for result_generator in list_result_generator:
620
+ beginning_of_chunk = True
621
async for res in result_generator:
622
# On first result.
623
if res.prompt_token_ids is not None:
@@ -614,6 +635,14 @@ class OpenAISpeechToText(OpenAIServing):
635
assert len(res.outputs) == 1
636
output = res.outputs[0]
637
638
+ # dont add separator to the first chunk
639
+ if (
640
+ result_generator is not list_result_generator[0]
641
+ and beginning_of_chunk
642
+ ):
643
+ output.text = separator + output.text
644
+ beginning_of_chunk = False
645
+
646
# TODO: For models that output structured formats (e.g.,
647
# Qwen3-ASR with "language X<asr_text>" prefix), streaming
648
# would need buffering to strip the prefix properly since
vllm/model_executor/models/cohere_asr.py
+1
@@ -2007,6 +2007,7 @@ class CohereAsrForConditionalGeneration(
2007
supports_transcription_only = True
2008
supported_languages = ISO639_1_SUPPORTED_LANGS
2009
skip_warmup_audio_preprocessing = True
2010
+ no_space_languages = {"ja", "zh"}
2011
2012
@classmethod
2013
def validate_language(cls, language: str | None) -> str | None:
vllm/model_executor/models/interfaces.py
+6
@@ -1098,6 +1098,12 @@ class SupportsTranscription(Protocol):
1098
:meth:`get_language_token_ids`.
1099
"""
1100
1101
+ no_space_languages: ClassVar[set[str]] = {"ja", "zh"}
1102
+ """
1103
+ Languages that don't need a space between words.
1104
+ For example, Japanese (ja) and Chinese (zh) don't need a space between words.
1105
+ """
1106
+
1107
def __init_subclass__(cls, **kwargs):
1108
super().__init_subclass__(**kwargs)
1109
# language codes in supported_languages