[Bugfix] Fix Qwen3-ASR transcription streaming postprocessing (#42478)
Signed-off-by: JooHo Lee <BWAAEEEK@users.noreply.github.com> Signed-off-by: JooHo Lee <jooho414@gmail.com> Co-authored-by: JooHo Lee <BWAAEEEK@users.noreply.github.com>
JooHo Lee committed
Jul 9, 2026 at 16:33 UTC
a07765c6bd102bc1beaa5226d0fe5291b30a8524
4 files changed
+207
-17
tests/entrypoints/speech_to_text/transcription/test_transcription_inter_chunk_spacing.py
+118
-3
@@ -32,7 +32,11 @@ from vllm.entrypoints.speech_to_text.transcription.protocol import Transcription
32
from vllm.entrypoints.speech_to_text.transcription.serving import (
33
OpenAIServingTranscription,
34
)
35
-from vllm.model_executor.models.interfaces import SupportsTranscription
35
+from vllm.model_executor.models.interfaces import (
36
+ StreamingTranscriptionPostProcessor,
37
+ SupportsTranscription,
38
+)
39
+from vllm.model_executor.models.qwen3_asr import Qwen3ASRForConditionalGeneration
40
from vllm.outputs import CompletionOutput, RequestOutput
41
42
# --- Unit: helper + protocol -------------------------------------------------
@@ -58,6 +62,63 @@ def test_asr_inter_chunk_separator_matches_protocol(language, expected_sep):
62
assert sep == expected_sep
63
64
65
+def test_qwen3_asr_stream_processor_passes_plain_text_without_prefix():
66
+ post_processor = (
67
+ Qwen3ASRForConditionalGeneration.get_streaming_post_processor_cls()()
68
+ )
69
+
70
+ assert post_processor.process_delta("Hello", False) == "Hello"
71
+ assert post_processor.process_delta(" world", True) == " world"
72
+
73
+
74
+def test_qwen3_asr_stream_processor_buffers_prefix_with_leading_space():
75
+ post_processor = (
76
+ Qwen3ASRForConditionalGeneration.get_streaming_post_processor_cls()()
77
+ )
78
+
79
+ assert post_processor.process_delta(" language Eng", False) == ""
80
+ assert post_processor.process_delta("lish<asr", False) == ""
81
+ assert post_processor.process_delta("_text>Hello", True) == "Hello"
82
+
83
+
84
+def test_qwen3_asr_stream_processor_keeps_independent_state():
85
+ processor_cls = Qwen3ASRForConditionalGeneration.get_streaming_post_processor_cls()
86
+ first_processor = processor_cls()
87
+ second_processor = processor_cls()
88
+
89
+ assert first_processor.process_delta(" language Eng", False) == ""
90
+ assert second_processor.process_delta("plain text", True) == "plain text"
91
+ assert first_processor.process_delta("lish<asr_text>Hello", True) == "Hello"
92
+
93
+
94
+def test_qwen3_asr_stream_processor_emits_finished_incomplete_prefix():
95
+ post_processor = (
96
+ Qwen3ASRForConditionalGeneration.get_streaming_post_processor_cls()()
97
+ )
98
+
99
+ assert (
100
+ post_processor.process_delta(" language English", True) == " language English"
101
+ )
102
+
103
+
104
+def test_qwen3_asr_stream_processor_stops_buffering_long_plain_prefix():
105
+ post_processor = (
106
+ Qwen3ASRForConditionalGeneration.get_streaming_post_processor_cls()()
107
+ )
108
+ text = " language " + ("x" * 50)
109
+
110
+ assert post_processor.process_delta(text, False) == text
111
+
112
+
113
+def test_qwen3_asr_stream_processor_stops_buffering_prefix_with_newline():
114
+ post_processor = (
115
+ Qwen3ASRForConditionalGeneration.get_streaming_post_processor_cls()()
116
+ )
117
+ text = " language English\nhello"
118
+
119
+ assert post_processor.process_delta(text, False) == text
120
+
121
+
122
def test_joined_chunks_english_has_space_between():
123
sep = asr_inter_chunk_separator("en", SupportsTranscription.no_space_languages)
124
assert sep.join(["hello", "world"]) == "hello world"
@@ -90,8 +151,14 @@ class _StubTranscriptionModel:
151
def post_process_output(cls, text: str) -> str:
152
return text
153
154
+ @classmethod
155
+ def get_streaming_post_processor_cls(
156
+ cls,
157
+ ) -> type[StreamingTranscriptionPostProcessor]:
158
+ return StreamingTranscriptionPostProcessor
159
+
160
94
-def _request_output(text: str) -> RequestOutput:
161
+def _request_output(text: str, finish_reason: str | None = "stop") -> RequestOutput:
162
return RequestOutput(
163
request_id="rid",
164
prompt=None,
@@ -104,7 +171,7 @@ def _request_output(text: str) -> RequestOutput:
171
token_ids=(1, 2, 3),
172
cumulative_logprob=None,
173
logprobs=None,
107
- finish_reason="stop",
174
+ finish_reason=finish_reason,
175
)
176
],
177
finished=True,
@@ -141,6 +208,9 @@ async def test_transcription_stream_generator_english_inserts_space_between_chun
208
serving = OpenAIServingTranscription.__new__(OpenAIServingTranscription)
209
serving.enable_force_include_usage = False
210
serving.model_cls = _StubTranscriptionModel
211
+ serving.streaming_post_processor_cls = (
212
+ _StubTranscriptionModel.get_streaming_post_processor_cls()
213
+ )
214
serving.task_type = "transcribe"
215
request = SimpleNamespace(
216
model="stub-model",
@@ -178,6 +248,9 @@ async def test_transcription_stream_generator_chinese_no_space_between_chunks():
248
serving = OpenAIServingTranscription.__new__(OpenAIServingTranscription)
249
serving.enable_force_include_usage = False
250
serving.model_cls = _StubTranscriptionModel
251
+ serving.streaming_post_processor_cls = (
252
+ _StubTranscriptionModel.get_streaming_post_processor_cls()
253
+ )
254
serving.task_type = "transcribe"
255
request = SimpleNamespace(
256
model="stub-model",
@@ -203,6 +276,48 @@ async def test_transcription_stream_generator_chinese_no_space_between_chunks():
276
assert combined == "你好世界"
277
278
279
+@pytest.mark.asyncio
280
+async def test_transcription_stream_generator_strips_qwen3_asr_prefix_per_chunk():
281
+ async def gen_hello() -> AsyncGenerator[RequestOutput, None]:
282
+ yield _request_output("language Eng", finish_reason=None)
283
+ yield _request_output("lish<asr", finish_reason=None)
284
+ yield _request_output("_text>Hello", finish_reason=None)
285
+ yield _request_output("")
286
+
287
+ async def gen_world() -> AsyncGenerator[RequestOutput, None]:
288
+ yield _request_output(" language Eng", finish_reason=None)
289
+ yield _request_output("lish<asr_text>world")
290
+
291
+ serving = OpenAIServingTranscription.__new__(OpenAIServingTranscription)
292
+ serving.enable_force_include_usage = False
293
+ serving.model_cls = Qwen3ASRForConditionalGeneration
294
+ serving.streaming_post_processor_cls = (
295
+ Qwen3ASRForConditionalGeneration.get_streaming_post_processor_cls()
296
+ )
297
+ serving.task_type = "transcribe"
298
+ request = SimpleNamespace(
299
+ model="stub-qwen3-asr",
300
+ stream_include_usage=False,
301
+ stream_continuous_usage_stats=False,
302
+ )
303
+
304
+ out_lines: list[str] = []
305
+ agen = OpenAIServingTranscription.transcription_stream_generator(
306
+ serving,
307
+ request=request,
308
+ result_generator=[gen_hello(), gen_world()],
309
+ request_id="test-qwen3-asr",
310
+ request_metadata=RequestResponseMetadata(request_id="test-qwen3-asr"),
311
+ audio_duration_s=1.0,
312
+ separator=" ",
313
+ )
314
+ async for line in agen:
315
+ out_lines.append(line)
316
+
317
+ combined = "".join(_sse_delta_contents("".join(out_lines)))
318
+ assert combined == "Hello world"
319
+
320
+
321
@pytest.mark.asyncio
322
async def test_create_transcription_non_streaming_joins_chunks_by_language():
323
"""``create_transcription`` uses the same separator logic as the helper."""
vllm/entrypoints/speech_to_text/base/serving.py
+15
-6
@@ -114,6 +114,9 @@ class SpeechToTextBaseServing(GenerateBaseServing):
114
self.asr_config = self.model_cls.get_speech_to_text_config(
115
self.model_config, task_type
116
)
117
+ self.streaming_post_processor_cls = (
118
+ self.model_cls.get_streaming_post_processor_cls()
119
+ )
120
121
self.enable_force_include_usage = enable_force_include_usage
122
@@ -692,6 +695,7 @@ class SpeechToTextBaseServing(GenerateBaseServing):
695
try:
696
for result_generator in list_result_generator:
697
beginning_of_chunk = True
698
+ post_processor = self.streaming_post_processor_cls()
699
async for res in result_generator:
700
# On first result.
701
if res.prompt_token_ids is not None:
@@ -709,19 +713,24 @@ class SpeechToTextBaseServing(GenerateBaseServing):
713
assert len(res.outputs) == 1
714
output = res.outputs[0]
715
716
+ output_text = post_processor.process_delta(
717
+ output.text, output.finish_reason is not None
718
+ )
719
+
720
# dont add separator to the first chunk
721
if (
722
result_generator is not list_result_generator[0]
723
and beginning_of_chunk
724
+ and output_text
725
):
717
- output.text = separator + output.text
726
+ output_text = separator + output_text
727
beginning_of_chunk = False
728
720
- # TODO: For models that output structured formats (e.g.,
721
- # Qwen3-ASR with "language X<asr_text>" prefix), streaming
722
- # would need buffering to strip the prefix properly since
723
- # deltas may split the tag across chunks.
724
- delta_message = DeltaMessage(content=output.text)
729
+ if output.finish_reason is None and not output_text:
730
+ completion_tokens += len(output.token_ids)
731
+ continue
732
+
733
+ delta_message = DeltaMessage(content=output_text)
734
completion_tokens += len(output.token_ids)
735
736
if output.finish_reason is None:
vllm/model_executor/models/interfaces.py
+20
@@ -70,6 +70,13 @@ The output embeddings must be one of the following formats:
70
"""
71
72
73
+class StreamingTranscriptionPostProcessor:
74
+ """Stateful streaming post-processor for transcription deltas."""
75
+
76
+ def process_delta(self, text_delta: str, finished: bool) -> str:
77
+ return text_delta
78
+
79
+
80
def _require_is_multimodal(is_multimodal: Tensor | None) -> Tensor:
81
"""
82
A helper function to be used in the context of
@@ -1198,6 +1205,19 @@ class SupportsTranscription(Protocol):
1205
"""
1206
return text
1207
1208
+ @classmethod
1209
+ def get_streaming_post_processor_cls(
1210
+ cls,
1211
+ ) -> type[StreamingTranscriptionPostProcessor]:
1212
+ """
1213
+ Return a stateful post-processor class for streaming output deltas.
1214
+
1215
+ Each instance receives the next decoded text delta and whether the
1216
+ request output is final. It returns the cleaned delta that should be
1217
+ sent to the client.
1218
+ """
1219
+ return StreamingTranscriptionPostProcessor
1220
+
1221
@classmethod
1222
def get_language_detection_prompt(
1223
cls,
vllm/model_executor/models/qwen3_asr.py
+54
-8
@@ -38,6 +38,7 @@ from vllm.inputs import ModalityData, MultiModalDataDict, PromptType, TokensProm
38
from vllm.logger import init_logger
39
from vllm.model_executor.models.interfaces import (
40
MultiModalEmbeddings,
41
+ StreamingTranscriptionPostProcessor,
42
SupportsLoRA,
43
SupportsMRoPE,
44
SupportsMultiModal,
@@ -93,6 +94,53 @@ logger = init_logger(__name__)
94
_ASR_TEXT_TAG = "<asr_text>"
95
# User-supplied `prompt` / `response_prefix` must not inject extra ChatML turns.
96
_CHATML_LIKE_TOKEN = re.compile(r"<\|[^|]+\|>")
97
+_LANGUAGE_PREFIX = "language "
98
+_MAX_STREAMING_PREFIX_CHARS = 50
99
+
100
+
101
+def _post_process_qwen3_asr_output(text: str) -> str:
102
+ if not text or _ASR_TEXT_TAG not in text:
103
+ return text
104
+
105
+ _, text_part = text.rsplit(_ASR_TEXT_TAG, 1)
106
+ return text_part
107
+
108
+
109
+class Qwen3ASRStreamingPostProcessor(StreamingTranscriptionPostProcessor):
110
+ def __init__(self) -> None:
111
+ self.raw_text = ""
112
+ self.emitted_text = ""
113
+ self.is_structured_output: bool | None = None
114
+
115
+ def process_delta(self, text_delta: str, finished: bool) -> str:
116
+ self.raw_text += text_delta
117
+
118
+ if self.is_structured_output is None:
119
+ text_without_leading_space = self.raw_text.lstrip()
120
+ maybe_structured_output = (
121
+ text_without_leading_space == ""
122
+ or _LANGUAGE_PREFIX.startswith(text_without_leading_space)
123
+ or (
124
+ text_without_leading_space.startswith(_LANGUAGE_PREFIX)
125
+ and len(text_without_leading_space) < _MAX_STREAMING_PREFIX_CHARS
126
+ and "\n" not in text_without_leading_space
127
+ )
128
+ )
129
+ if maybe_structured_output and _ASR_TEXT_TAG not in self.raw_text:
130
+ if not finished:
131
+ return ""
132
+ self.is_structured_output = False
133
+ else:
134
+ self.is_structured_output = _ASR_TEXT_TAG in self.raw_text
135
+
136
+ processed_text = (
137
+ _post_process_qwen3_asr_output(self.raw_text)
138
+ if self.is_structured_output
139
+ else self.raw_text
140
+ )
141
+ new_text = processed_text[len(self.emitted_text) :]
142
+ self.emitted_text = processed_text
143
+ return new_text
144
145
146
def _sanitize_transcription_user_text(text: str) -> str:
@@ -632,12 +680,10 @@ class Qwen3ASRForConditionalGeneration(
680
The model outputs in format: "language {lang}<asr_text>{transcription}"
681
This method strips the language prefix and asr_text tags.
682
"""
635
- if not text:
636
- return ""
637
-
638
- if _ASR_TEXT_TAG not in text:
639
- return text
683
+ return _post_process_qwen3_asr_output(text)
684
641
- # Split on <asr_text> tag and take the transcription part
642
- _, text_part = text.rsplit(_ASR_TEXT_TAG, 1)
643
- return text_part
685
+ @classmethod
686
+ def get_streaming_post_processor_cls(
687
+ cls,
688
+ ) -> type[StreamingTranscriptionPostProcessor]:
689
+ return Qwen3ASRStreamingPostProcessor