A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 06:15 UTC
[Bugfix] Fix `Qwen3ASR` language asr tag in output (#33410)
Signed-off-by: NickLucche <nlucches@redhat.com>
Nicolò Lucchesi committed
Jan 31, 2026 at 06:24 UTC
e77f162cf59d6e132c8b6449dc98b410b347a5b5
3 files changed
+42
-2
vllm/entrypoints/openai/translations/speech_to_text.py
+6
-1
@@ -518,7 +518,8 @@ class OpenAISpeechToText(OpenAIServing):
518
total_segments.extend(segments)
519
text_parts.extend([seg.text for seg in segments])
520
else:
521
- text_parts.append(op.outputs[0].text)
521
+ raw_text = op.outputs[0].text
522
+ text_parts.append(self.model_cls.post_process_output(raw_text))
523
text = "".join(text_parts)
524
if self.task_type == "transcribe":
525
final_response: ResponseType
@@ -607,6 +608,10 @@ class OpenAISpeechToText(OpenAIServing):
608
assert len(res.outputs) == 1
609
output = res.outputs[0]
610
611
+ # TODO: For models that output structured formats (e.g.,
612
+ # Qwen3-ASR with "language X<asr_text>" prefix), streaming
613
+ # would need buffering to strip the prefix properly since
614
+ # deltas may split the tag across chunks.
615
delta_message = DeltaMessage(content=output.text)
616
completion_tokens += len(output.token_ids)
617
vllm/model_executor/models/interfaces.py
+16
@@ -1145,6 +1145,22 @@ class SupportsTranscription(Protocol):
1145
"""
1146
return None
1147
1148
+ @classmethod
1149
+ def post_process_output(cls, text: str) -> str:
1150
+ """
1151
+ Post-process the raw model output text.
1152
+
1153
+ Some ASR models output structured formats (e.g., language tags,
1154
+ special tokens) that need to be stripped before returning to the user.
1155
+
1156
+ Args:
1157
+ text: Raw decoded text from the model.
1158
+
1159
+ Returns:
1160
+ Cleaned transcription text.
1161
+ """
1162
+ return text
1163
+
1164
1165
@overload
1166
def supports_transcription(
vllm/model_executor/models/qwen3_asr.py
+20
-1
@@ -90,6 +90,7 @@ from vllm.transformers_utils.processors.qwen3_asr import (
90
)
91
92
logger = init_logger(__name__)
93
+_ASR_TEXT_TAG = "<asr_text>"
94
95
96
def _get_feat_extract_output_lengths(input_lengths: torch.Tensor):
@@ -556,7 +557,7 @@ class Qwen3ASRForConditionalGeneration(
557
else:
558
prompt = (
559
f"<|im_start|>user\n{audio_placeholder}<|im_end|>\n"
559
- f"<|im_start|>assistant\nlanguage {full_lang_name_to}<asr_text>"
560
+ f"<|im_start|>assistant\nlanguage {full_lang_name_to}{_ASR_TEXT_TAG}"
561
)
562
563
prompt_token_ids = tokenizer.encode(prompt)
@@ -565,3 +566,21 @@ class Qwen3ASRForConditionalGeneration(
566
"multi_modal_data": {"audio": audio},
567
}
568
return cast(PromptType, prompt_dict)
569
+
570
+ @classmethod
571
+ def post_process_output(cls, text: str) -> str:
572
+ """
573
+ Post-process Qwen3-ASR raw output to extract clean transcription.
574
+
575
+ The model outputs in format: "language {lang}<asr_text>{transcription}"
576
+ This method strips the language prefix and asr_text tags.
577
+ """
578
+ if not text:
579
+ return ""
580
+
581
+ if _ASR_TEXT_TAG not in text:
582
+ return text
583
+
584
+ # Split on <asr_text> tag and take the transcription part
585
+ _, text_part = text.rsplit(_ASR_TEXT_TAG, 1)
586
+ return text_part