A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 20:00 UTC
support hotwords for FunASR model (#39674)
Signed-off-by: zixiao <shunli.dsl@alibaba-inc.com> Co-authored-by: zixiao <shunli.dsl@alibaba-inc.com>
AllenDou committed
Apr 22, 2026 at 17:25 UTC
9047288b68f387331932598a8ba398c8d03a7d8c
4 files changed
+50
-4
examples/online_serving/openai_transcription_client.py
+22
-3
@@ -27,7 +27,12 @@ from vllm.assets.audio import AudioAsset
27
28
29
def sync_openai(
30
- audio_path: str, client: OpenAI, model: str, *, repetition_penalty: float = 1.3
30
+ audio_path: str,
31
+ client: OpenAI,
32
+ model: str,
33
+ *,
34
+ repetition_penalty: float = 1.3,
35
+ hotwords: str = None,
36
):
37
"""
38
Perform synchronous transcription using OpenAI-compatible API.
@@ -43,12 +48,15 @@ def sync_openai(
48
extra_body=dict(
49
seed=4419,
50
repetition_penalty=repetition_penalty,
51
+ hotwords=hotwords,
52
),
53
)
54
print("transcription result [sync]:", transcription.text)
55
56
51
-async def stream_openai_response(audio_path: str, client: AsyncOpenAI, model: str):
57
+async def stream_openai_response(
58
+ audio_path: str, client: AsyncOpenAI, model: str, hotwords: str = None
59
+):
60
"""
61
Perform asynchronous transcription using OpenAI-compatible API.
62
"""
@@ -64,6 +72,7 @@ async def stream_openai_response(audio_path: str, client: AsyncOpenAI, model: st
72
extra_body=dict(
73
seed=420,
74
top_p=0.6,
75
+ hotwords=hotwords,
76
),
77
stream=True,
78
)
@@ -136,6 +145,7 @@ def main(args):
145
client=client,
146
model=model,
147
repetition_penalty=args.repetition_penalty,
148
+ hotwords=args.hotwords,
149
)
150
151
# Run the asynchronous function
@@ -146,7 +156,10 @@ def main(args):
156
)
157
asyncio.run(
158
stream_openai_response(
149
- args.audio_path if args.audio_path else winning_call, client, model
159
+ args.audio_path if args.audio_path else winning_call,
160
+ client,
161
+ model,
162
+ hotwords=args.hotwords,
163
)
164
)
165
else:
@@ -174,5 +187,11 @@ if __name__ == "__main__":
187
default=1.3,
188
help="repetition penalty",
189
)
190
+ parser.add_argument(
191
+ "--hotwords",
192
+ type=str,
193
+ default=None,
194
+ help="hotwords",
195
+ )
196
args = parser.parse_args()
197
main(args)
vllm/config/speech_to_text.py
+6
@@ -35,6 +35,12 @@ class SpeechToTextParams:
35
language: str | None = None
36
"""ISO 639-1 language code (validated / auto-detected)."""
37
38
+ hotwords: str | None = None
39
+ """
40
+ hotwords refers to a list of important words or phrases that the model
41
+ should pay extra attention to during transcription.
42
+ """
43
+
44
task_type: str = "transcribe"
45
"""``"transcribe"`` or ``"translate"``."""
46
vllm/entrypoints/openai/speech_to_text/protocol.py
+14
@@ -78,6 +78,12 @@ class TranscriptionRequest(OpenAIBaseModel):
78
will improve accuracy and latency.
79
"""
80
81
+ hotwords: str | None = None
82
+ """
83
+ hotwords refers to a list of important words or phrases that the model
84
+ should pay extra attention to during transcription.
85
+ """
86
+
87
prompt: str = Field(default="")
88
"""An optional text to guide the model's style or continue a previous audio
89
segment.
@@ -205,6 +211,7 @@ class TranscriptionRequest(OpenAIBaseModel):
211
task_type=task_type,
212
request_prompt=self.prompt,
213
to_language=self.to_language,
214
+ hotwords=self.hotwords,
215
)
216
217
def to_beam_search_params(
@@ -481,6 +488,12 @@ class TranslationRequest(OpenAIBaseModel):
488
will improve accuracy.
489
"""
490
491
+ hotwords: str | None = None
492
+ """
493
+ hotwords refers to a list of important words or phrases that the model
494
+ should pay extra attention to during transcription.
495
+ """
496
+
497
to_language: str | None = None
498
"""The language of the input audio we translate to.
499
@@ -522,6 +535,7 @@ class TranslationRequest(OpenAIBaseModel):
535
task_type=task_type,
536
request_prompt=self.prompt,
537
to_language=self.to_language,
538
+ hotwords=self.hotwords,
539
)
540
541
def to_beam_search_params(
vllm/model_executor/models/funasr.py
+8
-1
@@ -881,13 +881,20 @@ class FunASRForConditionalGeneration(
881
audio = stt_params.audio
882
stt_config = stt_params.stt_config
883
language = stt_params.language
884
+ hotwords = stt_params.hotwords
885
886
if language is None:
887
raise ValueError(
888
"Language must be specified when creating the funasr prompt"
889
)
890
890
- funasr_prompt = "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n语音转写：<|AUDIO|><|im_end|>\n<|im_start|>assistant\n" # noqa: E501
891
+ if hotwords is not None:
892
+ funasr_prompt = "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n请结合上下文信息，更加准确地完成语音转写任务。如果没有相关信息，我们会留空。\n\n\n**上下文信息：**\n\n\n热词列表：[{}]\n语音转写：<|AUDIO|><|im_end|>\n<|im_start|>assistant\n".format( # noqa: E501
893
+ hotwords
894
+ )
895
+ else:
896
+ funasr_prompt = "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n语音转写：<|AUDIO|><|im_end|>\n<|im_start|>assistant\n" # noqa: E501
897
+
898
prompt = {
899
"prompt": funasr_prompt,
900
"multi_modal_data": {