A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 10:15 UTC
[Doc] Remove hardcoded Whisper in example openai translation client (#32027)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Isotr0py committed
Jan 9, 2026 at 22:44 UTC
2d0c5b630e87cd48675fbdd833fe24785a57b1a2
1 file changed
+12
-6
examples/online_serving/openai_translation_client.py
+12
-6
@@ -9,11 +9,11 @@ from openai import OpenAI
9
from vllm.assets.audio import AudioAsset
10
11
12
-def sync_openai(audio_path: str, client: OpenAI):
12
+def sync_openai(audio_path: str, client: OpenAI, model: str):
13
with open(audio_path, "rb") as f:
14
translation = client.audio.translations.create(
15
file=f,
16
- model="openai/whisper-large-v3",
16
+ model=model,
17
response_format="json",
18
temperature=0.0,
19
# Additional params not provided by OpenAI API.
@@ -26,11 +26,13 @@ def sync_openai(audio_path: str, client: OpenAI):
26
print("translation result:", translation.text)
27
28
29
-async def stream_openai_response(audio_path: str, base_url: str, api_key: str):
29
+async def stream_openai_response(
30
+ audio_path: str, base_url: str, api_key: str, model: str
31
+):
32
data = {
33
"language": "it",
34
"stream": True,
33
- "model": "openai/whisper-large-v3",
35
+ "model": model,
36
}
37
url = base_url + "/audio/translations"
38
headers = {"Authorization": f"Bearer {api_key}"}
@@ -66,9 +68,13 @@ def main():
68
api_key=openai_api_key,
69
base_url=openai_api_base,
70
)
69
- sync_openai(foscolo, client)
71
+
72
+ model = client.models.list().data[0].id
73
+ print(f"Using model: {model}")
74
+
75
+ sync_openai(foscolo, client, model)
76
# Run the asynchronous function
71
- asyncio.run(stream_openai_response(foscolo, openai_api_base, openai_api_key))
77
+ asyncio.run(stream_openai_response(foscolo, openai_api_base, openai_api_key, model))
78
79
80
if __name__ == "__main__":