A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 04:45 UTC
[Misc] refactor example - openai_transcription_client (#19851)
Signed-off-by: reidliu41 <reid201711@gmail.com> Co-authored-by: reidliu41 <reid201711@gmail.com>
Reid committed
Jun 20, 2025 at 16:02 UTC
e384f2f10824df7789c6da35256cf957788c0208
1 file changed
+34
-5
examples/online_serving/openai_transcription_client.py
+34
-5
@@ -1,5 +1,23 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""
4
+This script demonstrates how to use the vLLM API server to perform audio
5
+transcription with the `openai/whisper-large-v3` model.
6
+
7
+Before running this script, you must start the vLLM server with the following command:
8
+
9
+ vllm serve openai/whisper-large-v3
10
+
11
+Requirements:
12
+- vLLM with audio support
13
+- openai Python SDK
14
+- httpx for streaming support
15
+
16
+The script performs:
17
+1. Synchronous transcription using OpenAI-compatible API.
18
+2. Streaming transcription using raw HTTP request to the vLLM server.
19
+"""
20
+
21
import asyncio
22
import json
23
@@ -21,6 +39,9 @@ client = OpenAI(
39
40
41
def sync_openai():
42
+ """
43
+ Perform synchronous transcription using OpenAI-compatible API.
44
+ """
45
with open(str(mary_had_lamb), "rb") as f:
46
transcription = client.audio.transcriptions.create(
47
file=f,
@@ -37,11 +58,11 @@ def sync_openai():
58
print("transcription result:", transcription.text)
59
60
40
-sync_openai()
41
-
42
-
61
# OpenAI Transcription API client does not support streaming.
62
async def stream_openai_response():
63
+ """
64
+ Perform streaming transcription using vLLM's raw HTTP streaming API.
65
+ """
66
data = {
67
"language": "en",
68
"stream": True,
@@ -68,7 +89,15 @@ async def stream_openai_response():
89
# Extract and print the content
90
content = chunk["choices"][0].get("delta", {}).get("content")
91
print(content, end="")
92
+ print() # Final newline after stream ends
93
+
94
+
95
+def main():
96
+ sync_openai()
97
+
98
+ # Run the asynchronous function
99
+ asyncio.run(stream_openai_response())
100
101
73
-# Run the asynchronous function
74
-asyncio.run(stream_openai_response())
102
+if __name__ == "__main__":
103
+ main()