A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 03:30 UTC
[Docs] Update transcriptions API to use openai client with `stream=True` (#20271)
Signed-off-by: NickLucche <nlucches@redhat.com>
Nicolò Lucchesi committed
Jul 1, 2025 at 17:47 UTC
314af8617c628be1eeb89626bff1be5bc8f81e6b
2 files changed
+31
-37
examples/online_serving/openai_transcription_client.py
+28
-33
@@ -19,10 +19,8 @@ The script performs:
19
"""
20
21
import asyncio
22
-import json
22
24
-import httpx
25
-from openai import OpenAI
23
+from openai import AsyncOpenAI, OpenAI
24
25
from vllm.assets.audio import AudioAsset
26
@@ -47,37 +45,30 @@ def sync_openai(audio_path: str, client: OpenAI):
45
print("transcription result:", transcription.text)
46
47
50
-async def stream_openai_response(audio_path: str, base_url: str, api_key: str):
48
+async def stream_openai_response(audio_path: str, client: AsyncOpenAI):
49
"""
52
- Perform streaming transcription using vLLM's raw HTTP streaming API.
50
+ Perform asynchronous transcription using OpenAI-compatible API.
51
"""
54
- data = {
55
- "language": "en",
56
- "stream": True,
57
- "model": "openai/whisper-large-v3",
58
- }
59
- url = base_url + "/audio/transcriptions"
60
- headers = {"Authorization": f"Bearer {api_key}"}
61
- print("transcription result:", end=" ")
62
- # OpenAI Transcription API client does not support streaming.
63
- async with httpx.AsyncClient() as client:
64
- with open(audio_path, "rb") as f:
65
- async with client.stream(
66
- "POST", url, files={"file": f}, data=data, headers=headers
67
- ) as response:
68
- async for line in response.aiter_lines():
69
- # Each line is a JSON object prefixed with 'data: '
70
- if line:
71
- if line.startswith("data: "):
72
- line = line[len("data: ") :]
73
- # Last chunk, stream ends
74
- if line.strip() == "[DONE]":
75
- break
76
- # Parse the JSON response
77
- chunk = json.loads(line)
78
- # Extract and print the content
79
- content = chunk["choices"][0].get("delta", {}).get("content")
80
- print(content, end="")
52
+ print("\ntranscription result:", end=" ")
53
+ with open(audio_path, "rb") as f:
54
+ transcription = await client.audio.transcriptions.create(
55
+ file=f,
56
+ model="openai/whisper-large-v3",
57
+ language="en",
58
+ response_format="json",
59
+ temperature=0.0,
60
+ # Additional sampling params not provided by OpenAI API.
61
+ extra_body=dict(
62
+ seed=420,
63
+ top_p=0.6,
64
+ ),
65
+ stream=True,
66
+ )
67
+ async for chunk in transcription:
68
+ if chunk.choices:
69
+ content = chunk.choices[0].get("delta", {}).get("content")
70
+ print(content, end="", flush=True)
71
+
72
print() # Final newline after stream ends
73
74
@@ -95,7 +86,11 @@ def main():
86
87
sync_openai(mary_had_lamb, client)
88
# Run the asynchronous function
98
- asyncio.run(stream_openai_response(winning_call, openai_api_base, openai_api_key))
89
+ client = AsyncOpenAI(
90
+ api_key=openai_api_key,
91
+ base_url=openai_api_base,
92
+ )
93
+ asyncio.run(stream_openai_response(winning_call, client))
94
95
96
if __name__ == "__main__":
vllm/entrypoints/openai/protocol.py
+3
-4
@@ -1750,12 +1750,11 @@ class TranscriptionRequest(OpenAIBaseModel):
1750
timestamps incurs additional latency.
1751
"""
1752
1753
- # --8<-- [start:transcription-extra-params]
1753
stream: Optional[bool] = False
1755
- """Custom field not present in the original OpenAI definition. When set,
1756
- it will enable output to be streamed in a similar fashion as the Chat
1757
- Completion endpoint.
1754
+ """When set, it will enable output to be streamed in a similar fashion
1755
+ as the Chat Completion endpoint.
1756
"""
1757
+ # --8<-- [start:transcription-extra-params]
1758
# Flattened stream option to simplify form data.
1759
stream_include_usage: Optional[bool] = False
1760
stream_continuous_usage_stats: Optional[bool] = False