A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 20:15 UTC
[Audio] Improve Audio Inference Scripts (offline/online) (#29279)
Signed-off-by: Ekagra Ranjan <3116519+ekagra-ranjan@users.noreply.github.com>
Ekagra Ranjan committed
Dec 31, 2025 at 18:34 UTC
adcf682fc7d1835d037da331922751e880c8bc25
2 files changed
+113
-32
examples/offline_inference/audio_language.py
+31
-18
@@ -495,27 +495,40 @@ def main(args):
495
temperature=0.2, max_tokens=64, stop_token_ids=req_data.stop_token_ids
496
)
497
498
- mm_data = req_data.multi_modal_data
499
- if not mm_data:
500
- mm_data = {}
501
- if audio_count > 0:
502
- mm_data = {
503
- "audio": [
504
- asset.audio_and_sample_rate for asset in audio_assets[:audio_count]
505
- ]
506
- }
507
-
498
+ def get_input(start, end):
499
+ mm_data = req_data.multi_modal_data
500
+ if not mm_data:
501
+ mm_data = {}
502
+ if end - start > 0:
503
+ mm_data = {
504
+ "audio": [
505
+ asset.audio_and_sample_rate for asset in audio_assets[start:end]
506
+ ]
507
+ }
508
+
509
+ inputs = {"multi_modal_data": mm_data}
510
+
511
+ if req_data.prompt:
512
+ inputs["prompt"] = req_data.prompt
513
+ else:
514
+ inputs["prompt_token_ids"] = req_data.prompt_token_ids
515
+
516
+ return inputs
517
+
518
+ # Batch inference
519
assert args.num_prompts > 0
509
- inputs = {"multi_modal_data": mm_data}
510
-
511
- if req_data.prompt:
512
- inputs["prompt"] = req_data.prompt
520
+ if audio_count != 1:
521
+ inputs = get_input(0, audio_count)
522
+ inputs = [inputs] * args.num_prompts
523
else:
514
- inputs["prompt_token_ids"] = req_data.prompt_token_ids
524
+ # For single audio input, we need to vary the audio input
525
+ # to avoid deduplication in vLLM engine.
526
+ inputs = []
527
+ for i in range(args.num_prompts):
528
+ start = i % len(audio_assets)
529
+ inp = get_input(start, start + 1)
530
+ inputs.append(inp)
531
516
- if args.num_prompts > 1:
517
- # Batch inference
518
- inputs = [inputs] * args.num_prompts
532
# Add LoRA request if applicable
533
lora_request = (
534
req_data.lora_requests * args.num_prompts if req_data.lora_requests else None
examples/online_serving/openai_transcription_client.py
+82
-14
@@ -18,6 +18,7 @@ The script performs:
18
2. Streaming transcription using raw HTTP request to the vLLM server.
19
"""
20
21
+import argparse
22
import asyncio
23
24
from openai import AsyncOpenAI, OpenAI
@@ -25,14 +26,14 @@ from openai import AsyncOpenAI, OpenAI
26
from vllm.assets.audio import AudioAsset
27
28
28
-def sync_openai(audio_path: str, client: OpenAI):
29
+def sync_openai(audio_path: str, client: OpenAI, model: str):
30
"""
31
Perform synchronous transcription using OpenAI-compatible API.
32
"""
33
with open(audio_path, "rb") as f:
34
transcription = client.audio.transcriptions.create(
35
file=f,
35
- model="openai/whisper-large-v3",
36
+ model=model,
37
language="en",
38
response_format="json",
39
temperature=0.0,
@@ -42,18 +43,18 @@ def sync_openai(audio_path: str, client: OpenAI):
43
repetition_penalty=1.3,
44
),
45
)
45
- print("transcription result:", transcription.text)
46
+ print("transcription result [sync]:", transcription.text)
47
48
48
-async def stream_openai_response(audio_path: str, client: AsyncOpenAI):
49
+async def stream_openai_response(audio_path: str, client: AsyncOpenAI, model: str):
50
"""
51
Perform asynchronous transcription using OpenAI-compatible API.
52
"""
52
- print("\ntranscription result:", end=" ")
53
+ print("\ntranscription result [stream]:", end=" ")
54
with open(audio_path, "rb") as f:
55
transcription = await client.audio.transcriptions.create(
56
file=f,
56
- model="openai/whisper-large-v3",
57
+ model=model,
58
language="en",
59
response_format="json",
60
temperature=0.0,
@@ -72,7 +73,47 @@ async def stream_openai_response(audio_path: str, client: AsyncOpenAI):
73
print() # Final newline after stream ends
74
75
75
-def main():
76
+def stream_api_response(audio_path: str, model: str, openai_api_base: str):
77
+ """
78
+ Perform streaming transcription using raw HTTP requests to the vLLM API server.
79
+ """
80
+ import json
81
+ import os
82
+
83
+ import requests
84
+
85
+ api_url = f"{openai_api_base}/audio/transcriptions"
86
+ headers = {"User-Agent": "Transcription-Client"}
87
+ with open(audio_path, "rb") as f:
88
+ files = {"file": (os.path.basename(audio_path), f)}
89
+ data = {
90
+ "stream": "true",
91
+ "model": model,
92
+ "language": "en",
93
+ "response_format": "json",
94
+ }
95
+
96
+ print("\ntranscription result [stream]:", end=" ")
97
+ response = requests.post(
98
+ api_url, headers=headers, files=files, data=data, stream=True
99
+ )
100
+ for chunk in response.iter_lines(
101
+ chunk_size=8192, decode_unicode=False, delimiter=b"\n"
102
+ ):
103
+ if chunk:
104
+ data = chunk[len("data: ") :]
105
+ data = json.loads(data.decode("utf-8"))
106
+ data = data["choices"][0]
107
+ delta = data["delta"]["content"]
108
+ print(delta, end="", flush=True)
109
+
110
+ finish_reason = data.get("finish_reason")
111
+ if finish_reason is not None:
112
+ print(f"\n[Stream finished reason: {finish_reason}]")
113
+ break
114
+
115
+
116
+def main(args):
117
mary_had_lamb = str(AudioAsset("mary_had_lamb").get_local_path())
118
winning_call = str(AudioAsset("winning_call").get_local_path())
119
@@ -84,14 +125,41 @@ def main():
125
base_url=openai_api_base,
126
)
127
87
- sync_openai(mary_had_lamb, client)
128
+ model = client.models.list().data[0].id
129
+ print(f"Using model: {model}")
130
+
131
+ # Run the synchronous function
132
+ sync_openai(args.audio_path if args.audio_path else mary_had_lamb, client, model)
133
+
134
# Run the asynchronous function
89
- client = AsyncOpenAI(
90
- api_key=openai_api_key,
91
- base_url=openai_api_base,
92
- )
93
- asyncio.run(stream_openai_response(winning_call, client))
135
+ if "openai" in model:
136
+ client = AsyncOpenAI(
137
+ api_key=openai_api_key,
138
+ base_url=openai_api_base,
139
+ )
140
+ asyncio.run(
141
+ stream_openai_response(
142
+ args.audio_path if args.audio_path else winning_call, client, model
143
+ )
144
+ )
145
+ else:
146
+ stream_api_response(
147
+ args.audio_path if args.audio_path else winning_call,
148
+ model,
149
+ openai_api_base,
150
+ )
151
152
153
if __name__ == "__main__":
97
- main()
154
+ # setup argparser
155
+ parser = argparse.ArgumentParser(
156
+ description="OpenAI Transcription Client using vLLM API Server"
157
+ )
158
+ parser.add_argument(
159
+ "--audio_path",
160
+ type=str,
161
+ default=None,
162
+ help="The path to the audio file to transcribe.",
163
+ )
164
+ args = parser.parse_args()
165
+ main(args)