@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e5a621b724
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 12, 11:00 UTC

[CI] Add batched audios Whisper test (#29308)

Signed-off-by: NickLucche <nlucches@redhat.com>

Nicolò Lucchesi committed Nov 27, 2025 at 20:31 UTC e5a621b724e5570aaffc4bbf9c5f6ec9bca63333
2 files changed +238 -196
tests/entrypoints/openai/test_transcription_validation.py
+1 -196
@@ -2,20 +2,12 @@
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 # imports for structured outputs tests
5 -import io
5 import json
6
8 -import librosa
9 -import numpy as np
10 -import openai
7 import pytest
12 -import pytest_asyncio
13 -import soundfile as sf
8
9 from ...utils import RemoteOpenAIServer
10
17 -MODEL_NAME = "openai/whisper-large-v3-turbo"
18 -SERVER_ARGS = ["--enforce-eager"]
11 MISTRAL_FORMAT_ARGS = [
12 "--tokenizer_mode",
13 "mistral",
@@ -26,22 +18,8 @@ MISTRAL_FORMAT_ARGS = [
18 ]
19
20
29 -@pytest.fixture(scope="module")
30 -def server():
31 - with RemoteOpenAIServer(MODEL_NAME, SERVER_ARGS) as remote_server:
32 - yield remote_server
33 -
34 -
35 -@pytest_asyncio.fixture
36 -async def client(server):
37 - async with server.get_async_client() as async_client:
38 - yield async_client
39 -
40 -
21 @pytest.mark.asyncio
42 -@pytest.mark.parametrize(
43 - "model_name", ["openai/whisper-large-v3-turbo", "mistralai/Voxtral-Mini-3B-2507"]
44 -)
22 +@pytest.mark.parametrize("model_name", ["mistralai/Voxtral-Mini-3B-2507"])
23 async def test_basic_audio(mary_had_lamb, model_name):
24 server_args = ["--enforce-eager"]
25
@@ -120,176 +98,3 @@ async def test_basic_audio_gemma(foscolo):
98 )
99 out = json.loads(transcription)["text"]
100 assert "da cui vergine nacque Venere" in out
123 -
124 -
125 -@pytest.mark.asyncio
126 -async def test_non_asr_model(winning_call):
127 - # text to text model
128 - model_name = "JackFram/llama-68m"
129 - with RemoteOpenAIServer(model_name, SERVER_ARGS) as remote_server:
130 - client = remote_server.get_async_client()
131 - res = await client.audio.transcriptions.create(
132 - model=model_name, file=winning_call, language="en", temperature=0.0
133 - )
134 - err = res.error
135 - assert err["code"] == 400 and not res.text
136 - assert err["message"] == "The model does not support Transcriptions API"
137 -
138 -
139 -@pytest.mark.asyncio
140 -async def test_bad_requests(mary_had_lamb, client):
141 - # invalid language
142 - with pytest.raises(openai.BadRequestError):
143 - await client.audio.transcriptions.create(
144 - model=MODEL_NAME, file=mary_had_lamb, language="hh", temperature=0.0
145 - )
146 -
147 -
148 -@pytest.mark.asyncio
149 -async def test_long_audio_request(mary_had_lamb, client):
150 - mary_had_lamb.seek(0)
151 - audio, sr = librosa.load(mary_had_lamb)
152 - # Add small silence after each audio for repeatability in the split process
153 - audio = np.pad(audio, (0, 1600))
154 - repeated_audio = np.tile(audio, 10)
155 - # Repeated audio to buffer
156 - buffer = io.BytesIO()
157 - sf.write(buffer, repeated_audio, sr, format="WAV")
158 - buffer.seek(0)
159 - transcription = await client.audio.transcriptions.create(
160 - model=MODEL_NAME,
161 - file=buffer,
162 - language="en",
163 - response_format="text",
164 - temperature=0.0,
165 - )
166 - out = json.loads(transcription)
167 - out_text = out["text"]
168 - out_usage = out["usage"]
169 - counts = out_text.count("Mary had a little lamb")
170 - assert counts == 10, counts
171 - assert out_usage["seconds"] == 161, out_usage["seconds"]
172 -
173 -
174 -@pytest.mark.asyncio
175 -async def test_completion_endpoints(client):
176 - # text to text model
177 - res = await client.chat.completions.create(
178 - model=MODEL_NAME,
179 - messages=[{"role": "system", "content": "You are a helpful assistant."}],
180 - )
181 - err = res.error
182 - assert err["code"] == 400
183 - assert err["message"] == "The model does not support Chat Completions API"
184 -
185 - res = await client.completions.create(model=MODEL_NAME, prompt="Hello")
186 - err = res.error
187 - assert err["code"] == 400
188 - assert err["message"] == "The model does not support Completions API"
189 -
190 -
191 -@pytest.mark.asyncio
192 -async def test_streaming_response(winning_call, client):
193 - transcription = ""
194 - res_no_stream = await client.audio.transcriptions.create(
195 - model=MODEL_NAME,
196 - file=winning_call,
197 - response_format="json",
198 - language="en",
199 - temperature=0.0,
200 - )
201 - res = await client.audio.transcriptions.create(
202 - model=MODEL_NAME,
203 - file=winning_call,
204 - language="en",
205 - temperature=0.0,
206 - stream=True,
207 - timeout=30,
208 - )
209 - # Reconstruct from chunks and validate
210 - async for chunk in res:
211 - text = chunk.choices[0]["delta"]["content"]
212 - transcription += text
213 -
214 - assert transcription == res_no_stream.text
215 -
216 -
217 -@pytest.mark.asyncio
218 -async def test_stream_options(winning_call, client):
219 - res = await client.audio.transcriptions.create(
220 - model=MODEL_NAME,
221 - file=winning_call,
222 - language="en",
223 - temperature=0.0,
224 - stream=True,
225 - extra_body=dict(stream_include_usage=True, stream_continuous_usage_stats=True),
226 - timeout=30,
227 - )
228 - final = False
229 - continuous = True
230 - async for chunk in res:
231 - if not len(chunk.choices):
232 - # final usage sent
233 - final = True
234 - else:
235 - continuous = continuous and hasattr(chunk, "usage")
236 - assert final and continuous
237 -
238 -
239 -@pytest.mark.asyncio
240 -async def test_sampling_params(mary_had_lamb, client):
241 - """
242 - Compare sampling with params and greedy sampling to assert results
243 - are different when extreme sampling parameters values are picked.
244 - """
245 - transcription = await client.audio.transcriptions.create(
246 - model=MODEL_NAME,
247 - file=mary_had_lamb,
248 - language="en",
249 - temperature=0.8,
250 - extra_body=dict(
251 - seed=42,
252 - repetition_penalty=1.9,
253 - top_k=12,
254 - top_p=0.4,
255 - min_p=0.5,
256 - frequency_penalty=1.8,
257 - presence_penalty=2.0,
258 - ),
259 - )
260 -
261 - greedy_transcription = await client.audio.transcriptions.create(
262 - model=MODEL_NAME,
263 - file=mary_had_lamb,
264 - language="en",
265 - temperature=0.0,
266 - extra_body=dict(seed=42),
267 - )
268 -
269 - assert greedy_transcription.text != transcription.text
270 -
271 -
272 -@pytest.mark.asyncio
273 -async def test_audio_prompt(mary_had_lamb, client):
274 - prompt = "This is a speech, recorded in a phonograph."
275 - # Prompts should not omit the part of original prompt while transcribing.
276 - prefix = "The first words I spoke in the original phonograph"
277 - transcription = await client.audio.transcriptions.create(
278 - model=MODEL_NAME,
279 - file=mary_had_lamb,
280 - language="en",
281 - response_format="text",
282 - temperature=0.0,
283 - )
284 - out = json.loads(transcription)["text"]
285 - assert prefix in out
286 - transcription_wprompt = await client.audio.transcriptions.create(
287 - model=MODEL_NAME,
288 - file=mary_had_lamb,
289 - language="en",
290 - response_format="text",
291 - prompt=prompt,
292 - temperature=0.0,
293 - )
294 - out_prompt = json.loads(transcription_wprompt)["text"]
295 - assert prefix in out_prompt
tests/entrypoints/openai/test_transcription_validation_whisper.py new
+237
@@ -0,0 +1,237 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +# imports for structured outputs tests
5 +import asyncio
6 +import io
7 +import json
8 +
9 +import librosa
10 +import numpy as np
11 +import openai
12 +import pytest
13 +import pytest_asyncio
14 +import soundfile as sf
15 +
16 +from ...utils import RemoteOpenAIServer
17 +
18 +MODEL_NAME = "openai/whisper-large-v3-turbo"
19 +SERVER_ARGS = ["--enforce-eager"]
20 +
21 +
22 +@pytest.fixture(scope="module")
23 +def server():
24 + with RemoteOpenAIServer(MODEL_NAME, SERVER_ARGS) as remote_server:
25 + yield remote_server
26 +
27 +
28 +@pytest_asyncio.fixture
29 +async def whisper_client(server):
30 + async with server.get_async_client() as async_client:
31 + yield async_client
32 +
33 +
34 +@pytest.mark.asyncio
35 +async def test_basic_audio(mary_had_lamb):
36 + server_args = ["--enforce-eager"]
37 +
38 + # Based on https://github.com/openai/openai-cookbook/blob/main/examples/Whisper_prompting_guide.ipynb.
39 + with RemoteOpenAIServer(MODEL_NAME, server_args) as remote_server:
40 + client = remote_server.get_async_client()
41 + transcription = await client.audio.transcriptions.create(
42 + model=MODEL_NAME,
43 + file=mary_had_lamb,
44 + language="en",
45 + response_format="text",
46 + temperature=0.0,
47 + )
48 + out = json.loads(transcription)
49 + out_text = out["text"]
50 + out_usage = out["usage"]
51 + assert "Mary had a little lamb," in out_text
52 + assert out_usage["seconds"] == 16, out_usage["seconds"]
53 +
54 +
55 +@pytest.mark.asyncio
56 +async def test_basic_audio_batched(mary_had_lamb, winning_call, whisper_client):
57 + transcription = whisper_client.audio.transcriptions.create(
58 + model=MODEL_NAME,
59 + file=mary_had_lamb,
60 + language="en",
61 + response_format="text",
62 + temperature=0.0,
63 + )
64 + transcription2 = whisper_client.audio.transcriptions.create(
65 + model=MODEL_NAME,
66 + file=winning_call,
67 + language="en",
68 + response_format="text",
69 + temperature=0.0,
70 + )
71 + # Await both transcriptions by scheduling coroutines together
72 + transcription, transcription2 = await asyncio.gather(transcription, transcription2)
73 + out = json.loads(transcription)
74 + out_text = out["text"]
75 + assert "Mary had a little lamb," in out_text
76 + out2 = json.loads(transcription2)
77 + out_text2 = out2["text"]
78 + assert "Edgar Martinez" in out_text2
79 +
80 +
81 +@pytest.mark.asyncio
82 +async def test_bad_requests(mary_had_lamb, whisper_client):
83 + # invalid language
84 + with pytest.raises(openai.BadRequestError):
85 + await whisper_client.audio.transcriptions.create(
86 + model=MODEL_NAME, file=mary_had_lamb, language="hh", temperature=0.0
87 + )
88 +
89 +
90 +@pytest.mark.asyncio
91 +async def test_long_audio_request(mary_had_lamb, whisper_client):
92 + mary_had_lamb.seek(0)
93 + audio, sr = librosa.load(mary_had_lamb)
94 + # Add small silence after each audio for repeatability in the split process
95 + audio = np.pad(audio, (0, 1600))
96 + repeated_audio = np.tile(audio, 10)
97 + # Repeated audio to buffer
98 + buffer = io.BytesIO()
99 + sf.write(buffer, repeated_audio, sr, format="WAV")
100 + buffer.seek(0)
101 + transcription = await whisper_client.audio.transcriptions.create(
102 + model=MODEL_NAME,
103 + file=buffer,
104 + language="en",
105 + response_format="text",
106 + temperature=0.0,
107 + )
108 + out = json.loads(transcription)
109 + out_text = out["text"]
110 + out_usage = out["usage"]
111 + counts = out_text.count("Mary had a little lamb")
112 + assert counts == 10, counts
113 + assert out_usage["seconds"] == 161, out_usage["seconds"]
114 +
115 +
116 +@pytest.mark.asyncio
117 +async def test_completion_endpoints(whisper_client):
118 + # text to text model
119 + res = await whisper_client.chat.completions.create(
120 + model=MODEL_NAME,
121 + messages=[{"role": "system", "content": "You are a helpful assistant."}],
122 + )
123 + err = res.error
124 + assert err["code"] == 400
125 + assert err["message"] == "The model does not support Chat Completions API"
126 +
127 + res = await whisper_client.completions.create(model=MODEL_NAME, prompt="Hello")
128 + err = res.error
129 + assert err["code"] == 400
130 + assert err["message"] == "The model does not support Completions API"
131 +
132 +
133 +@pytest.mark.asyncio
134 +async def test_streaming_response(winning_call, whisper_client):
135 + transcription = ""
136 + res_no_stream = await whisper_client.audio.transcriptions.create(
137 + model=MODEL_NAME,
138 + file=winning_call,
139 + response_format="json",
140 + language="en",
141 + temperature=0.0,
142 + )
143 + res = await whisper_client.audio.transcriptions.create(
144 + model=MODEL_NAME,
145 + file=winning_call,
146 + language="en",
147 + temperature=0.0,
148 + stream=True,
149 + timeout=30,
150 + )
151 + # Reconstruct from chunks and validate
152 + async for chunk in res:
153 + text = chunk.choices[0]["delta"]["content"]
154 + transcription += text
155 +
156 + assert transcription == res_no_stream.text
157 +
158 +
159 +@pytest.mark.asyncio
160 +async def test_stream_options(winning_call, whisper_client):
161 + res = await whisper_client.audio.transcriptions.create(
162 + model=MODEL_NAME,
163 + file=winning_call,
164 + language="en",
165 + temperature=0.0,
166 + stream=True,
167 + extra_body=dict(stream_include_usage=True, stream_continuous_usage_stats=True),
168 + timeout=30,
169 + )
170 + final = False
171 + continuous = True
172 + async for chunk in res:
173 + if not len(chunk.choices):
174 + # final usage sent
175 + final = True
176 + else:
177 + continuous = continuous and hasattr(chunk, "usage")
178 + assert final and continuous
179 +
180 +
181 +@pytest.mark.asyncio
182 +async def test_sampling_params(mary_had_lamb, whisper_client):
183 + """
184 + Compare sampling with params and greedy sampling to assert results
185 + are different when extreme sampling parameters values are picked.
186 + """
187 + transcription = await whisper_client.audio.transcriptions.create(
188 + model=MODEL_NAME,
189 + file=mary_had_lamb,
190 + language="en",
191 + temperature=0.8,
192 + extra_body=dict(
193 + seed=42,
194 + repetition_penalty=1.9,
195 + top_k=12,
196 + top_p=0.4,
197 + min_p=0.5,
198 + frequency_penalty=1.8,
199 + presence_penalty=2.0,
200 + ),
201 + )
202 +
203 + greedy_transcription = await whisper_client.audio.transcriptions.create(
204 + model=MODEL_NAME,
205 + file=mary_had_lamb,
206 + language="en",
207 + temperature=0.0,
208 + extra_body=dict(seed=42),
209 + )
210 +
211 + assert greedy_transcription.text != transcription.text
212 +
213 +
214 +@pytest.mark.asyncio
215 +async def test_audio_prompt(mary_had_lamb, whisper_client):
216 + prompt = "This is a speech, recorded in a phonograph."
217 + # Prompts should not omit the part of original prompt while transcribing.
218 + prefix = "The first words I spoke in the original phonograph"
219 + transcription = await whisper_client.audio.transcriptions.create(
220 + model=MODEL_NAME,
221 + file=mary_had_lamb,
222 + language="en",
223 + response_format="text",
224 + temperature=0.0,
225 + )
226 + out = json.loads(transcription)["text"]
227 + assert prefix in out
228 + transcription_wprompt = await whisper_client.audio.transcriptions.create(
229 + model=MODEL_NAME,
230 + file=mary_had_lamb,
231 + language="en",
232 + response_format="text",
233 + prompt=prompt,
234 + temperature=0.0,
235 + )
236 + out_prompt = json.loads(transcription_wprompt)["text"]
237 + assert prefix in out_prompt