| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | # imports for structured outputs tests |
| 5 | import asyncio |
| 6 | import io |
| 7 | import json |
| 8 | |
| 9 | import numpy as np |
| 10 | import openai |
| 11 | import pytest |
| 12 | import pytest_asyncio |
| 13 | import soundfile as sf |
| 14 | |
| 15 | from tests.utils import RemoteOpenAIServer |
| 16 | from vllm.multimodal.media.audio import load_audio |
| 17 | from vllm.platforms import current_platform |
| 18 | |
| 19 | MODEL_NAME = "openai/whisper-large-v3-turbo" |
| 20 | |
| 21 | # Disable prefix caching on ROCm to reduce non-determinism in |
| 22 | # streaming-vs-non-streaming comparisons. |
| 23 | _ROCM_ARGS = ["--no-enable-prefix-caching"] if current_platform.is_rocm() else [] |
| 24 | |
| 25 | |
| 26 | def _get_attention_backend_params() -> list[str | None]: |
| 27 | """Return attention backends to parametrize the server fixture with. |
| 28 | |
| 29 | On ROCm, we test multiple backends explicitly: |
| 30 | - None: default auto-selection (ROCM_ATTN for decoder self-attention, |
| 31 | falls back to ROCM_AITER_UNIFIED_ATTN or TRITON_ATTN for |
| 32 | cross-attention since ROCM_ATTN doesn't support ENCODER_DECODER) |
| 33 | - TRITON_ATTN: always available on ROCm |
| 34 | - ROCM_AITER_UNIFIED_ATTN: only on gfx942/gfx950/gfx1250 |
| 35 | |
| 36 | On non-ROCm platforms, we just run with the default backend. |
| 37 | """ |
| 38 | try: |
| 39 | from vllm.platforms import current_platform |
| 40 | |
| 41 | if current_platform.is_rocm(): |
| 42 | backends: list[str | None] = [None, "TRITON_ATTN"] |
| 43 | from vllm.platforms.rocm import get_cdna_version |
| 44 | |
| 45 | if get_cdna_version() > 2: |
| 46 | backends.append("ROCM_AITER_UNIFIED_ATTN") |
| 47 | return backends |
| 48 | except Exception: |
| 49 | pass |
| 50 | return [None] |
| 51 | |
| 52 | |
| 53 | # Aiter backends need VLLM_ROCM_USE_AITER=1 (and MHA=1 for ROCM_AITER_FA) |
| 54 | # to be enabled in the server subprocess. |
| 55 | _AITER_ENV = { |
| 56 | "VLLM_ROCM_USE_AITER": "1", |
| 57 | "VLLM_ROCM_USE_AITER_MHA": "1", |
| 58 | } |
| 59 | |
| 60 | _ATTN_BACKENDS = _get_attention_backend_params() |
| 61 | _ATTN_IDS = [b or "default" for b in _ATTN_BACKENDS] |
| 62 | |
| 63 | |
| 64 | @pytest.fixture(scope="module", params=_ATTN_BACKENDS, ids=_ATTN_IDS) |
| 65 | def server(request): |
| 66 | args = [*_ROCM_ARGS] |
| 67 | env_dict = None |
| 68 | if request.param is not None: |
| 69 | args += ["--attention-backend", request.param] |
| 70 | if "AITER" in request.param: |
| 71 | # TODO: re-enable once AITER reenables fp16 unified attention. |
| 72 | pytest.skip("ROCM_AITER_UNIFIED_ATTN does not support fp16") |
| 73 | with RemoteOpenAIServer(MODEL_NAME, args, env_dict=env_dict) as remote_server: |
| 74 | yield remote_server |
| 75 | |
| 76 | |
| 77 | @pytest_asyncio.fixture |
| 78 | async def whisper_client(server): |
| 79 | async with server.get_async_client() as async_client: |
| 80 | yield async_client |
| 81 | |
| 82 | |
| 83 | @pytest.mark.asyncio |
| 84 | async def test_basic_audio(whisper_client, mary_had_lamb): |
| 85 | # Based on https://github.com/openai/openai-cookbook/blob/main/examples/Whisper_prompting_guide.ipynb. |
| 86 | transcription = await whisper_client.audio.transcriptions.create( |
| 87 | model=MODEL_NAME, |
| 88 | file=mary_had_lamb, |
| 89 | language="en", |
| 90 | response_format="text", |
| 91 | temperature=0.0, |
| 92 | ) |
| 93 | out = json.loads(transcription) |
| 94 | out_text = out["text"] |
| 95 | out_usage = out["usage"] |
| 96 | assert "Mary had a little lamb," in out_text |
| 97 | assert out_usage["seconds"] == 16, out_usage["seconds"] |
| 98 | |
| 99 | |
| 100 | @pytest.mark.asyncio |
| 101 | async def test_basic_audio_batched(mary_had_lamb, winning_call, whisper_client): |
| 102 | transcription = whisper_client.audio.transcriptions.create( |
| 103 | model=MODEL_NAME, |
| 104 | file=mary_had_lamb, |
| 105 | language="en", |
| 106 | response_format="text", |
| 107 | temperature=0.0, |
| 108 | ) |
| 109 | transcription2 = whisper_client.audio.transcriptions.create( |
| 110 | model=MODEL_NAME, |
| 111 | file=winning_call, |
| 112 | language="en", |
| 113 | response_format="text", |
| 114 | temperature=0.0, |
| 115 | ) |
| 116 | # Await both transcriptions by scheduling coroutines together |
| 117 | transcription, transcription2 = await asyncio.gather(transcription, transcription2) |
| 118 | out = json.loads(transcription) |
| 119 | out_text = out["text"] |
| 120 | assert "Mary had a little lamb," in out_text |
| 121 | out2 = json.loads(transcription2) |
| 122 | out_text2 = out2["text"] |
| 123 | assert "Edgar Martinez" in out_text2 |
| 124 | |
| 125 | |
| 126 | @pytest.mark.asyncio |
| 127 | async def test_bad_requests(mary_had_lamb, whisper_client): |
| 128 | # invalid language |
| 129 | with pytest.raises(openai.BadRequestError): |
| 130 | await whisper_client.audio.transcriptions.create( |
| 131 | model=MODEL_NAME, file=mary_had_lamb, language="hh", temperature=0.0 |
| 132 | ) |
| 133 | |
| 134 | |
| 135 | @pytest.mark.asyncio |
| 136 | async def test_long_audio_request(mary_had_lamb, whisper_client): |
| 137 | mary_had_lamb.seek(0) |
| 138 | audio, sr = load_audio(mary_had_lamb) |
| 139 | # Add small silence after each audio for repeatability in the split process |
| 140 | audio = np.pad(audio, (0, 1600)) |
| 141 | repeated_audio = np.tile(audio, 10) |
| 142 | # Repeated audio to buffer |
| 143 | buffer = io.BytesIO() |
| 144 | sf.write(buffer, repeated_audio, sr, format="WAV") |
| 145 | buffer.seek(0) |
| 146 | transcription = await whisper_client.audio.transcriptions.create( |
| 147 | model=MODEL_NAME, |
| 148 | file=buffer, |
| 149 | language="en", |
| 150 | response_format="text", |
| 151 | temperature=0.0, |
| 152 | ) |
| 153 | out = json.loads(transcription) |
| 154 | out_text = out["text"] |
| 155 | out_usage = out["usage"] |
| 156 | counts = out_text.count("Mary had a little lamb") |
| 157 | assert counts == 10, counts |
| 158 | assert out_usage["seconds"] == 161, out_usage["seconds"] |
| 159 | |
| 160 | |
| 161 | @pytest.mark.asyncio |
| 162 | async def test_invalid_audio_file(whisper_client): |
| 163 | """Corrupted audio should surface as HTTP 400.""" |
| 164 | invalid_audio = io.BytesIO(b"not a valid audio file") |
| 165 | invalid_audio.name = "invalid.wav" |
| 166 | |
| 167 | with pytest.raises(openai.BadRequestError) as exc_info: |
| 168 | await whisper_client.audio.transcriptions.create( |
| 169 | model=MODEL_NAME, |
| 170 | file=invalid_audio, |
| 171 | language="en", |
| 172 | ) |
| 173 | |
| 174 | assert exc_info.value.status_code == 400 |
| 175 | assert "Invalid or corrupted audio data" in exc_info.value.message |
| 176 | |
| 177 | |
| 178 | @pytest.mark.asyncio |
| 179 | async def test_completion_endpoints(whisper_client): |
| 180 | # text to text model |
| 181 | with pytest.raises(openai.NotFoundError): |
| 182 | await whisper_client.chat.completions.create( |
| 183 | model=MODEL_NAME, |
| 184 | messages=[{"role": "system", "content": "You are a helpful assistant."}], |
| 185 | ) |
| 186 | |
| 187 | with pytest.raises(openai.NotFoundError): |
| 188 | await whisper_client.completions.create(model=MODEL_NAME, prompt="Hello") |
| 189 | |
| 190 | |
| 191 | @pytest.mark.asyncio |
| 192 | async def test_streaming_response(winning_call, whisper_client): |
| 193 | transcription = "" |
| 194 | res_no_stream = await whisper_client.audio.transcriptions.create( |
| 195 | model=MODEL_NAME, |
| 196 | file=winning_call, |
| 197 | response_format="json", |
| 198 | language="en", |
| 199 | temperature=0.0, |
| 200 | ) |
| 201 | res = await whisper_client.audio.transcriptions.create( |
| 202 | model=MODEL_NAME, |
| 203 | file=winning_call, |
| 204 | language="en", |
| 205 | temperature=0.0, |
| 206 | stream=True, |
| 207 | timeout=30, |
| 208 | ) |
| 209 | # Reconstruct from chunks and validate |
| 210 | async for chunk in res: |
| 211 | text = chunk.choices[0]["delta"]["content"] |
| 212 | transcription += text |
| 213 | |
| 214 | assert transcription == res_no_stream.text |
| 215 | |
| 216 | |
| 217 | @pytest.mark.asyncio |
| 218 | async def test_stream_options(winning_call, whisper_client): |
| 219 | res = await whisper_client.audio.transcriptions.create( |
| 220 | model=MODEL_NAME, |
| 221 | file=winning_call, |
| 222 | language="en", |
| 223 | temperature=0.0, |
| 224 | stream=True, |
| 225 | extra_body=dict(stream_include_usage=True, stream_continuous_usage_stats=True), |
| 226 | timeout=30, |
| 227 | ) |
| 228 | final = False |
| 229 | continuous = True |
| 230 | async for chunk in res: |
| 231 | if not len(chunk.choices): |
| 232 | # final usage sent |
| 233 | final = True |
| 234 | else: |
| 235 | continuous = continuous and hasattr(chunk, "usage") |
| 236 | assert final and continuous |
| 237 | |
| 238 | |
| 239 | @pytest.mark.asyncio |
| 240 | async def test_sampling_params(mary_had_lamb, whisper_client): |
| 241 | """Compare sampling with params and greedy sampling to assert results |
| 242 | are different when extreme sampling parameters values are picked. |
| 243 | """ |
| 244 | transcription = await whisper_client.audio.transcriptions.create( |
| 245 | model=MODEL_NAME, |
| 246 | file=mary_had_lamb, |
| 247 | language="en", |
| 248 | temperature=0.8, |
| 249 | extra_body=dict( |
| 250 | seed=42, |
| 251 | repetition_penalty=1.9, |
| 252 | top_k=12, |
| 253 | top_p=0.4, |
| 254 | min_p=0.5, |
| 255 | frequency_penalty=1.8, |
| 256 | presence_penalty=2.0, |
| 257 | ), |
| 258 | ) |
| 259 | |
| 260 | greedy_transcription = await whisper_client.audio.transcriptions.create( |
| 261 | model=MODEL_NAME, |
| 262 | file=mary_had_lamb, |
| 263 | language="en", |
| 264 | temperature=0.0, |
| 265 | extra_body=dict(seed=42), |
| 266 | ) |
| 267 | |
| 268 | assert greedy_transcription.text != transcription.text |
| 269 | |
| 270 | |
| 271 | @pytest.mark.asyncio |
| 272 | async def test_audio_prompt(mary_had_lamb, whisper_client): |
| 273 | prompt = "This is a speech, recorded in a phonograph." |
| 274 | # Prompts should not omit the part of original prompt while transcribing. |
| 275 | prefix = "The first words I spoke in the original phonograph" |
| 276 | transcription = await whisper_client.audio.transcriptions.create( |
| 277 | model=MODEL_NAME, |
| 278 | file=mary_had_lamb, |
| 279 | language="en", |
| 280 | response_format="text", |
| 281 | temperature=0.0, |
| 282 | ) |
| 283 | out = json.loads(transcription)["text"] |
| 284 | assert prefix in out |
| 285 | transcription_wprompt = await whisper_client.audio.transcriptions.create( |
| 286 | model=MODEL_NAME, |
| 287 | file=mary_had_lamb, |
| 288 | language="en", |
| 289 | response_format="text", |
| 290 | prompt=prompt, |
| 291 | temperature=0.0, |
| 292 | ) |
| 293 | out_prompt = json.loads(transcription_wprompt)["text"] |
| 294 | assert prefix in out_prompt |
| 295 | |
| 296 | |
| 297 | @pytest.mark.asyncio |
| 298 | async def test_audio_with_timestamp(mary_had_lamb, whisper_client): |
| 299 | transcription = await whisper_client.audio.transcriptions.create( |
| 300 | model=MODEL_NAME, |
| 301 | file=mary_had_lamb, |
| 302 | language="en", |
| 303 | response_format="verbose_json", |
| 304 | temperature=0.0, |
| 305 | ) |
| 306 | assert transcription.segments is not None |
| 307 | assert len(transcription.segments) > 0 |
| 308 | assert transcription.segments[0].avg_logprob is not None |
| 309 | assert transcription.segments[0].compression_ratio is not None |
| 310 | |
| 311 | |
| 312 | @pytest.mark.asyncio |
| 313 | async def test_audio_with_max_tokens(whisper_client, mary_had_lamb): |
| 314 | transcription = await whisper_client.audio.transcriptions.create( |
| 315 | model=MODEL_NAME, |
| 316 | file=mary_had_lamb, |
| 317 | language="en", |
| 318 | response_format="text", |
| 319 | temperature=0.0, |
| 320 | extra_body={"max_completion_tokens": 1}, |
| 321 | ) |
| 322 | out = json.loads(transcription) |
| 323 | out_text = out["text"] |
| 324 | from transformers import AutoTokenizer |
| 325 | |
| 326 | tok = AutoTokenizer.from_pretrained(MODEL_NAME) |
| 327 | out_tokens = tok(out_text, add_special_tokens=False)["input_ids"] |
| 328 | assert len(out_tokens) == 1 |
| 329 | # max_completion_tokens > max_model_len |
| 330 | transcription = await whisper_client.audio.transcriptions.create( |
| 331 | model=MODEL_NAME, |
| 332 | file=mary_had_lamb, |
| 333 | language="en", |
| 334 | response_format="text", |
| 335 | temperature=0.0, |
| 336 | extra_body={"max_completion_tokens": int(1e6)}, |
| 337 | ) |
| 338 | out = json.loads(transcription) |
| 339 | out_text = out["text"] |
| 340 | out_tokens = tok(out_text, add_special_tokens=False)["input_ids"] |
| 341 | assert len(out_tokens) < 450 # ~Whisper max output len |
| 342 | |
| 343 | |
| 344 | @pytest.mark.asyncio |
| 345 | @pytest.mark.parametrize( |
| 346 | ("fixture_name", "expected_lang", "expected_text"), |
| 347 | [ |
| 348 | ("mary_had_lamb", "en", ["Mary had a little lamb"]), |
| 349 | ("foscolo", "it", ["zacinto", "sacre"]), |
| 350 | ], |
| 351 | ids=["english", "italian"], |
| 352 | ) |
| 353 | async def test_language_auto_detect( |
| 354 | whisper_client, fixture_name, expected_lang, expected_text, request |
| 355 | ): |
| 356 | """Auto-detect language when no language param is provided.""" |
| 357 | audio_file = request.getfixturevalue(fixture_name) |
| 358 | transcription = await whisper_client.audio.transcriptions.create( |
| 359 | model=MODEL_NAME, |
| 360 | file=audio_file, |
| 361 | response_format="verbose_json", |
| 362 | temperature=0.0, |
| 363 | ) |
| 364 | assert transcription.language == expected_lang |
| 365 | text_lower = transcription.text.lower() |
| 366 | assert any(word.lower() in text_lower for word in expected_text), ( |
| 367 | f"Expected {expected_lang} text but got: {transcription.text}" |
| 368 | ) |
| 369 | |
| 370 | |
| 371 | @pytest.mark.asyncio |
| 372 | async def test_whisper_beam_search_single_beam(mary_had_lamb, whisper_client): |
| 373 | """Test beam search with encoder-decoder model (Whisper) on transcriptions with |
| 374 | one beam aligns with greedy decoding. |
| 375 | """ |
| 376 | beam_transcription = await whisper_client.audio.transcriptions.create( |
| 377 | model=MODEL_NAME, |
| 378 | file=mary_had_lamb, |
| 379 | language="en", |
| 380 | response_format="text", |
| 381 | temperature=0.0, |
| 382 | extra_body=dict( |
| 383 | use_beam_search=True, |
| 384 | n=1, |
| 385 | ), |
| 386 | ) |
| 387 | |
| 388 | greedy_transcription = await whisper_client.audio.transcriptions.create( |
| 389 | model=MODEL_NAME, |
| 390 | file=mary_had_lamb, |
| 391 | response_format="text", |
| 392 | temperature=0.0, |
| 393 | ) |
| 394 | |
| 395 | greedy_res = json.loads(greedy_transcription)["text"] |
| 396 | beam_res = json.loads(beam_transcription)["text"] |
| 397 | assert greedy_res == beam_res |
| 398 | |
| 399 | |
| 400 | @pytest.mark.asyncio |
| 401 | async def test_whisper_beam_search_multibeam(mary_had_lamb, whisper_client): |
| 402 | """Test n>1 for beam search returns one transcription (best beam).""" |
| 403 | transcription = await whisper_client.audio.transcriptions.create( |
| 404 | model=MODEL_NAME, |
| 405 | file=mary_had_lamb, |
| 406 | language="en", |
| 407 | response_format="text", |
| 408 | temperature=0.0, |
| 409 | extra_body=dict( |
| 410 | use_beam_search=True, |
| 411 | n=2, |
| 412 | ), |
| 413 | ) |
| 414 | |
| 415 | result = json.loads(transcription) |
| 416 | |
| 417 | text = result["text"] |
| 418 | |
| 419 | assert text is not None |
| 420 | assert len(text) > 0 |
| 421 | assert "mary had a little lamb" in text.lower() |
| 422 | |
| 423 | |
| 424 | @pytest.mark.asyncio |
| 425 | async def test_stream_with_beams_raises(winning_call, whisper_client): |
| 426 | """Test that stream=True + beam search raises bad request for now.""" |
| 427 | with pytest.raises(openai.BadRequestError): |
| 428 | await whisper_client.audio.transcriptions.create( |
| 429 | model=MODEL_NAME, |
| 430 | file=winning_call, |
| 431 | language="en", |
| 432 | stream=True, |
| 433 | extra_body=dict( |
| 434 | use_beam_search=True, |
| 435 | n=2, |
| 436 | ), |
| 437 | ) |