| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | # imports for structured outputs tests |
| 5 | import json |
| 6 | |
| 7 | import pytest |
| 8 | |
| 9 | from tests.utils import ROCM_EXTRA_ARGS, RemoteOpenAIServer |
| 10 | |
| 11 | MISTRAL_FORMAT_ARGS = [ |
| 12 | "--tokenizer_mode", |
| 13 | "mistral", |
| 14 | "--config_format", |
| 15 | "mistral", |
| 16 | "--load_format", |
| 17 | "mistral", |
| 18 | ] |
| 19 | |
| 20 | |
| 21 | async def transcribe_and_check( |
| 22 | client, |
| 23 | model_name: str, |
| 24 | file, |
| 25 | *, |
| 26 | language: str, |
| 27 | expected_text: str, |
| 28 | expected_seconds: int | None = None, |
| 29 | case_sensitive: bool = False, |
| 30 | ): |
| 31 | """Run a transcription request and assert the output contains |
| 32 | *expected_text* and optionally that usage reports *expected_seconds*. |
| 33 | |
| 34 | Provides detailed failure messages with the actual transcription output. |
| 35 | """ |
| 36 | transcription = await client.audio.transcriptions.create( |
| 37 | model=model_name, |
| 38 | file=file, |
| 39 | language=language, |
| 40 | response_format="text", |
| 41 | temperature=0.0, |
| 42 | ) |
| 43 | out = json.loads(transcription) |
| 44 | out_text = out["text"] |
| 45 | out_usage = out["usage"] |
| 46 | |
| 47 | if case_sensitive: |
| 48 | assert expected_text in out_text, ( |
| 49 | f"Expected {expected_text!r} in transcription output, got: {out_text!r}" |
| 50 | ) |
| 51 | else: |
| 52 | assert expected_text.lower() in out_text.lower(), ( |
| 53 | f"Expected {expected_text!r} (case-insensitive) in transcription " |
| 54 | f"output, got: {out_text!r}" |
| 55 | ) |
| 56 | |
| 57 | if expected_seconds is not None: |
| 58 | assert out_usage["seconds"] == expected_seconds, ( |
| 59 | f"Expected {expected_seconds}s of audio, " |
| 60 | f"got {out_usage['seconds']}s. Full usage: {out_usage!r}" |
| 61 | ) |
| 62 | |
| 63 | |
| 64 | @pytest.mark.asyncio |
| 65 | @pytest.mark.parametrize( |
| 66 | "model_name", ["mistralai/Voxtral-Mini-3B-2507", "Qwen/Qwen3-ASR-0.6B"] |
| 67 | ) |
| 68 | async def test_basic_audio(mary_had_lamb, model_name): |
| 69 | server_args = ["--enforce-eager", *ROCM_EXTRA_ARGS] |
| 70 | |
| 71 | if model_name.startswith("mistralai"): |
| 72 | server_args += MISTRAL_FORMAT_ARGS |
| 73 | |
| 74 | # Based on https://github.com/openai/openai-cookbook/blob/main/examples/Whisper_prompting_guide.ipynb. |
| 75 | with RemoteOpenAIServer(model_name, server_args) as remote_server: |
| 76 | client = remote_server.get_async_client() |
| 77 | await transcribe_and_check( |
| 78 | client, |
| 79 | model_name, |
| 80 | mary_had_lamb, |
| 81 | language="en", |
| 82 | expected_text="Mary had a little lamb", |
| 83 | expected_seconds=16, |
| 84 | ) |
| 85 | |
| 86 | |
| 87 | @pytest.mark.asyncio |
| 88 | async def test_basic_audio_with_lora(mary_had_lamb): |
| 89 | """Ensure STT (transcribe) requests can pass LoRA through to generate.""" |
| 90 | # ROCm SPECIFIC CONFIGURATION: |
| 91 | # To ensure the test passes on ROCm, we modify the max model length to 512. |
| 92 | # We DO NOT apply this to other platforms to maintain strict upstream parity. |
| 93 | from vllm.platforms import current_platform |
| 94 | |
| 95 | model_name = "ibm-granite/granite-speech-3.3-2b" |
| 96 | lora_model_name = "speech" |
| 97 | server_args = [ |
| 98 | "--enforce-eager", |
| 99 | "--enable-lora", |
| 100 | "--max-lora-rank", |
| 101 | "64", |
| 102 | "--lora-modules", |
| 103 | f"{lora_model_name}={model_name}", |
| 104 | "--max-model-len", |
| 105 | "512" if current_platform.is_rocm() else "2048", |
| 106 | "--max-num-seqs", |
| 107 | "1", |
| 108 | ] |
| 109 | |
| 110 | # Based on https://github.com/openai/openai-cookbook/blob/main/examples/Whisper_prompting_guide.ipynb. |
| 111 | with RemoteOpenAIServer(model_name, server_args) as remote_server: |
| 112 | client = remote_server.get_async_client() |
| 113 | await transcribe_and_check( |
| 114 | client, |
| 115 | lora_model_name, |
| 116 | mary_had_lamb, |
| 117 | language="en", |
| 118 | expected_text="mary had a little lamb", |
| 119 | expected_seconds=16, |
| 120 | ) |
| 121 | |
| 122 | |
| 123 | @pytest.mark.asyncio |
| 124 | @pytest.mark.parametrize( |
| 125 | "model_name", ["google/gemma-3n-E2B-it", "Qwen/Qwen3-ASR-0.6B"] |
| 126 | ) |
| 127 | async def test_basic_audio_foscolo(foscolo, model_name): |
| 128 | # Gemma accuracy on some of the audio samples we use is particularly bad, |
| 129 | # hence we use a different one here. WER is evaluated separately. |
| 130 | server_args = ["--enforce-eager", *ROCM_EXTRA_ARGS] |
| 131 | |
| 132 | with RemoteOpenAIServer( |
| 133 | model_name, |
| 134 | server_args, |
| 135 | max_wait_seconds=480, |
| 136 | ) as remote_server: |
| 137 | client = remote_server.get_async_client() |
| 138 | await transcribe_and_check( |
| 139 | client, |
| 140 | model_name, |
| 141 | foscolo, |
| 142 | language="it", |
| 143 | expected_text="ove il mio corpo fanciulletto", |
| 144 | ) |