| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import io |
| 5 | |
| 6 | # imports for structured outputs tests |
| 7 | import json |
| 8 | |
| 9 | import httpx |
| 10 | import numpy as np |
| 11 | import openai |
| 12 | import pytest |
| 13 | import pytest_asyncio |
| 14 | import soundfile as sf |
| 15 | |
| 16 | from tests.entrypoints.speech_to_text.conftest import add_attention_backend |
| 17 | from tests.utils import RemoteOpenAIServer |
| 18 | from vllm.logger import init_logger |
| 19 | from vllm.multimodal.media.audio import load_audio |
| 20 | |
| 21 | logger = init_logger(__name__) |
| 22 | |
| 23 | SERVER_ARGS = ["--enforce-eager"] |
| 24 | |
| 25 | |
| 26 | def _get_rocm_attention_config(model_name): |
| 27 | """Return appropriate ROCm attention config for the given model. |
| 28 | |
| 29 | Whisper uses cross-attention (ENCODER_DECODER) which ROCM_AITER_FA does |
| 30 | not support. For Whisper we use ROCM_AITER_UNIFIED_ATTN (or TRITON_ATTN |
| 31 | as fallback). Other models use ROCM_AITER_FA only on supported CDNA3+ |
| 32 | hardware and otherwise use the platform default. |
| 33 | """ |
| 34 | from vllm.platforms import current_platform |
| 35 | |
| 36 | if not current_platform.is_rocm(): |
| 37 | return None |
| 38 | |
| 39 | try: |
| 40 | from vllm.platforms.rocm import get_cdna_version |
| 41 | |
| 42 | cdna_version = get_cdna_version() |
| 43 | except ImportError: |
| 44 | logger.warning( |
| 45 | "Could not check cdna version from rocm platform; " |
| 46 | "using a compatible fallback attention backend." |
| 47 | ) |
| 48 | cdna_version = -1 |
| 49 | |
| 50 | if "whisper" in model_name.lower(): |
| 51 | if cdna_version > 2: |
| 52 | return {"backend": "ROCM_AITER_UNIFIED_ATTN"} |
| 53 | return {"backend": "TRITON_ATTN"} |
| 54 | |
| 55 | if cdna_version > 2: |
| 56 | return {"backend": "ROCM_AITER_FA"} |
| 57 | return None |
| 58 | |
| 59 | |
| 60 | def _get_server_args(attention_config): |
| 61 | """Get server args with attention backend if specified.""" |
| 62 | args = SERVER_ARGS.copy() |
| 63 | add_attention_backend(args, attention_config) |
| 64 | return args |
| 65 | |
| 66 | |
| 67 | @pytest.fixture( |
| 68 | scope="module", params=["openai/whisper-small", "google/gemma-3n-E2B-it"] |
| 69 | ) |
| 70 | def server(request): |
| 71 | # Parametrize over model name |
| 72 | attention_config = _get_rocm_attention_config(request.param) |
| 73 | with RemoteOpenAIServer( |
| 74 | request.param, _get_server_args(attention_config) |
| 75 | ) as remote_server: |
| 76 | yield remote_server, request.param |
| 77 | |
| 78 | |
| 79 | @pytest_asyncio.fixture |
| 80 | async def client_and_model(server): |
| 81 | server, model_name = server |
| 82 | async with server.get_async_client() as async_client: |
| 83 | yield async_client, model_name |
| 84 | |
| 85 | |
| 86 | @pytest.mark.asyncio |
| 87 | async def test_non_asr_model(foscolo): |
| 88 | # text to text model |
| 89 | model_name = "JackFram/llama-68m" |
| 90 | attention_config = _get_rocm_attention_config(model_name) |
| 91 | with RemoteOpenAIServer( |
| 92 | model_name, _get_server_args(attention_config) |
| 93 | ) as remote_server: |
| 94 | client = remote_server.get_async_client() |
| 95 | |
| 96 | with pytest.raises(openai.NotFoundError): |
| 97 | await client.audio.translations.create( |
| 98 | model=model_name, file=foscolo, temperature=0.0 |
| 99 | ) |
| 100 | |
| 101 | |
| 102 | @pytest.mark.asyncio |
| 103 | async def test_basic_audio_with_lora(mary_had_lamb): |
| 104 | """Ensure STT (translate) requests can pass LoRA through to generate.""" |
| 105 | # ROCm SPECIFIC CONFIGURATION: |
| 106 | # To ensure the test passes on ROCm, we modify the max model length to 512. |
| 107 | # We DO NOT apply this to other platforms to maintain strict upstream parity. |
| 108 | from vllm.platforms import current_platform |
| 109 | |
| 110 | # NOTE - careful to call this test before the module scoped server |
| 111 | # fixture, otherwise it'll OOMkill the CI |
| 112 | model_name = "ibm-granite/granite-speech-3.3-2b" |
| 113 | lora_model_name = "speech" |
| 114 | server_args = [ |
| 115 | "--enforce-eager", |
| 116 | "--enable-lora", |
| 117 | "--max-lora-rank", |
| 118 | "64", |
| 119 | "--lora-modules", |
| 120 | f"{lora_model_name}={model_name}", |
| 121 | "--max-model-len", |
| 122 | "512" if current_platform.is_rocm() else "2048", |
| 123 | "--max-num-seqs", |
| 124 | "1", |
| 125 | ] |
| 126 | |
| 127 | add_attention_backend(server_args, _get_rocm_attention_config(model_name)) |
| 128 | |
| 129 | # Based on https://github.com/openai/openai-cookbook/blob/main/examples/Whisper_prompting_guide.ipynb. |
| 130 | with RemoteOpenAIServer(model_name, server_args) as remote_server: |
| 131 | client = remote_server.get_async_client() |
| 132 | translation = await client.audio.translations.create( |
| 133 | model=lora_model_name, |
| 134 | file=mary_had_lamb, |
| 135 | extra_body=dict(language="en", to_language="es"), |
| 136 | response_format="text", |
| 137 | temperature=0.0, |
| 138 | ) |
| 139 | out = json.loads(translation)["text"].strip().lower() |
| 140 | assert "pequeño" in out.split(" ") |
| 141 | |
| 142 | |
| 143 | # NOTE: (NickLucche) the large-v3-turbo model was not trained on translation! |
| 144 | @pytest.mark.asyncio |
| 145 | async def test_basic_audio(foscolo, client_and_model): |
| 146 | client, model_name = client_and_model |
| 147 | translation = await client.audio.translations.create( |
| 148 | model=model_name, |
| 149 | file=foscolo, |
| 150 | response_format="text", |
| 151 | # TODO remove `language="it"` once language detection is implemented |
| 152 | extra_body=dict(language="it", to_language="en"), |
| 153 | temperature=0.0, |
| 154 | ) |
| 155 | out = json.loads(translation)["text"].strip().lower() |
| 156 | assert "greek sea" in out |
| 157 | |
| 158 | |
| 159 | @pytest.mark.asyncio |
| 160 | async def test_audio_prompt(foscolo, client_and_model): |
| 161 | client, model_name = client_and_model |
| 162 | # Condition whisper on starting text |
| 163 | prompt = "Nor have I ever" |
| 164 | transcription = await client.audio.translations.create( |
| 165 | model=model_name, |
| 166 | file=foscolo, |
| 167 | prompt=prompt, |
| 168 | extra_body=dict(language="it", to_language="en"), |
| 169 | response_format="text", |
| 170 | temperature=0.0, |
| 171 | ) |
| 172 | out = json.loads(transcription)["text"] |
| 173 | assert "Nor will I ever touch the sacred" not in out |
| 174 | assert prompt not in out |
| 175 | |
| 176 | |
| 177 | @pytest.mark.asyncio |
| 178 | async def test_streaming_response(foscolo, client_and_model, server): |
| 179 | client, model_name = client_and_model |
| 180 | translation = "" |
| 181 | res_no_stream = await client.audio.translations.create( |
| 182 | model=model_name, |
| 183 | file=foscolo, |
| 184 | response_format="json", |
| 185 | extra_body=dict(language="it", to_language="en", seed=42), |
| 186 | temperature=0.0, |
| 187 | ) |
| 188 | |
| 189 | # Stream via HTTPX since OpenAI translation client doesn't expose streaming |
| 190 | server, model_name = server |
| 191 | url = server.url_for("v1/audio/translations") |
| 192 | headers = {"Authorization": f"Bearer {server.DUMMY_API_KEY}"} |
| 193 | data = { |
| 194 | "model": model_name, |
| 195 | "language": "it", |
| 196 | "to_language": "en", |
| 197 | "stream": True, |
| 198 | "temperature": 0.0, |
| 199 | "seed": 42, |
| 200 | } |
| 201 | foscolo.seek(0) |
| 202 | async with httpx.AsyncClient() as http_client: |
| 203 | files = {"file": foscolo} |
| 204 | async with http_client.stream( |
| 205 | "POST", url, headers=headers, data=data, files=files |
| 206 | ) as response: |
| 207 | async for line in response.aiter_lines(): |
| 208 | if not line: |
| 209 | continue |
| 210 | if line.startswith("data: "): |
| 211 | line = line[len("data: ") :] |
| 212 | if line.strip() == "[DONE]": |
| 213 | break |
| 214 | chunk = json.loads(line) |
| 215 | text = chunk["choices"][0].get("delta", {}).get("content") |
| 216 | translation += text or "" |
| 217 | |
| 218 | res_stream = translation.split() |
| 219 | # NOTE There's a small non-deterministic issue here, likely in the attn |
| 220 | # computation, which will cause a few tokens to be different, while still |
| 221 | # being very close semantically. |
| 222 | assert ( |
| 223 | sum([x == y for x, y in zip(res_stream, res_no_stream.text.split())]) |
| 224 | >= len(res_stream) * 0.87 |
| 225 | ) |
| 226 | |
| 227 | |
| 228 | @pytest.mark.asyncio |
| 229 | async def test_stream_options(foscolo, server): |
| 230 | server, model_name = server |
| 231 | url = server.url_for("v1/audio/translations") |
| 232 | headers = {"Authorization": f"Bearer {server.DUMMY_API_KEY}"} |
| 233 | data = { |
| 234 | "model": model_name, |
| 235 | "language": "it", |
| 236 | "to_language": "en", |
| 237 | "stream": True, |
| 238 | "stream_include_usage": True, |
| 239 | "stream_continuous_usage_stats": True, |
| 240 | "temperature": 0.0, |
| 241 | } |
| 242 | foscolo.seek(0) |
| 243 | final = False |
| 244 | continuous = True |
| 245 | async with httpx.AsyncClient() as http_client: |
| 246 | files = {"file": foscolo} |
| 247 | async with http_client.stream( |
| 248 | "POST", url, headers=headers, data=data, files=files |
| 249 | ) as response: |
| 250 | async for line in response.aiter_lines(): |
| 251 | if not line: |
| 252 | continue |
| 253 | if line.startswith("data: "): |
| 254 | line = line[len("data: ") :] |
| 255 | if line.strip() == "[DONE]": |
| 256 | break |
| 257 | chunk = json.loads(line) |
| 258 | choices = chunk.get("choices", []) |
| 259 | if not choices: |
| 260 | # final usage sent |
| 261 | final = True |
| 262 | else: |
| 263 | continuous = continuous and ("usage" in chunk) |
| 264 | assert final and continuous |
| 265 | |
| 266 | |
| 267 | @pytest.mark.asyncio |
| 268 | async def test_long_audio_request(foscolo, client_and_model): |
| 269 | client, model_name = client_and_model |
| 270 | if model_name == "google/gemma-3n-E2B-it": |
| 271 | pytest.skip("Gemma3n does not support long audio requests") |
| 272 | foscolo.seek(0) |
| 273 | audio, sr = load_audio(foscolo) |
| 274 | repeated_audio = np.tile(audio, 2) |
| 275 | # Repeated audio to buffer |
| 276 | buffer = io.BytesIO() |
| 277 | sf.write(buffer, repeated_audio, sr, format="WAV") |
| 278 | buffer.seek(0) |
| 279 | translation = await client.audio.translations.create( |
| 280 | model=model_name, |
| 281 | file=buffer, |
| 282 | extra_body=dict(language="it", to_language="en"), |
| 283 | response_format="text", |
| 284 | temperature=0.0, |
| 285 | ) |
| 286 | out = json.loads(translation)["text"].strip().lower() |
| 287 | assert out.count("greek sea") == 2 |
| 288 | |
| 289 | |
| 290 | @pytest.mark.asyncio |
| 291 | async def test_audio_with_max_tokens(mary_had_lamb, client_and_model): |
| 292 | client, model_name = client_and_model |
| 293 | transcription = await client.audio.translations.create( |
| 294 | model=model_name, |
| 295 | file=mary_had_lamb, |
| 296 | response_format="text", |
| 297 | temperature=0.0, |
| 298 | extra_body={"max_completion_tokens": 1}, |
| 299 | ) |
| 300 | out = json.loads(transcription) |
| 301 | out_text = out["text"] |
| 302 | print(out_text) |
| 303 | from transformers import AutoTokenizer |
| 304 | |
| 305 | tok = AutoTokenizer.from_pretrained(model_name) |
| 306 | out_tokens = tok(out_text, add_special_tokens=False)["input_ids"] |
| 307 | assert len(out_tokens) == 1 |
| 308 | # max_completion_tokens > max_model_len |
| 309 | # max_model_len=32768 for Gemma-3n-E2B-it |
| 310 | transcription = await client.audio.transcriptions.create( |
| 311 | model=model_name, |
| 312 | file=mary_had_lamb, |
| 313 | response_format="text", |
| 314 | temperature=0.0, |
| 315 | extra_body={ |
| 316 | "max_completion_tokens": int(1e6), |
| 317 | "repetition_penalty": 1.3, |
| 318 | }, |
| 319 | ) |
| 320 | out = json.loads(transcription) |
| 321 | out_text = out["text"] |
| 322 | print(out_text) |
| 323 | out_tokens = tok(out_text, add_special_tokens=False)["input_ids"] |
| 324 | assert len(out_tokens) < 450 # ~Whisper max output len |