| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import json |
| 5 | from typing import Any |
| 6 | |
| 7 | import numpy as np |
| 8 | import pytest |
| 9 | import pytest_asyncio |
| 10 | from transformers import AutoTokenizer |
| 11 | |
| 12 | from ....conftest import AUDIO_ASSETS, AudioTestAssets, VllmRunner |
| 13 | from ....utils import RemoteOpenAIServer |
| 14 | from ...registry import HF_EXAMPLE_MODELS |
| 15 | |
| 16 | MODEL_NAME = "fixie-ai/ultravox-v0_5-llama-3_2-1b" |
| 17 | |
| 18 | AUDIO_PROMPTS = AUDIO_ASSETS.prompts( |
| 19 | { |
| 20 | "mary_had_lamb": "Transcribe this into English.", |
| 21 | "winning_call": "What is happening in this audio clip?", |
| 22 | } |
| 23 | ) |
| 24 | |
| 25 | MULTI_AUDIO_PROMPT = "Describe each of the audios above." |
| 26 | |
| 27 | AudioTuple = tuple[np.ndarray, int] |
| 28 | |
| 29 | VLLM_PLACEHOLDER = "<|audio|>" |
| 30 | HF_PLACEHOLDER = "<|audio|>" |
| 31 | |
| 32 | CHUNKED_PREFILL_KWARGS = { |
| 33 | "enable_chunked_prefill": True, |
| 34 | "max_num_seqs": 2, |
| 35 | # Use a very small limit to exercise chunked prefill. |
| 36 | "max_num_batched_tokens": 16, |
| 37 | } |
| 38 | |
| 39 | |
| 40 | def params_kwargs_to_cli_args(params_kwargs: dict[str, Any]) -> list[str]: |
| 41 | """Convert kwargs to CLI args.""" |
| 42 | args = [] |
| 43 | for key, value in params_kwargs.items(): |
| 44 | if isinstance(value, bool): |
| 45 | if value: |
| 46 | args.append(f"--{key.replace('_', '-')}") |
| 47 | else: |
| 48 | args.append(f"--{key.replace('_', '-')}={value}") |
| 49 | return args |
| 50 | |
| 51 | |
| 52 | @pytest.fixture( |
| 53 | params=[ |
| 54 | pytest.param({}, marks=pytest.mark.cpu_model), |
| 55 | pytest.param(CHUNKED_PREFILL_KWARGS), |
| 56 | ] |
| 57 | ) |
| 58 | def server(request, audio_assets: AudioTestAssets): |
| 59 | args = [ |
| 60 | "--dtype", |
| 61 | "bfloat16", |
| 62 | "--max-model-len", |
| 63 | "4096", |
| 64 | "--enforce-eager", |
| 65 | "--limit-mm-per-prompt", |
| 66 | json.dumps({"audio": len(audio_assets)}), |
| 67 | "--trust-remote-code", |
| 68 | ] + params_kwargs_to_cli_args(request.param) |
| 69 | |
| 70 | with RemoteOpenAIServer( |
| 71 | MODEL_NAME, args, env_dict={"VLLM_AUDIO_FETCH_TIMEOUT": "30"} |
| 72 | ) as remote_server: |
| 73 | yield remote_server |
| 74 | |
| 75 | |
| 76 | @pytest_asyncio.fixture |
| 77 | async def client(server): |
| 78 | async with server.get_async_client() as async_client: |
| 79 | yield async_client |
| 80 | |
| 81 | |
| 82 | def _get_prompt(audio_count, question, placeholder): |
| 83 | tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME) |
| 84 | placeholder = f"{placeholder}\n" * audio_count |
| 85 | |
| 86 | return tokenizer.apply_chat_template( |
| 87 | [{"role": "user", "content": f"{placeholder}{question}"}], |
| 88 | tokenize=False, |
| 89 | add_generation_prompt=True, |
| 90 | ) |
| 91 | |
| 92 | |
| 93 | def run_multi_audio_test( |
| 94 | vllm_runner: type[VllmRunner], |
| 95 | prompts_and_audios: list[tuple[str, list[AudioTuple]]], |
| 96 | model: str, |
| 97 | *, |
| 98 | dtype: str, |
| 99 | max_tokens: int, |
| 100 | num_logprobs: int, |
| 101 | **kwargs, |
| 102 | ): |
| 103 | model_info = HF_EXAMPLE_MODELS.find_hf_info(model) |
| 104 | model_info.check_available_online(on_fail="skip") |
| 105 | model_info.check_transformers_version(on_fail="skip") |
| 106 | |
| 107 | with vllm_runner( |
| 108 | model, |
| 109 | dtype=dtype, |
| 110 | enforce_eager=True, |
| 111 | limit_mm_per_prompt={ |
| 112 | "audio": max((len(audio) for _, audio in prompts_and_audios)) |
| 113 | }, |
| 114 | **kwargs, |
| 115 | ) as vllm_model: |
| 116 | vllm_outputs = vllm_model.generate_greedy_logprobs( |
| 117 | [prompt for prompt, _ in prompts_and_audios], |
| 118 | max_tokens, |
| 119 | num_logprobs=num_logprobs, |
| 120 | audios=[audios for _, audios in prompts_and_audios], |
| 121 | ) |
| 122 | |
| 123 | # The HuggingFace model doesn't support multiple audios yet, so |
| 124 | # just assert that some tokens were generated. |
| 125 | assert all(tokens for tokens, *_ in vllm_outputs) |
| 126 | |
| 127 | |
| 128 | @pytest.mark.core_model |
| 129 | @pytest.mark.parametrize("dtype", ["half"]) |
| 130 | @pytest.mark.parametrize("max_tokens", [128]) |
| 131 | @pytest.mark.parametrize("num_logprobs", [5]) |
| 132 | @pytest.mark.parametrize( |
| 133 | "vllm_kwargs", |
| 134 | [ |
| 135 | pytest.param({}, marks=pytest.mark.cpu_model), |
| 136 | pytest.param(CHUNKED_PREFILL_KWARGS), |
| 137 | ], |
| 138 | ) |
| 139 | def test_models_with_multiple_audios( |
| 140 | vllm_runner, |
| 141 | audio_assets: AudioTestAssets, |
| 142 | dtype: str, |
| 143 | max_tokens: int, |
| 144 | num_logprobs: int, |
| 145 | vllm_kwargs: dict, |
| 146 | ) -> None: |
| 147 | vllm_prompt = _get_prompt(len(audio_assets), MULTI_AUDIO_PROMPT, VLLM_PLACEHOLDER) |
| 148 | run_multi_audio_test( |
| 149 | vllm_runner, |
| 150 | [(vllm_prompt, [audio.audio_and_sample_rate for audio in audio_assets])], |
| 151 | MODEL_NAME, |
| 152 | dtype=dtype, |
| 153 | max_tokens=max_tokens, |
| 154 | num_logprobs=num_logprobs, |
| 155 | **vllm_kwargs, |
| 156 | ) |
| 157 | |
| 158 | |
| 159 | @pytest.mark.core_model |
| 160 | @pytest.mark.parametrize("dtype", ["half"]) |
| 161 | @pytest.mark.parametrize("max_tokens", [32]) |
| 162 | def test_variable_length_audio_batching( |
| 163 | vllm_runner, |
| 164 | audio_assets: AudioTestAssets, |
| 165 | dtype: str, |
| 166 | max_tokens: int, |
| 167 | ) -> None: |
| 168 | """Test batching of requests with different audio durations. |
| 169 | |
| 170 | This exercises the variable-length tensor handling in |
| 171 | MultiModalFlatField._reduce_data() which was buggy before |
| 172 | https://github.com/vllm-project/vllm/issues/31658 was fixed. |
| 173 | """ |
| 174 | model_info = HF_EXAMPLE_MODELS.find_hf_info(MODEL_NAME) |
| 175 | model_info.check_available_online(on_fail="skip") |
| 176 | model_info.check_transformers_version(on_fail="skip") |
| 177 | |
| 178 | # Create prompts with single audio each (different durations) |
| 179 | prompts_and_audios = [] |
| 180 | for audio, question in zip(audio_assets, AUDIO_PROMPTS): |
| 181 | prompt = _get_prompt(1, question, VLLM_PLACEHOLDER) |
| 182 | prompts_and_audios.append((prompt, [audio.audio_and_sample_rate])) |
| 183 | |
| 184 | with vllm_runner( |
| 185 | MODEL_NAME, |
| 186 | dtype=dtype, |
| 187 | enforce_eager=True, |
| 188 | limit_mm_per_prompt={"audio": 1}, |
| 189 | ) as vllm_model: |
| 190 | # Generate for all prompts in a single batch |
| 191 | # This triggers the variable-length batching code path |
| 192 | outputs = vllm_model.generate_greedy( |
| 193 | [prompt for prompt, _ in prompts_and_audios], |
| 194 | max_tokens, |
| 195 | audios=[audios for _, audios in prompts_and_audios], |
| 196 | ) |
| 197 | |
| 198 | # Verify outputs were generated for each request |
| 199 | assert len(outputs) == len(prompts_and_audios) |
| 200 | for output in outputs: |
| 201 | assert len(output[1]) > 0, "Expected non-empty output" |
| 202 | |
| 203 | |
| 204 | @pytest.mark.asyncio |
| 205 | async def test_online_serving(client, audio_assets: AudioTestAssets): |
| 206 | """Exercises online serving with/without chunked prefill enabled.""" |
| 207 | messages = [ |
| 208 | { |
| 209 | "role": "user", |
| 210 | "content": [ |
| 211 | *[ |
| 212 | {"type": "audio_url", "audio_url": {"url": audio.url}} |
| 213 | for audio in audio_assets |
| 214 | ], |
| 215 | { |
| 216 | "type": "text", |
| 217 | "text": f"What's happening in these {len(audio_assets)} audio clips?", # noqa: E501 |
| 218 | }, |
| 219 | ], |
| 220 | } |
| 221 | ] |
| 222 | |
| 223 | chat_completion = await client.chat.completions.create( |
| 224 | model=MODEL_NAME, messages=messages, max_tokens=10 |
| 225 | ) |
| 226 | |
| 227 | assert len(chat_completion.choices) == 1 |
| 228 | choice = chat_completion.choices[0] |
| 229 | assert choice.finish_reason == "length" |