| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | from collections.abc import Sequence |
| 5 | from typing import Any |
| 6 | |
| 7 | import pytest |
| 8 | import torch |
| 9 | from transformers import AutoModelForSpeechSeq2Seq |
| 10 | |
| 11 | from vllm.assets.audio import AudioAsset |
| 12 | from vllm.multimodal.audio import AudioResampler |
| 13 | from vllm.platforms import current_platform |
| 14 | |
| 15 | from ....conftest import HfRunner, PromptAudioInput, VllmRunner |
| 16 | from ....utils import create_new_process_for_each_test, multi_gpu_test |
| 17 | from ...registry import HF_EXAMPLE_MODELS |
| 18 | from ...utils import check_logprobs_close |
| 19 | |
| 20 | VLLM_PROMPT = "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>" |
| 21 | HF_PROMPT = "" |
| 22 | # Whisper expects 16kHz audio |
| 23 | WHISPER_SAMPLE_RATE = 16000 |
| 24 | BEAM_WIDTHS = (1, 2) |
| 25 | |
| 26 | |
| 27 | @pytest.fixture(autouse=True) |
| 28 | def use_spawn_for_whisper(monkeypatch): |
| 29 | """Whisper has issues with forked workers, use spawn instead.""" |
| 30 | monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn") |
| 31 | |
| 32 | |
| 33 | def run_test( |
| 34 | hf_runner: type[HfRunner], |
| 35 | vllm_runner: type[VllmRunner], |
| 36 | inputs: Sequence[tuple[list[str], list[str], PromptAudioInput]], |
| 37 | model: str, |
| 38 | *, |
| 39 | max_model_len: int, |
| 40 | dtype: str, |
| 41 | max_tokens: int, |
| 42 | num_logprobs: int, |
| 43 | tensor_parallel_size: int, |
| 44 | distributed_executor_backend: str | None = None, |
| 45 | enforce_eager: bool = True, |
| 46 | gpu_memory_utilization: float = 0.9, |
| 47 | ) -> None: |
| 48 | """Inference result should be the same between hf and vllm. |
| 49 | |
| 50 | All the audio fixtures for the test are from AudioAsset. |
| 51 | For huggingface runner, we provide the audio as input. |
| 52 | For vllm runner, we provide MultiModalDataDict objects |
| 53 | and corresponding MultiModalConfig as input. |
| 54 | """ |
| 55 | with vllm_runner( |
| 56 | model, |
| 57 | dtype=dtype, |
| 58 | max_model_len=max_model_len, |
| 59 | tensor_parallel_size=tensor_parallel_size, |
| 60 | distributed_executor_backend=distributed_executor_backend, |
| 61 | limit_mm_per_prompt={"audio": 2}, |
| 62 | enforce_eager=enforce_eager, |
| 63 | gpu_memory_utilization=gpu_memory_utilization, |
| 64 | disable_custom_all_reduce=True, |
| 65 | ) as vllm_model: |
| 66 | vllm_outputs_per_case = [ |
| 67 | vllm_model.generate_greedy_logprobs( |
| 68 | vllm_prompts, |
| 69 | max_tokens, |
| 70 | num_logprobs=num_logprobs, |
| 71 | audios=audios, |
| 72 | ) |
| 73 | for vllm_prompts, _, audios in inputs |
| 74 | ] |
| 75 | |
| 76 | with hf_runner(model, dtype=dtype, auto_cls=AutoModelForSpeechSeq2Seq) as hf_model: |
| 77 | hf_outputs_per_case = [ |
| 78 | hf_model.generate_greedy_logprobs_limit( |
| 79 | hf_prompts, |
| 80 | max_tokens, |
| 81 | num_logprobs=num_logprobs, |
| 82 | audios=audios, |
| 83 | ) |
| 84 | for _, hf_prompts, audios in inputs |
| 85 | ] |
| 86 | |
| 87 | for hf_outputs, vllm_outputs in zip(hf_outputs_per_case, vllm_outputs_per_case): |
| 88 | check_logprobs_close( |
| 89 | outputs_0_lst=hf_outputs, |
| 90 | outputs_1_lst=vllm_outputs, |
| 91 | name_0="hf", |
| 92 | name_1="vllm", |
| 93 | ) |
| 94 | |
| 95 | |
| 96 | @pytest.fixture |
| 97 | def resampled_assets() -> list[tuple[Any, int]]: |
| 98 | audio_assets = [AudioAsset("mary_had_lamb"), AudioAsset("winning_call")] |
| 99 | sampled_assets = [] |
| 100 | resampler = AudioResampler(target_sr=WHISPER_SAMPLE_RATE) |
| 101 | for asset in audio_assets: |
| 102 | audio, orig_sr = asset.audio_and_sample_rate |
| 103 | # Resample to Whisper's expected sample rate (16kHz) |
| 104 | if orig_sr != WHISPER_SAMPLE_RATE: |
| 105 | audio = resampler.resample(audio, orig_sr=orig_sr) |
| 106 | sampled_assets.append( |
| 107 | (audio, WHISPER_SAMPLE_RATE), |
| 108 | ) |
| 109 | return sampled_assets |
| 110 | |
| 111 | |
| 112 | @pytest.fixture |
| 113 | def input_audios( |
| 114 | resampled_assets, |
| 115 | ) -> list[tuple[list[str], list[str], list[tuple[Any, int]]]]: |
| 116 | inputs = [] |
| 117 | # audio assets are resampled to WHISPER_SAMPLE_RATE |
| 118 | for audio_info in resampled_assets: |
| 119 | # vLLM prompts, HF prompts, audio inputs |
| 120 | inputs.append(([VLLM_PROMPT], [HF_PROMPT], [audio_info])) |
| 121 | return inputs |
| 122 | |
| 123 | |
| 124 | def check_model_available(model: str) -> None: |
| 125 | model_info = HF_EXAMPLE_MODELS.find_hf_info(model) |
| 126 | model_info.check_available_online(on_fail="skip") |
| 127 | model_info.check_transformers_version(on_fail="skip") |
| 128 | |
| 129 | |
| 130 | @pytest.mark.parametrize("dtype", ["half"]) |
| 131 | @pytest.mark.parametrize("max_tokens", [64]) |
| 132 | def test_beam_search_encoder_decoder( |
| 133 | hf_runner, |
| 134 | vllm_runner, |
| 135 | dtype: str, |
| 136 | max_tokens: int, |
| 137 | resampled_assets, |
| 138 | ) -> None: |
| 139 | """Test beam search with encoder-decoder models (Whisper).""" |
| 140 | model = "openai/whisper-large-v3-turbo" |
| 141 | check_model_available(model) |
| 142 | |
| 143 | hf_prompts = [ |
| 144 | "<|startoftranscript|>", |
| 145 | "<|startoftranscript|>", |
| 146 | ] |
| 147 | |
| 148 | with hf_runner(model, dtype=dtype, auto_cls=AutoModelForSpeechSeq2Seq) as hf_model: |
| 149 | hf_outputs_by_beam_width = [ |
| 150 | hf_model.generate_beam_search( |
| 151 | hf_prompts, |
| 152 | beam_width=beam_width, |
| 153 | max_tokens=max_tokens, |
| 154 | audios=resampled_assets, |
| 155 | ) |
| 156 | for beam_width in BEAM_WIDTHS |
| 157 | ] |
| 158 | |
| 159 | # Test both explicit encoder/decoder prompts |
| 160 | vllm_prompts = [ |
| 161 | # Implicit encoder/decoder prompt |
| 162 | { |
| 163 | "prompt": "<|startoftranscript|>", |
| 164 | "multi_modal_data": {"audio": resampled_assets[0]}, |
| 165 | }, |
| 166 | # Explicit encoder/decover prompt |
| 167 | { |
| 168 | "encoder_prompt": { |
| 169 | "prompt": "", |
| 170 | "multi_modal_data": {"audio": resampled_assets[1]}, |
| 171 | }, |
| 172 | "decoder_prompt": "<|startoftranscript|>", |
| 173 | }, |
| 174 | ] |
| 175 | |
| 176 | with vllm_runner( |
| 177 | model, |
| 178 | dtype="half", |
| 179 | max_model_len=448, |
| 180 | tensor_parallel_size=1, |
| 181 | max_num_seqs=4, |
| 182 | limit_mm_per_prompt={"audio": 2}, |
| 183 | enforce_eager=True, |
| 184 | ) as vllm_model: |
| 185 | vllm_outputs_by_beam_width = [ |
| 186 | vllm_model.generate_beam_search( |
| 187 | vllm_prompts, |
| 188 | beam_width=beam_width, |
| 189 | max_tokens=max_tokens, |
| 190 | ) |
| 191 | for beam_width in BEAM_WIDTHS |
| 192 | ] |
| 193 | |
| 194 | for beam_width, hf_outputs, vllm_outputs in zip( |
| 195 | BEAM_WIDTHS, hf_outputs_by_beam_width, vllm_outputs_by_beam_width |
| 196 | ): |
| 197 | for i in range(len(vllm_prompts)): |
| 198 | hf_output_ids, hf_output_texts = hf_outputs[i] |
| 199 | vllm_output_ids, vllm_output_texts = vllm_outputs[i] |
| 200 | |
| 201 | for j, (hf_text, vllm_text) in enumerate( |
| 202 | zip(hf_output_texts, vllm_output_texts) |
| 203 | ): |
| 204 | print(f">>>{j}-th hf output [NOTE: special tokens are filtered]:") |
| 205 | print(hf_text) |
| 206 | print(f">>>{j}-th vllm output:") |
| 207 | print(vllm_text) |
| 208 | |
| 209 | # Check that we got the same number of beams |
| 210 | assert len(hf_output_ids) == len(vllm_output_ids) == beam_width |
| 211 | |
| 212 | # For encoder-decoder models, we primarily want to verify that: |
| 213 | # 1. Beam search completes without errors |
| 214 | # 2. We get the expected number of beams |
| 215 | # 3. Outputs are reasonable (non-empty, diverse beams) |
| 216 | for j in range(len(vllm_output_ids)): |
| 217 | # Check that outputs are not empty |
| 218 | assert len(vllm_output_ids[j]) > 0, ( |
| 219 | f"Prompt {i}, beam {j}: empty output" |
| 220 | ) |
| 221 | # Check that decoded text is not empty |
| 222 | assert len(vllm_output_texts[j].strip()) > 0, ( |
| 223 | f"Prompt {i}, beam {j}: empty text output" |
| 224 | ) |
| 225 | |
| 226 | |
| 227 | def test_parse_language_detection_output(): |
| 228 | """Unit test for WhisperForConditionalGeneration.parse_language_detection_output. |
| 229 | |
| 230 | No GPU or model loading required. |
| 231 | """ |
| 232 | from unittest.mock import MagicMock |
| 233 | |
| 234 | from vllm.model_executor.models.whisper import ( |
| 235 | WhisperForConditionalGeneration, |
| 236 | ) |
| 237 | |
| 238 | cls = WhisperForConditionalGeneration |
| 239 | |
| 240 | def make_tokenizer(return_value: str) -> MagicMock: |
| 241 | tok = MagicMock() |
| 242 | tok.decode = MagicMock(return_value=return_value) |
| 243 | return tok |
| 244 | |
| 245 | # English |
| 246 | assert ( |
| 247 | cls.parse_language_detection_output([50259], make_tokenizer("<|en|>")) == "en" |
| 248 | ) |
| 249 | |
| 250 | # German |
| 251 | assert ( |
| 252 | cls.parse_language_detection_output([50261], make_tokenizer("<|de|>")) == "de" |
| 253 | ) |
| 254 | |
| 255 | # Unsupported language code |
| 256 | with pytest.raises(AssertionError): |
| 257 | cls.parse_language_detection_output([99999], make_tokenizer("<|xx|>")) |
| 258 | |
| 259 | # No special token format |
| 260 | with pytest.raises(AssertionError): |
| 261 | cls.parse_language_detection_output([1], make_tokenizer("hello")) |
| 262 | |
| 263 | # Empty token_ids |
| 264 | with pytest.raises((AssertionError, IndexError)): |
| 265 | cls.parse_language_detection_output([], make_tokenizer("anything")) |
| 266 | |
| 267 | |
| 268 | def test_create_fake_bias_for_k_proj_injects_out_features(): |
| 269 | """Packed k_proj weights get a zeros bias of length ``out_features``.""" |
| 270 | from vllm.model_executor.models.whisper import _create_fake_bias_for_k_proj |
| 271 | |
| 272 | packed = torch.ones(2, 16) |
| 273 | result = dict( |
| 274 | _create_fake_bias_for_k_proj( |
| 275 | [("model.encoder.layers.0.self_attn.k_proj.weight_packed", packed)], |
| 276 | ".k_proj.weight_packed", |
| 277 | out_features=8, |
| 278 | ) |
| 279 | ) |
| 280 | bias_name = "model.encoder.layers.0.self_attn.k_proj.bias" |
| 281 | assert bias_name in result |
| 282 | assert "model.encoder.layers.0.self_attn.k_proj.bias_packed" not in result |
| 283 | assert tuple(result[bias_name].shape) == (8,) |
| 284 | assert torch.equal(result[bias_name], torch.zeros(8)) |
| 285 | |
| 286 | |
| 287 | def test_create_fake_bias_for_k_proj_forwards_real_bias(): |
| 288 | """A checkpoint bias is forwarded and no extra fake bias is emitted.""" |
| 289 | from vllm.model_executor.models.whisper import _create_fake_bias_for_k_proj |
| 290 | |
| 291 | weight = torch.ones(8, 16) |
| 292 | real_bias = torch.ones(8) |
| 293 | items = list( |
| 294 | _create_fake_bias_for_k_proj( |
| 295 | [ |
| 296 | ("layers.0.self_attn.k_proj.weight", weight), |
| 297 | ("layers.0.self_attn.k_proj.bias", real_bias), |
| 298 | ], |
| 299 | ".k_proj.weight", |
| 300 | out_features=8, |
| 301 | ) |
| 302 | ) |
| 303 | names = [n for n, _ in items] |
| 304 | assert names == [ |
| 305 | "layers.0.self_attn.k_proj.weight", |
| 306 | "layers.0.self_attn.k_proj.bias", |
| 307 | ] |
| 308 | assert items[1][1] is real_bias |
| 309 | |
| 310 | |
| 311 | @pytest.mark.core_model |
| 312 | @pytest.mark.cpu_model |
| 313 | @pytest.mark.parametrize("model", ["openai/whisper-large-v3-turbo"]) |
| 314 | @pytest.mark.parametrize("dtype", ["half", "float"]) |
| 315 | @pytest.mark.parametrize("num_logprobs", [5]) |
| 316 | @pytest.mark.parametrize("enforce_eager", [True, False]) |
| 317 | def test_models( |
| 318 | hf_runner, |
| 319 | vllm_runner, |
| 320 | model: str, |
| 321 | dtype: str, |
| 322 | num_logprobs: int, |
| 323 | input_audios, |
| 324 | enforce_eager: bool, |
| 325 | ) -> None: |
| 326 | check_model_available(model) |
| 327 | if current_platform.is_cpu() and not enforce_eager: |
| 328 | pytest.skip("Skipping test for CPU with non-eager mode") |
| 329 | run_test( |
| 330 | hf_runner, |
| 331 | vllm_runner, |
| 332 | input_audios, |
| 333 | model, |
| 334 | dtype=dtype, |
| 335 | max_model_len=448, |
| 336 | max_tokens=200, |
| 337 | num_logprobs=num_logprobs, |
| 338 | tensor_parallel_size=1, |
| 339 | enforce_eager=enforce_eager, |
| 340 | ) |
| 341 | |
| 342 | |
| 343 | @multi_gpu_test(num_gpus=2) |
| 344 | @pytest.mark.core_model |
| 345 | @pytest.mark.parametrize("model", ["openai/whisper-large-v3-turbo"]) |
| 346 | @pytest.mark.parametrize("distributed_executor_backend", ["ray", "mp"]) |
| 347 | @pytest.mark.parametrize("dtype", ["half"]) |
| 348 | @pytest.mark.parametrize("max_tokens", [200]) |
| 349 | @pytest.mark.parametrize("num_logprobs", [5]) |
| 350 | @create_new_process_for_each_test("spawn") |
| 351 | def test_models_distributed( |
| 352 | hf_runner, |
| 353 | vllm_runner, |
| 354 | model: str, |
| 355 | distributed_executor_backend: str, |
| 356 | dtype: str, |
| 357 | max_tokens: int, |
| 358 | num_logprobs: int, |
| 359 | input_audios, |
| 360 | ) -> None: |
| 361 | check_model_available(model) |
| 362 | run_test( |
| 363 | hf_runner, |
| 364 | vllm_runner, |
| 365 | input_audios, |
| 366 | model, |
| 367 | dtype=dtype, |
| 368 | max_model_len=448, |
| 369 | max_tokens=max_tokens, |
| 370 | num_logprobs=num_logprobs, |
| 371 | tensor_parallel_size=2, |
| 372 | distributed_executor_backend=distributed_executor_backend, |
| 373 | enforce_eager=False, |
| 374 | gpu_memory_utilization=0.65, |
| 375 | ) |
| 376 | |
| 377 | |
| 378 | @pytest.mark.core_model |
| 379 | @pytest.mark.parametrize("model", ["openai/whisper-large-v3-turbo"]) |
| 380 | def test_encoder_cache_cleanup( |
| 381 | vllm_runner, |
| 382 | model: str, |
| 383 | input_audios, |
| 384 | monkeypatch, |
| 385 | ) -> None: |
| 386 | """Test that encoder cache is properly cleaned up after requests complete. |
| 387 | |
| 388 | This is a regression test for a bug where encoder cache entries were freed |
| 389 | in the same scheduling step they were allocated, before the model could use |
| 390 | them. |
| 391 | """ |
| 392 | # Set single-process mode to access the model runner's encoder cache directly |
| 393 | monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0") |
| 394 | check_model_available(model) |
| 395 | |
| 396 | with vllm_runner( |
| 397 | model, |
| 398 | dtype="half", |
| 399 | max_model_len=448, |
| 400 | tensor_parallel_size=1, |
| 401 | limit_mm_per_prompt={"audio": 2}, |
| 402 | enforce_eager=True, |
| 403 | ) as vllm_model: |
| 404 | engine_core = vllm_model.llm.llm_engine.engine_core.engine_core |
| 405 | model_runner = engine_core.model_executor.driver_worker.worker.model_runner |
| 406 | encoder_cache = model_runner.encoder_cache |
| 407 | |
| 408 | # Run multiple sequential requests to ensure cache is properly managed |
| 409 | for vllm_prompts, _, audios in input_audios: |
| 410 | vllm_model.generate_greedy(vllm_prompts, max_tokens=50, audios=audios) |
| 411 | |
| 412 | # After all requests complete, encoder cache should be empty |
| 413 | cache_size = len(encoder_cache) |
| 414 | assert cache_size == 0, ( |
| 415 | f"Encoder cache should be empty after all requests complete, " |
| 416 | f"but has {cache_size} entries. This indicates encoder cache " |
| 417 | f"entries are not being properly freed." |
| 418 | ) |