| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Tests for Whisper's multimodal preprocessing.""" |
| 4 | |
| 5 | import numpy as np |
| 6 | import pytest |
| 7 | |
| 8 | from vllm.multimodal import MULTIMODAL_REGISTRY |
| 9 | |
| 10 | from ...utils import build_model_context |
| 11 | |
| 12 | |
| 13 | @pytest.mark.parametrize("model_id", ["openai/whisper-large-v3-turbo"]) |
| 14 | @pytest.mark.parametrize("audio_duration_s", [5, 30, 35]) |
| 15 | def test_audio_features_fit_encoder_window( |
| 16 | model_id: str, |
| 17 | audio_duration_s: int, |
| 18 | ) -> None: |
| 19 | """The encoder only has positions for one 30 s window, so a longer clip |
| 20 | has to be cut to that window like the HF feature extractor does by default. |
| 21 | """ |
| 22 | ctx = build_model_context(model_id, limit_mm_per_prompt={"audio": 1}) |
| 23 | processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) |
| 24 | feature_extractor = processor.info.get_feature_extractor() |
| 25 | |
| 26 | sampling_rate = feature_extractor.sampling_rate |
| 27 | rng = np.random.RandomState(0) |
| 28 | audio = rng.rand(sampling_rate * audio_duration_s).astype(np.float32) |
| 29 | |
| 30 | processed_inputs = processor( |
| 31 | "<|startoftranscript|>", |
| 32 | mm_items=processor.info.parse_mm_data({"audio": [(audio, sampling_rate)]}), |
| 33 | hf_processor_mm_kwargs={}, |
| 34 | ) |
| 35 | |
| 36 | input_features = processed_inputs["mm_kwargs"].get_data()["input_features"] |
| 37 | assert input_features.shape[-1] == feature_extractor.nb_max_frames |