| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Tests for Qwen3 Omni audio processing and sample rate handling.""" |
| 4 | |
| 5 | from typing import Any |
| 6 | |
| 7 | import numpy as np |
| 8 | import pytest |
| 9 | |
| 10 | from vllm.multimodal import MULTIMODAL_REGISTRY |
| 11 | |
| 12 | from ...utils import build_model_context |
| 13 | |
| 14 | |
| 15 | @pytest.mark.parametrize("model_id", ["Qwen/Qwen3-Omni-30B-A3B-Instruct"]) |
| 16 | @pytest.mark.parametrize( |
| 17 | ("audio_sample_rate", "audio_duration_sec"), |
| 18 | [ |
| 19 | (16000, 1.0), # Native Whisper sample rate, 1 second |
| 20 | (16000, 2.0), # Native Whisper sample rate, 2 seconds |
| 21 | ], |
| 22 | ) |
| 23 | def test_processor_with_audio_sample_rate( |
| 24 | model_id: str, |
| 25 | audio_sample_rate: int, |
| 26 | audio_duration_sec: float, |
| 27 | ) -> None: |
| 28 | """Test that vLLM's processor generates expected outputs with audio_sample_rate. |
| 29 | |
| 30 | This validates that the processor correctly handles audio_sample_rate |
| 31 | passed via hf_processor_mm_kwargs and generates audio tokens. |
| 32 | """ |
| 33 | ctx = build_model_context( |
| 34 | model_id, |
| 35 | limit_mm_per_prompt={"audio": 1, "image": 0, "video": 0}, |
| 36 | ) |
| 37 | processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) |
| 38 | tokenizer = processor.info.get_tokenizer() |
| 39 | |
| 40 | # Create audio data at the specified sample rate |
| 41 | audio_length = int(audio_sample_rate * audio_duration_sec) |
| 42 | rng = np.random.RandomState(42) |
| 43 | audio_data = rng.rand(audio_length).astype(np.float32) |
| 44 | |
| 45 | # Build prompt with audio placeholder |
| 46 | prompt = "<|audio_start|><|audio_pad|><|audio_end|>" |
| 47 | mm_data = {"audio": [(audio_data, audio_sample_rate)]} |
| 48 | |
| 49 | # Apply processor with audio_sample_rate in mm_kwargs |
| 50 | hf_processor_mm_kwargs: dict[str, Any] = { |
| 51 | "audio_sample_rate": audio_sample_rate, |
| 52 | } |
| 53 | processed_inputs = processor( |
| 54 | prompt, |
| 55 | mm_items=processor.info.parse_mm_data(mm_data), |
| 56 | hf_processor_mm_kwargs=hf_processor_mm_kwargs, |
| 57 | ) |
| 58 | |
| 59 | # Verify audio tokens are generated |
| 60 | hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs) |
| 61 | audio_token_id = tokenizer.convert_tokens_to_ids(hf_processor.audio_token) |
| 62 | aud_tok_count = processed_inputs["prompt_token_ids"].count(audio_token_id) |
| 63 | |
| 64 | assert aud_tok_count >= 1, ( |
| 65 | f"Expected at least 1 audio token but got {aud_tok_count}. " |
| 66 | f"sample_rate: {audio_sample_rate}Hz, duration: {audio_duration_sec}s" |
| 67 | ) |
| 68 | |
| 69 | |
| 70 | @pytest.mark.parametrize("model_id", ["Qwen/Qwen3-Omni-30B-A3B-Instruct"]) |
| 71 | def test_longer_audio_generates_more_tokens(model_id: str) -> None: |
| 72 | """Test that longer audio generates more tokens than shorter audio. |
| 73 | |
| 74 | This validates that audio_sample_rate is being used correctly by checking |
| 75 | that audio duration affects token count as expected. |
| 76 | """ |
| 77 | ctx = build_model_context( |
| 78 | model_id, |
| 79 | limit_mm_per_prompt={"audio": 1, "image": 0, "video": 0}, |
| 80 | ) |
| 81 | processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) |
| 82 | tokenizer = processor.info.get_tokenizer() |
| 83 | |
| 84 | audio_sample_rate = 16000 |
| 85 | rng = np.random.RandomState(42) |
| 86 | |
| 87 | def get_token_count(duration: float) -> int: |
| 88 | audio_length = int(audio_sample_rate * duration) |
| 89 | audio_data = rng.rand(audio_length).astype(np.float32) |
| 90 | prompt = "<|audio_start|><|audio_pad|><|audio_end|>" |
| 91 | mm_data = {"audio": [(audio_data, audio_sample_rate)]} |
| 92 | hf_processor_mm_kwargs: dict[str, Any] = { |
| 93 | "audio_sample_rate": audio_sample_rate, |
| 94 | } |
| 95 | processed = processor( |
| 96 | prompt, |
| 97 | mm_items=processor.info.parse_mm_data(mm_data), |
| 98 | hf_processor_mm_kwargs=hf_processor_mm_kwargs, |
| 99 | ) |
| 100 | hf_proc = processor.info.get_hf_processor(**hf_processor_mm_kwargs) |
| 101 | audio_token_id = tokenizer.convert_tokens_to_ids(hf_proc.audio_token) |
| 102 | return processed["prompt_token_ids"].count(audio_token_id) |
| 103 | |
| 104 | short_tokens = get_token_count(1.0) |
| 105 | long_tokens = get_token_count(2.0) |
| 106 | |
| 107 | assert long_tokens > short_tokens, ( |
| 108 | f"Expected longer audio (2s) to have more tokens than shorter (1s). " |
| 109 | f"Got short={short_tokens}, long={long_tokens}" |
| 110 | ) |