| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | # test_audio.py |
| 4 | import math |
| 5 | from unittest.mock import patch |
| 6 | |
| 7 | import numpy as np |
| 8 | import pytest |
| 9 | import torch |
| 10 | |
| 11 | from vllm.multimodal.audio import ( |
| 12 | MONO_AUDIO_SPEC, |
| 13 | PASSTHROUGH_AUDIO_SPEC, |
| 14 | AudioResampler, |
| 15 | AudioSpec, |
| 16 | ChannelReduction, |
| 17 | _get_torchaudio_resampler, |
| 18 | normalize_audio, |
| 19 | resample_audio_pyav, |
| 20 | resample_audio_scipy, |
| 21 | resample_audio_torchaudio, |
| 22 | split_audio, |
| 23 | ) |
| 24 | |
| 25 | |
| 26 | @pytest.fixture |
| 27 | def dummy_audio(): |
| 28 | return np.array([0.0, 0.1, 0.2, 0.3, 0.4], dtype=float) |
| 29 | |
| 30 | |
| 31 | def test_resample_audio_pyav(dummy_audio): |
| 32 | out_down = resample_audio_pyav(dummy_audio, orig_sr=4, target_sr=2) |
| 33 | out_up = resample_audio_pyav(dummy_audio, orig_sr=2, target_sr=4) |
| 34 | out_same = resample_audio_pyav(dummy_audio, orig_sr=4, target_sr=4) |
| 35 | |
| 36 | assert len(out_down) == 3 |
| 37 | assert len(out_up) == 10 |
| 38 | assert np.all(out_same == dummy_audio) |
| 39 | |
| 40 | |
| 41 | def test_resample_audio_scipy(dummy_audio): |
| 42 | out_down = resample_audio_scipy(dummy_audio, orig_sr=4, target_sr=2) |
| 43 | out_up = resample_audio_scipy(dummy_audio, orig_sr=2, target_sr=4) |
| 44 | out_same = resample_audio_scipy(dummy_audio, orig_sr=4, target_sr=4) |
| 45 | |
| 46 | assert len(out_down) == 3 |
| 47 | assert len(out_up) == 10 |
| 48 | assert np.all(out_same == dummy_audio) |
| 49 | |
| 50 | |
| 51 | def test_resample_audio_scipy_non_integer_ratio(dummy_audio): |
| 52 | out = resample_audio_scipy(dummy_audio, orig_sr=5, target_sr=3) |
| 53 | |
| 54 | expected_len = int(round(len(dummy_audio) * 3 / 5)) |
| 55 | assert len(out) == expected_len |
| 56 | |
| 57 | assert isinstance(out, np.ndarray) |
| 58 | assert np.isfinite(out).all() |
| 59 | |
| 60 | |
| 61 | def test_resample_audio_scipy_non_divisible_sample_rates(): |
| 62 | audio = np.arange(441, dtype=float) |
| 63 | out = resample_audio_scipy(audio, orig_sr=44100, target_sr=16000) |
| 64 | |
| 65 | expected_len = math.ceil(len(audio) * 16000 / 44100) |
| 66 | assert len(out) == expected_len |
| 67 | |
| 68 | assert isinstance(out, np.ndarray) |
| 69 | assert np.isfinite(out).all() |
| 70 | |
| 71 | |
| 72 | def test_resample_audio_scipy_resamples_last_axis_for_multichannel(): |
| 73 | audio = np.arange(2 * 441, dtype=float).reshape(2, 441) |
| 74 | out = resample_audio_scipy(audio, orig_sr=44100, target_sr=16000) |
| 75 | |
| 76 | expected_len = math.ceil(audio.shape[-1] * 16000 / 44100) |
| 77 | assert out.shape == (2, expected_len) |
| 78 | assert np.isfinite(out).all() |
| 79 | |
| 80 | |
| 81 | def test_resample_audio_torchaudio(dummy_audio): |
| 82 | out_down = resample_audio_torchaudio(dummy_audio, orig_sr=4, target_sr=2) |
| 83 | out_up = resample_audio_torchaudio(dummy_audio, orig_sr=2, target_sr=4) |
| 84 | out_same = resample_audio_torchaudio(dummy_audio, orig_sr=4, target_sr=4) |
| 85 | |
| 86 | assert len(out_down) == 3 |
| 87 | assert len(out_up) == 10 |
| 88 | assert np.all(out_same == dummy_audio) |
| 89 | |
| 90 | |
| 91 | def test_resample_audio_torchaudio_non_divisible_sample_rates(): |
| 92 | audio = np.arange(441, dtype=float) |
| 93 | out = resample_audio_torchaudio(audio, orig_sr=44100, target_sr=16000) |
| 94 | |
| 95 | expected_len = math.ceil(len(audio) * 16000 / 44100) |
| 96 | assert len(out) == expected_len |
| 97 | |
| 98 | assert isinstance(out, np.ndarray) |
| 99 | assert np.isfinite(out).all() |
| 100 | |
| 101 | |
| 102 | def test_resample_audio_torchaudio_resamples_last_axis_for_multichannel(): |
| 103 | audio = np.arange(2 * 441, dtype=float).reshape(2, 441) |
| 104 | out = resample_audio_torchaudio(audio, orig_sr=44100, target_sr=16000) |
| 105 | |
| 106 | expected_len = math.ceil(audio.shape[-1] * 16000 / 44100) |
| 107 | assert out.shape == (2, expected_len) |
| 108 | assert np.isfinite(out).all() |
| 109 | |
| 110 | |
| 111 | def test_resample_audio_torchaudio_short_input_needs_no_padding(): |
| 112 | # Unlike the PyAV resampler, torchaudio produces output for inputs |
| 113 | # shorter than libswresample's minimum frame size. |
| 114 | audio = np.arange(8, dtype=float) |
| 115 | out = resample_audio_torchaudio(audio, orig_sr=44100, target_sr=16000) |
| 116 | |
| 117 | assert len(out) == math.ceil(len(audio) * 16000 / 44100) |
| 118 | assert np.isfinite(out).all() |
| 119 | |
| 120 | |
| 121 | def test_audio_resampler_pyav_calls_resample(dummy_audio): |
| 122 | resampler = AudioResampler(target_sr=22050, method="pyav") |
| 123 | with patch("vllm.multimodal.audio.resample_audio_pyav") as mock_resample: |
| 124 | mock_resample.return_value = dummy_audio |
| 125 | out = resampler.resample(dummy_audio, orig_sr=44100) |
| 126 | mock_resample.assert_called_once_with( |
| 127 | dummy_audio, orig_sr=44100, target_sr=22050 |
| 128 | ) |
| 129 | assert np.all(out == dummy_audio) |
| 130 | |
| 131 | |
| 132 | def test_audio_resampler_scipy_calls_resample(dummy_audio): |
| 133 | resampler = AudioResampler(target_sr=22050, method="scipy") |
| 134 | with patch("vllm.multimodal.audio.resample_audio_scipy") as mock_resample: |
| 135 | mock_resample.return_value = dummy_audio |
| 136 | out = resampler.resample(dummy_audio, orig_sr=44100) |
| 137 | mock_resample.assert_called_once_with( |
| 138 | dummy_audio, orig_sr=44100, target_sr=22050 |
| 139 | ) |
| 140 | assert np.all(out == dummy_audio) |
| 141 | |
| 142 | |
| 143 | def test_audio_resampler_torchaudio(dummy_audio): |
| 144 | resampler = AudioResampler(target_sr=22050, method="torchaudio") |
| 145 | out_down = resampler.resample(dummy_audio, orig_sr=44100) |
| 146 | |
| 147 | expected_len = math.ceil(len(dummy_audio) * 22050 / 44100) |
| 148 | assert len(out_down) == expected_len |
| 149 | assert np.isfinite(out_down).all() |
| 150 | |
| 151 | |
| 152 | def test_resample_audio_torchaudio_caches_kernel(): |
| 153 | # The sinc kernel is rebuilt per (orig_sr, target_sr) pair; repeated |
| 154 | # requests at the same pair must reuse the cached one. |
| 155 | _get_torchaudio_resampler.cache_clear() |
| 156 | audio = np.arange(441, dtype=float) |
| 157 | |
| 158 | resample_audio_torchaudio(audio, orig_sr=44100, target_sr=16000) |
| 159 | resample_audio_torchaudio(audio, orig_sr=44100, target_sr=16000) |
| 160 | resample_audio_torchaudio(audio, orig_sr=48000, target_sr=16000) |
| 161 | |
| 162 | info = _get_torchaudio_resampler.cache_info() |
| 163 | assert info.misses == 2 |
| 164 | assert info.hits == 1 |
| 165 | |
| 166 | |
| 167 | def test_audio_resampler_default_is_torchaudio(): |
| 168 | # Deliberate behavior change: torchaudio is the default resampling backend |
| 169 | # for both AudioResampler and the data parser that wraps it. |
| 170 | assert AudioResampler(target_sr=16000).method == "torchaudio" |
| 171 | |
| 172 | from vllm.multimodal.parse import MultiModalDataParser |
| 173 | |
| 174 | parser = MultiModalDataParser(target_sr=16000) |
| 175 | assert parser.audio_resampler.method == "torchaudio" |
| 176 | |
| 177 | |
| 178 | def test_audio_resampler_invalid_method(dummy_audio): |
| 179 | # Validated eagerly so a bad method fails at construction, not on the |
| 180 | # first audio request. |
| 181 | with pytest.raises(ValueError, match="Invalid resampling method"): |
| 182 | AudioResampler(target_sr=22050, method="invalid") |
| 183 | |
| 184 | |
| 185 | def test_audio_resampler_no_target_sr(dummy_audio): |
| 186 | resampler = AudioResampler(target_sr=None) |
| 187 | with pytest.raises(RuntimeError): |
| 188 | resampler.resample(dummy_audio, orig_sr=44100) |
| 189 | |
| 190 | |
| 191 | # ============================================================ |
| 192 | # Tests for normalize_audio function |
| 193 | # ============================================================ |
| 194 | |
| 195 | |
| 196 | class TestNormalizeAudio: |
| 197 | """Tests for normalize_audio function with different specs.""" |
| 198 | |
| 199 | def test_passthrough_preserves_audio(self): |
| 200 | """Passthrough spec should not modify audio.""" |
| 201 | stereo = np.array([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]], dtype=np.float32) |
| 202 | result = normalize_audio(stereo, PASSTHROUGH_AUDIO_SPEC) |
| 203 | np.testing.assert_array_equal(result, stereo) |
| 204 | |
| 205 | def test_mono_spec_with_numpy_stereo(self): |
| 206 | """Mono spec should reduce stereo numpy array to 1D.""" |
| 207 | stereo = np.array([[1.0, 2.0], [-1.0, 0.0]], dtype=np.float32) |
| 208 | result = normalize_audio(stereo, MONO_AUDIO_SPEC) |
| 209 | assert result.ndim == 1 |
| 210 | np.testing.assert_array_almost_equal(result, [0.0, 1.0]) |
| 211 | |
| 212 | def test_mono_spec_with_torch_stereo(self): |
| 213 | """Mono spec should reduce stereo torch tensor to 1D.""" |
| 214 | stereo = torch.tensor([[1.0, 2.0], [-1.0, 0.0]]) |
| 215 | result = normalize_audio(stereo, MONO_AUDIO_SPEC) |
| 216 | assert result.ndim == 1 |
| 217 | torch.testing.assert_close(result, torch.tensor([0.0, 1.0])) |
| 218 | |
| 219 | def test_mono_passthrough_for_1d_numpy(self): |
| 220 | """1D numpy array should pass through unchanged with mono spec.""" |
| 221 | mono = np.array([1.0, 2.0, 3.0], dtype=np.float32) |
| 222 | result = normalize_audio(mono, MONO_AUDIO_SPEC) |
| 223 | assert result.ndim == 1 |
| 224 | np.testing.assert_array_equal(result, mono) |
| 225 | |
| 226 | def test_mono_passthrough_for_1d_torch(self): |
| 227 | """1D torch tensor should pass through unchanged with mono spec.""" |
| 228 | mono = torch.tensor([1.0, 2.0, 3.0]) |
| 229 | result = normalize_audio(mono, MONO_AUDIO_SPEC) |
| 230 | assert result.ndim == 1 |
| 231 | torch.testing.assert_close(result, mono) |
| 232 | |
| 233 | def test_first_channel_reduction(self): |
| 234 | """FIRST reduction should take only the first channel.""" |
| 235 | spec = AudioSpec(target_channels=1, channel_reduction=ChannelReduction.FIRST) |
| 236 | stereo = np.array([[1.0, 2.0], [3.0, 4.0]], dtype=np.float32) |
| 237 | result = normalize_audio(stereo, spec) |
| 238 | np.testing.assert_array_equal(result, [1.0, 2.0]) |
| 239 | |
| 240 | def test_max_channel_reduction(self): |
| 241 | """MAX reduction should take max across channels.""" |
| 242 | spec = AudioSpec(target_channels=1, channel_reduction=ChannelReduction.MAX) |
| 243 | stereo = np.array([[1.0, 4.0], [3.0, 2.0]], dtype=np.float32) |
| 244 | result = normalize_audio(stereo, spec) |
| 245 | np.testing.assert_array_equal(result, [3.0, 4.0]) |
| 246 | |
| 247 | def test_sum_channel_reduction(self): |
| 248 | """SUM reduction should sum across channels.""" |
| 249 | spec = AudioSpec(target_channels=1, channel_reduction=ChannelReduction.SUM) |
| 250 | stereo = np.array([[1.0, 2.0], [3.0, 4.0]], dtype=np.float32) |
| 251 | result = normalize_audio(stereo, spec) |
| 252 | np.testing.assert_array_equal(result, [4.0, 6.0]) |
| 253 | |
| 254 | def test_invalid_3d_array_raises(self): |
| 255 | """3D arrays should raise ValueError.""" |
| 256 | audio_3d = np.random.randn(2, 3, 4).astype(np.float32) |
| 257 | with pytest.raises(ValueError, match="Unsupported audio"): |
| 258 | normalize_audio(audio_3d, MONO_AUDIO_SPEC) |
| 259 | |
| 260 | def test_channel_expansion_raises(self): |
| 261 | """Expanding from mono to stereo should raise ValueError.""" |
| 262 | mono = np.array([1.0, 2.0, 3.0], dtype=np.float32) |
| 263 | spec = AudioSpec(target_channels=2) |
| 264 | with pytest.raises(ValueError, match="Cannot expand"): |
| 265 | normalize_audio(mono, spec) |
| 266 | |
| 267 | def test_time_channels_format_numpy(self): |
| 268 | """Audio in (time, channels) format should be transposed to (channels, time). |
| 269 | |
| 270 | This handles the case where audio loaders like soundfile return |
| 271 | (time, channels) format instead of (channels, time) like torchaudio. |
| 272 | """ |
| 273 | # Create audio in (time, channels) format: 1000 samples, 2 channels |
| 274 | audio_time_channels = np.array( |
| 275 | [[1.0, -1.0]] * 1000, # 1000 time steps, 2 channels |
| 276 | dtype=np.float32, |
| 277 | ) |
| 278 | assert audio_time_channels.shape == (1000, 2) # (time, channels) |
| 279 | |
| 280 | result = normalize_audio(audio_time_channels, MONO_AUDIO_SPEC) |
| 281 | |
| 282 | # Should be reduced to mono 1D |
| 283 | assert result.ndim == 1 |
| 284 | assert result.shape == (1000,) |
| 285 | # Mean of [1.0, -1.0] at each time step should be 0.0 |
| 286 | np.testing.assert_array_almost_equal(result, np.zeros(1000)) |
| 287 | |
| 288 | def test_time_channels_format_torch(self): |
| 289 | """Torch tensor in (time, channels) format should be transposed.""" |
| 290 | # Create audio in (time, channels) format: 1000 samples, 2 channels |
| 291 | audio_time_channels = torch.tensor( |
| 292 | [[1.0, -1.0]] * 1000, # 1000 time steps, 2 channels |
| 293 | ) |
| 294 | assert audio_time_channels.shape == (1000, 2) # (time, channels) |
| 295 | |
| 296 | result = normalize_audio(audio_time_channels, MONO_AUDIO_SPEC) |
| 297 | |
| 298 | # Should be reduced to mono 1D |
| 299 | assert result.ndim == 1 |
| 300 | assert result.shape == (1000,) |
| 301 | # Mean of [1.0, -1.0] at each time step should be 0.0 |
| 302 | torch.testing.assert_close(result, torch.zeros(1000)) |
| 303 | |
| 304 | def test_channels_time_format_preserved(self): |
| 305 | """Audio already in (channels, time) format should work correctly.""" |
| 306 | # Create audio in standard (channels, time) format: 2 channels, 1000 samples |
| 307 | audio_channels_time = np.array( |
| 308 | [[1.0] * 1000, [-1.0] * 1000], # 2 channels, 1000 time steps |
| 309 | dtype=np.float32, |
| 310 | ) |
| 311 | assert audio_channels_time.shape == (2, 1000) # (channels, time) |
| 312 | |
| 313 | result = normalize_audio(audio_channels_time, MONO_AUDIO_SPEC) |
| 314 | |
| 315 | # Should be reduced to mono 1D |
| 316 | assert result.ndim == 1 |
| 317 | assert result.shape == (1000,) |
| 318 | # Mean of [1.0, -1.0] at each time step should be 0.0 |
| 319 | np.testing.assert_array_almost_equal(result, np.zeros(1000)) |
| 320 | |
| 321 | def test_ambiguous_square_audio_numpy(self): |
| 322 | """Square audio arrays (N, N) should use shape[0] > shape[1] heuristic. |
| 323 | |
| 324 | For a square array, shape[0] == shape[1], so no transpose happens |
| 325 | and we assume (channels, time) format. |
| 326 | """ |
| 327 | # Create square audio: 4 channels, 4 samples |
| 328 | audio_square = np.array( |
| 329 | [ |
| 330 | [1.0, 2.0, 3.0, 4.0], |
| 331 | [5.0, 6.0, 7.0, 8.0], |
| 332 | [9.0, 10.0, 11.0, 12.0], |
| 333 | [13.0, 14.0, 15.0, 16.0], |
| 334 | ], |
| 335 | dtype=np.float32, |
| 336 | ) |
| 337 | assert audio_square.shape == (4, 4) |
| 338 | |
| 339 | result = normalize_audio(audio_square, MONO_AUDIO_SPEC) |
| 340 | |
| 341 | # Should be reduced to mono 1D with mean across channels (axis 0) |
| 342 | assert result.ndim == 1 |
| 343 | assert result.shape == (4,) |
| 344 | # Mean across 4 channels: [1+5+9+13, 2+6+10+14, ...] / 4 |
| 345 | expected = np.array([7.0, 8.0, 9.0, 10.0]) |
| 346 | np.testing.assert_array_almost_equal(result, expected) |
| 347 | |
| 348 | |
| 349 | # ============================================================ |
| 350 | # Tests for MultiModalDataParser integration with target_channels |
| 351 | # ============================================================ |
| 352 | |
| 353 | |
| 354 | class TestMultiModalDataParserChannelNormalization: |
| 355 | """Tests for MultiModalDataParser.target_channels integration. |
| 356 | |
| 357 | These tests verify that the target_channels parameter is properly used |
| 358 | in the _parse_audio_data method to normalize audio channels. |
| 359 | """ |
| 360 | |
| 361 | def test_parser_normalizes_stereo_to_mono(self): |
| 362 | """Parser should normalize stereo to mono when target_channels=1.""" |
| 363 | from vllm.multimodal.parse import MultiModalDataParser |
| 364 | |
| 365 | # Create parser with mono normalization enabled |
| 366 | parser = MultiModalDataParser( |
| 367 | target_sr=16000, |
| 368 | target_channels=1, |
| 369 | ) |
| 370 | |
| 371 | # Create stereo audio (simulating torchaudio output) |
| 372 | stereo_audio = np.array( |
| 373 | [[1.0, 1.0, 1.0], [-1.0, -1.0, -1.0]], # 2 channels, 3 samples |
| 374 | dtype=np.float32, |
| 375 | ) |
| 376 | |
| 377 | # Parse audio data |
| 378 | result = parser._parse_audio_data((stereo_audio, 16000)) |
| 379 | |
| 380 | # Check that result is mono (1D) |
| 381 | assert result is not None |
| 382 | audio_item = result.get(0) |
| 383 | assert audio_item.ndim == 1, f"Expected 1D mono audio, got {audio_item.ndim}D" |
| 384 | assert audio_item.shape == (3,), f"Expected shape (3,), got {audio_item.shape}" |
| 385 | # Channel average of [1, 1, 1] and [-1, -1, -1] should be [0, 0, 0] |
| 386 | np.testing.assert_array_almost_equal(audio_item, np.zeros(3)) |
| 387 | |
| 388 | def test_parser_preserves_stereo_when_target_channels_none(self): |
| 389 | """Parser should preserve stereo when target_channels=None.""" |
| 390 | from vllm.multimodal.parse import MultiModalDataParser |
| 391 | |
| 392 | # Create parser without channel normalization |
| 393 | parser = MultiModalDataParser( |
| 394 | target_sr=16000, |
| 395 | target_channels=None, |
| 396 | ) |
| 397 | |
| 398 | # Create stereo audio |
| 399 | stereo_audio = np.array( |
| 400 | [[1.0, 1.0, 1.0], [-1.0, -1.0, -1.0]], |
| 401 | dtype=np.float32, |
| 402 | ) |
| 403 | |
| 404 | # Parse audio data |
| 405 | result = parser._parse_audio_data((stereo_audio, 16000)) |
| 406 | |
| 407 | # Check that result preserves original shape (after resampling) |
| 408 | assert result is not None |
| 409 | audio_item = result.get(0) |
| 410 | # When target_channels=None, stereo audio should be preserved |
| 411 | assert audio_item.ndim == 2, f"Expected 2D stereo audio, got {audio_item.ndim}D" |
| 412 | |
| 413 | def test_parser_mono_passthrough_when_target_channels_1(self): |
| 414 | """Parser should pass through mono audio unchanged when target_channels=1.""" |
| 415 | from vllm.multimodal.parse import MultiModalDataParser |
| 416 | |
| 417 | # Create parser with mono normalization enabled |
| 418 | parser = MultiModalDataParser( |
| 419 | target_sr=16000, |
| 420 | target_channels=1, |
| 421 | ) |
| 422 | |
| 423 | # Create mono audio (already 1D) |
| 424 | mono_audio = np.random.randn(16000).astype(np.float32) |
| 425 | |
| 426 | # Parse audio data |
| 427 | result = parser._parse_audio_data((mono_audio, 16000)) |
| 428 | |
| 429 | # Check that result is still mono (1D) |
| 430 | assert result is not None |
| 431 | audio_item = result.get(0) |
| 432 | assert audio_item.ndim == 1 |
| 433 | assert audio_item.shape == (16000,) |
| 434 | |
| 435 | def test_parser_with_target_channels_2(self): |
| 436 | """Parser should reduce 6-channel to 2-channel when target_channels=2.""" |
| 437 | from vllm.multimodal.parse import MultiModalDataParser |
| 438 | |
| 439 | # Create parser with stereo target |
| 440 | parser = MultiModalDataParser( |
| 441 | target_sr=16000, |
| 442 | target_channels=2, |
| 443 | ) |
| 444 | |
| 445 | # Create 6-channel audio (5.1 surround) |
| 446 | surround_audio = np.random.randn(6, 1000).astype(np.float32) |
| 447 | |
| 448 | # Parse audio data |
| 449 | result = parser._parse_audio_data((surround_audio, 16000)) |
| 450 | |
| 451 | # Check that result is stereo (2 channels) |
| 452 | assert result is not None |
| 453 | audio_item = result.get(0) |
| 454 | assert audio_item.ndim == 2 |
| 455 | assert audio_item.shape[0] == 2 # 2 channels |
| 456 | |
| 457 | |
| 458 | # ============================================================ |
| 459 | # End-to-End Audio Pipeline Tests |
| 460 | # ============================================================ |
| 461 | |
| 462 | |
| 463 | class TestAudioPipelineE2E: |
| 464 | """End-to-end tests for audio normalization in the full pipeline. |
| 465 | |
| 466 | These tests verify the complete flow from raw audio input through |
| 467 | the MultiModalDataParser, simulating different audio loader formats. |
| 468 | """ |
| 469 | |
| 470 | def test_stereo_audio_normalized_to_mono_e2e(self): |
| 471 | """Full pipeline: stereo audio (torchaudio format) → mono output.""" |
| 472 | from vllm.multimodal.parse import MultiModalDataParser |
| 473 | |
| 474 | # Simulate torchaudio output: (channels, time) format |
| 475 | # Stereo audio with left channel = 1.0, right channel = -1.0 |
| 476 | stereo_torchaudio = np.array( |
| 477 | [[1.0] * 16000, [-1.0] * 16000], # 2 channels, 1 second at 16kHz |
| 478 | dtype=np.float32, |
| 479 | ) |
| 480 | assert stereo_torchaudio.shape == (2, 16000) |
| 481 | |
| 482 | # Create parser with mono normalization (like Whisper models) |
| 483 | parser = MultiModalDataParser( |
| 484 | target_sr=16000, |
| 485 | target_channels=1, |
| 486 | ) |
| 487 | |
| 488 | # Process audio through the parser |
| 489 | result = parser._parse_audio_data((stereo_torchaudio, 16000)) |
| 490 | assert result is not None |
| 491 | audio_output = result.get(0) |
| 492 | |
| 493 | # Verify output is mono 1D |
| 494 | assert audio_output.ndim == 1, f"Expected 1D, got {audio_output.ndim}D" |
| 495 | assert audio_output.shape == (16000,) |
| 496 | |
| 497 | # Verify channel averaging: mean of [1.0, -1.0] = 0.0 |
| 498 | np.testing.assert_array_almost_equal(audio_output, np.zeros(16000), decimal=5) |
| 499 | |
| 500 | def test_soundfile_format_normalized_to_mono_e2e(self): |
| 501 | """Full pipeline: soundfile format (time, channels) → mono output.""" |
| 502 | from vllm.multimodal.parse import MultiModalDataParser |
| 503 | |
| 504 | # Simulate soundfile output: (time, channels) format |
| 505 | # 16000 samples, 2 channels |
| 506 | stereo_soundfile = np.array( |
| 507 | [[0.5, -0.5]] * 16000, # Each row is [left, right] |
| 508 | dtype=np.float32, |
| 509 | ) |
| 510 | assert stereo_soundfile.shape == (16000, 2) |
| 511 | |
| 512 | # Create parser with mono normalization |
| 513 | parser = MultiModalDataParser( |
| 514 | target_sr=16000, |
| 515 | target_channels=1, |
| 516 | ) |
| 517 | |
| 518 | # Process audio through the parser |
| 519 | result = parser._parse_audio_data((stereo_soundfile, 16000)) |
| 520 | assert result is not None |
| 521 | audio_output = result.get(0) |
| 522 | |
| 523 | # Verify output is mono 1D |
| 524 | assert audio_output.ndim == 1, f"Expected 1D, got {audio_output.ndim}D" |
| 525 | assert audio_output.shape == (16000,) |
| 526 | |
| 527 | # Verify channel averaging: mean of [0.5, -0.5] = 0.0 |
| 528 | np.testing.assert_array_almost_equal(audio_output, np.zeros(16000), decimal=5) |
| 529 | |
| 530 | def test_pyav_mono_passthrough_e2e(self): |
| 531 | """Full pipeline: pyav mono format → preserved as mono.""" |
| 532 | from vllm.multimodal.parse import MultiModalDataParser |
| 533 | |
| 534 | # Simulate pyav output: already mono (time,) format |
| 535 | mono_pyav = np.random.randn(16000).astype(np.float32) |
| 536 | assert mono_pyav.shape == (16000,) |
| 537 | |
| 538 | # Create parser with mono normalization |
| 539 | parser = MultiModalDataParser( |
| 540 | target_sr=16000, |
| 541 | target_channels=1, |
| 542 | ) |
| 543 | |
| 544 | # Process audio through the parser |
| 545 | result = parser._parse_audio_data((mono_pyav, 16000)) |
| 546 | assert result is not None |
| 547 | audio_output = result.get(0) |
| 548 | |
| 549 | # Verify output is still mono 1D |
| 550 | assert audio_output.ndim == 1 |
| 551 | assert audio_output.shape == (16000,) |
| 552 | |
| 553 | # Verify audio content is preserved |
| 554 | np.testing.assert_array_almost_equal(audio_output, mono_pyav) |
| 555 | |
| 556 | def test_multichannel_5_1_surround_to_mono_e2e(self): |
| 557 | """Full pipeline: 5.1 surround (6 channels) → mono output.""" |
| 558 | from vllm.multimodal.parse import MultiModalDataParser |
| 559 | |
| 560 | # Simulate 5.1 surround audio: 6 channels |
| 561 | surround_audio = np.array( |
| 562 | [ |
| 563 | [1.0] * 8000, # Front Left |
| 564 | [2.0] * 8000, # Front Right |
| 565 | [3.0] * 8000, # Center |
| 566 | [4.0] * 8000, # LFE (subwoofer) |
| 567 | [5.0] * 8000, # Rear Left |
| 568 | [6.0] * 8000, # Rear Right |
| 569 | ], |
| 570 | dtype=np.float32, |
| 571 | ) |
| 572 | assert surround_audio.shape == (6, 8000) |
| 573 | |
| 574 | # Create parser with mono normalization |
| 575 | parser = MultiModalDataParser( |
| 576 | target_sr=16000, |
| 577 | target_channels=1, |
| 578 | ) |
| 579 | |
| 580 | # Process audio through the parser |
| 581 | result = parser._parse_audio_data((surround_audio, 16000)) |
| 582 | assert result is not None |
| 583 | audio_output = result.get(0) |
| 584 | |
| 585 | # Verify output is mono 1D |
| 586 | assert audio_output.ndim == 1 |
| 587 | |
| 588 | # Verify channel averaging: mean of [1,2,3,4,5,6] = 3.5 |
| 589 | expected_value = (1.0 + 2.0 + 3.0 + 4.0 + 5.0 + 6.0) / 6 |
| 590 | np.testing.assert_array_almost_equal( |
| 591 | audio_output, np.full(8000, expected_value), decimal=5 |
| 592 | ) |
| 593 | |
| 594 | def test_torch_tensor_input_e2e(self): |
| 595 | """Full pipeline: torch.Tensor stereo input → mono numpy output.""" |
| 596 | from vllm.multimodal.parse import MultiModalDataParser |
| 597 | |
| 598 | # Simulate torch tensor input (from torchaudio) |
| 599 | stereo_torch = torch.tensor( |
| 600 | [[1.0] * 8000, [-1.0] * 8000], # 2 channels |
| 601 | dtype=torch.float32, |
| 602 | ) |
| 603 | assert stereo_torch.shape == (2, 8000) |
| 604 | |
| 605 | # Create parser with mono normalization |
| 606 | parser = MultiModalDataParser( |
| 607 | target_sr=16000, |
| 608 | target_channels=1, |
| 609 | ) |
| 610 | |
| 611 | # Process audio through the parser |
| 612 | # Note: Parser expects numpy, so we convert first (simulating real usage) |
| 613 | result = parser._parse_audio_data((stereo_torch.numpy(), 16000)) |
| 614 | assert result is not None |
| 615 | audio_output = result.get(0) |
| 616 | |
| 617 | # Verify output is mono 1D numpy array |
| 618 | assert audio_output.ndim == 1 |
| 619 | assert isinstance(audio_output, np.ndarray) |
| 620 | |
| 621 | # Verify channel averaging |
| 622 | np.testing.assert_array_almost_equal(audio_output, np.zeros(8000), decimal=5) |
| 623 | |
| 624 | def test_passthrough_preserves_stereo_e2e(self): |
| 625 | """Full pipeline: stereo with target_channels=None → stereo preserved.""" |
| 626 | from vllm.multimodal.parse import MultiModalDataParser |
| 627 | |
| 628 | # Stereo audio |
| 629 | stereo_audio = np.array( |
| 630 | [[1.0] * 8000, [-1.0] * 8000], |
| 631 | dtype=np.float32, |
| 632 | ) |
| 633 | |
| 634 | # Create parser WITHOUT mono normalization (passthrough) |
| 635 | parser = MultiModalDataParser( |
| 636 | target_sr=16000, |
| 637 | target_channels=None, # Passthrough - no normalization |
| 638 | ) |
| 639 | |
| 640 | # Process audio through the parser |
| 641 | result = parser._parse_audio_data((stereo_audio, 16000)) |
| 642 | assert result is not None |
| 643 | audio_output = result.get(0) |
| 644 | |
| 645 | # Verify output preserves stereo (2D) |
| 646 | assert audio_output.ndim == 2 |
| 647 | assert audio_output.shape == (2, 8000) |
| 648 | |
| 649 | def test_resampling_with_channel_normalization_e2e(self): |
| 650 | """Full pipeline: resample + channel normalize in single pass.""" |
| 651 | from vllm.multimodal.parse import MultiModalDataParser |
| 652 | |
| 653 | # Stereo audio at 48kHz (common recording rate) |
| 654 | stereo_48k = np.array( |
| 655 | [[1.0] * 48000, [-1.0] * 48000], # 1 second at 48kHz |
| 656 | dtype=np.float32, |
| 657 | ) |
| 658 | |
| 659 | # Create parser with both resampling and mono normalization |
| 660 | parser = MultiModalDataParser( |
| 661 | target_sr=16000, # Resample to 16kHz |
| 662 | target_channels=1, # Normalize to mono |
| 663 | ) |
| 664 | |
| 665 | # Process audio through the parser |
| 666 | result = parser._parse_audio_data((stereo_48k, 48000)) |
| 667 | assert result is not None |
| 668 | audio_output = result.get(0) |
| 669 | |
| 670 | # Verify output is mono 1D at target sample rate |
| 671 | assert audio_output.ndim == 1 |
| 672 | # After resampling from 48kHz to 16kHz, length should be ~16000 |
| 673 | assert audio_output.shape[0] == 16000 |
| 674 | |
| 675 | def test_very_short_audio_e2e(self): |
| 676 | """Full pipeline: very short audio (< 1 frame) handled correctly.""" |
| 677 | from vllm.multimodal.parse import MultiModalDataParser |
| 678 | |
| 679 | # Very short stereo audio (10 samples) |
| 680 | short_stereo = np.array( |
| 681 | [[1.0] * 10, [-1.0] * 10], |
| 682 | dtype=np.float32, |
| 683 | ) |
| 684 | |
| 685 | parser = MultiModalDataParser( |
| 686 | target_sr=16000, |
| 687 | target_channels=1, |
| 688 | ) |
| 689 | |
| 690 | result = parser._parse_audio_data((short_stereo, 16000)) |
| 691 | assert result is not None |
| 692 | audio_output = result.get(0) |
| 693 | |
| 694 | # Should still produce mono output |
| 695 | assert audio_output.ndim == 1 |
| 696 | assert audio_output.shape == (10,) |
| 697 | np.testing.assert_array_almost_equal(audio_output, np.zeros(10)) |
| 698 | |
| 699 | |
| 700 | # ============================================================ |
| 701 | # Tests for Audio Chunking Utilities |
| 702 | # ============================================================ |
| 703 | |
| 704 | |
| 705 | class TestAudioChunking: |
| 706 | """Tests for split_audio and find_split_point utilities in vllm.multimodal.audio.""" |
| 707 | |
| 708 | def test_split_audio_short_clip(self): |
| 709 | """Audio shorter than max_clip_duration_s should not be split.""" |
| 710 | # 10 seconds of audio at 16kHz |
| 711 | audio = np.linspace(-1.0, 1.0, 160000, dtype=np.float32) |
| 712 | |
| 713 | chunks = split_audio( |
| 714 | audio_data=audio, |
| 715 | sample_rate=16000, |
| 716 | max_clip_duration_s=30.0, |
| 717 | overlap_duration_s=1.0, |
| 718 | min_energy_window_size=1600, |
| 719 | ) |
| 720 | |
| 721 | assert len(chunks) == 1 |
| 722 | np.testing.assert_array_equal(chunks[0], audio) |
| 723 | |
| 724 | def test_split_audio_exact_length(self): |
| 725 | """Audio exactly at max_clip_duration_s should not be split.""" |
| 726 | # Exactly 30 seconds at 16kHz |
| 727 | audio = np.linspace(-1.0, 1.0, 480000, dtype=np.float32) |
| 728 | |
| 729 | chunks = split_audio( |
| 730 | audio_data=audio, |
| 731 | sample_rate=16000, |
| 732 | max_clip_duration_s=30.0, |
| 733 | overlap_duration_s=1.0, |
| 734 | min_energy_window_size=1600, |
| 735 | ) |
| 736 | |
| 737 | assert len(chunks) == 1 |
| 738 | np.testing.assert_array_equal(chunks[0], audio) |
| 739 | |
| 740 | def test_split_audio_long_clip(self): |
| 741 | """Long audio should be split into multiple chunks.""" |
| 742 | # 65 seconds of audio at 16kHz |
| 743 | audio = np.linspace(-1.0, 1.0, 1040000, dtype=np.float32) |
| 744 | |
| 745 | chunks = split_audio( |
| 746 | audio_data=audio, |
| 747 | sample_rate=16000, |
| 748 | max_clip_duration_s=30.0, |
| 749 | overlap_duration_s=1.0, |
| 750 | min_energy_window_size=1600, |
| 751 | ) |
| 752 | |
| 753 | assert len(chunks) > 1 |
| 754 | # First sample preserved |
| 755 | assert chunks[0][0] == audio[0] |
| 756 | # Last sample preserved |
| 757 | assert chunks[-1][-1] == audio[-1] |
| 758 | |
| 759 | def test_split_audio_chunks_have_correct_length(self): |
| 760 | """Each chunk (except last) should be approximately max_clip_duration_s.""" |
| 761 | # 65 seconds of audio at 16kHz |
| 762 | audio = np.linspace(-1.0, 1.0, 1040000, dtype=np.float32) |
| 763 | |
| 764 | chunks = split_audio( |
| 765 | audio_data=audio, |
| 766 | sample_rate=16000, |
| 767 | max_clip_duration_s=30.0, |
| 768 | overlap_duration_s=1.0, |
| 769 | min_energy_window_size=1600, |
| 770 | ) |
| 771 | |
| 772 | max_samples = int(30.0 * 16000) |
| 773 | overlap_samples = int(1.0 * 16000) |
| 774 | |
| 775 | for chunk in chunks[:-1]: |
| 776 | assert chunk.shape[0] >= max_samples - overlap_samples |
| 777 | assert chunk.shape[0] <= max_samples |
| 778 | |
| 779 | def test_find_split_point_finds_quiet_region(self): |
| 780 | """find_split_point should identify low-energy regions.""" |
| 781 | from vllm.multimodal.audio import find_split_point |
| 782 | |
| 783 | # Create audio with a quiet section in the middle |
| 784 | segment = np.ones(32000, dtype=np.float32) |
| 785 | # Insert quiet region at sample 16000-17600 (100ms) |
| 786 | segment[16000:17600] = 0.01 |
| 787 | |
| 788 | split_idx = find_split_point( |
| 789 | wav=segment, |
| 790 | start_idx=0, |
| 791 | end_idx=32000, |
| 792 | min_energy_window=1600, |
| 793 | ) |
| 794 | |
| 795 | # Split should be in or near the quiet region |
| 796 | assert 16000 <= split_idx <= 17600 |
| 797 | |
| 798 | def test_find_split_point_handles_uniform_audio(self): |
| 799 | """find_split_point should handle uniform energy audio gracefully.""" |
| 800 | from vllm.multimodal.audio import find_split_point |
| 801 | |
| 802 | segment = np.ones(32000, dtype=np.float32) * 0.5 |
| 803 | |
| 804 | split_idx = find_split_point( |
| 805 | wav=segment, |
| 806 | start_idx=0, |
| 807 | end_idx=32000, |
| 808 | min_energy_window=1600, |
| 809 | ) |
| 810 | |
| 811 | assert 0 <= split_idx <= 32000 |
| 812 | |
| 813 | def test_find_split_point_silence(self): |
| 814 | """find_split_point should prefer the quietest scanned window.""" |
| 815 | from vllm.multimodal.audio import find_split_point |
| 816 | |
| 817 | # Deterministic signal: constant energy everywhere except silence. |
| 818 | segment = np.ones(32000, dtype=np.float32) |
| 819 | # Complete silence at 20000-21600. |
| 820 | segment[20000:21600] = 0.0 |
| 821 | |
| 822 | split_idx = find_split_point( |
| 823 | wav=segment, |
| 824 | start_idx=16000, |
| 825 | end_idx=28000, |
| 826 | min_energy_window=1600, |
| 827 | ) |
| 828 | |
| 829 | # Current implementation evaluates non-overlapping 1600-sample windows |
| 830 | # from start_idx, so the quietest scanned window starts at 19200. |
| 831 | assert split_idx == 19200 |
| 832 | |
| 833 | def test_split_audio_rejects_multi_channel(self): |
| 834 | """Chunking is mono-only; stereo must fail loudly rather than silently. |
| 835 | |
| 836 | find_split_point searches axis 0, so (channels, time) input would skip |
| 837 | the energy search entirely and split at fixed boundaries instead. |
| 838 | """ |
| 839 | stereo = np.ones((2, 16000 * 65), dtype=np.float32) |
| 840 | |
| 841 | with pytest.raises(ValueError, match="expects mono audio"): |
| 842 | split_audio( |
| 843 | audio_data=stereo, |
| 844 | sample_rate=16000, |
| 845 | max_clip_duration_s=30.0, |
| 846 | overlap_duration_s=1.0, |
| 847 | min_energy_window_size=1600, |
| 848 | ) |
| 849 | |
| 850 | def test_split_audio_preserves_boundaries(self): |
| 851 | """Verify first and last samples are preserved when chunking.""" |
| 852 | audio = np.arange(1120000, dtype=np.float32) # 70s at 16kHz |
| 853 | |
| 854 | chunks = split_audio( |
| 855 | audio_data=audio, |
| 856 | sample_rate=16000, |
| 857 | max_clip_duration_s=30.0, |
| 858 | overlap_duration_s=1.0, |
| 859 | min_energy_window_size=1600, |
| 860 | ) |
| 861 | |
| 862 | assert chunks[0][0] == audio[0] |
| 863 | assert chunks[-1][-1] == audio[-1] |
| 864 | |
| 865 | def test_find_split_point_nan_input(self): |
| 866 | """find_split_point must not return 0 for all-NaN input.""" |
| 867 | from vllm.multimodal.audio import find_split_point |
| 868 | |
| 869 | nan_audio = np.full(32000, float("nan"), dtype=np.float32) |
| 870 | start_idx = 16000 |
| 871 | end_idx = 32000 |
| 872 | |
| 873 | split_idx = find_split_point( |
| 874 | wav=nan_audio, |
| 875 | start_idx=start_idx, |
| 876 | end_idx=end_idx, |
| 877 | min_energy_window=1600, |
| 878 | ) |
| 879 | |
| 880 | # Must return start_idx (the safe fallback), not 0 |
| 881 | assert split_idx == start_idx |
| 882 | |
| 883 | def test_split_audio_nan_input_terminates(self): |
| 884 | """split_audio must terminate on all-NaN audio (no infinite loop).""" |
| 885 | # 31 seconds of NaN at 16kHz — longer than max_clip_duration_s |
| 886 | nan_audio = np.full(16000 * 31, float("nan"), dtype=np.float32) |
| 887 | |
| 888 | chunks = split_audio( |
| 889 | audio_data=nan_audio, |
| 890 | sample_rate=16000, |
| 891 | max_clip_duration_s=30.0, |
| 892 | overlap_duration_s=1.0, |
| 893 | min_energy_window_size=1600, |
| 894 | ) |
| 895 | |
| 896 | # Must produce at least 2 chunks and cover all samples |
| 897 | assert len(chunks) >= 2 |
| 898 | total_samples = sum(c.shape[-1] for c in chunks) |
| 899 | assert total_samples == nan_audio.shape[-1] |
| 900 | |
| 901 | def test_split_audio_with_different_sample_rates(self): |
| 902 | """Test chunking works with different sample rates.""" |
| 903 | # 40 seconds at 8kHz |
| 904 | audio_8k = np.linspace(-1.0, 1.0, 320000, dtype=np.float32) |
| 905 | |
| 906 | chunks = split_audio( |
| 907 | audio_data=audio_8k, |
| 908 | sample_rate=8000, |
| 909 | max_clip_duration_s=30.0, |
| 910 | overlap_duration_s=1.0, |
| 911 | min_energy_window_size=800, |
| 912 | ) |
| 913 | |
| 914 | assert len(chunks) >= 2 |
| 915 | |
| 916 | # 40 seconds at 48kHz |
| 917 | audio_48k = np.linspace(-1.0, 1.0, 1920000, dtype=np.float32) |
| 918 | |
| 919 | chunks_48k = split_audio( |
| 920 | audio_data=audio_48k, |
| 921 | sample_rate=48000, |
| 922 | max_clip_duration_s=30.0, |
| 923 | overlap_duration_s=1.0, |
| 924 | min_energy_window_size=4800, |
| 925 | ) |
| 926 | |
| 927 | assert len(chunks_48k) >= 2 |