@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 657e9c0e18

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 10, 10:15 UTC

[Fix] Introduce audio channels spec (#31595)

Signed-off-by: Jeremy Teboul <jeremyte@meta.com>

Jeremy Teboul committed Jan 9, 2026 at 11:34 UTC 657e9c0e183d4bdb157b07db987644d2d9d36245

9 files changed +717 -189

docs/features/multimodal_inputs.md

+38

@@ -356,6 +356,44 @@ You can pass a tuple `(array, sampling_rate)` to the `'audio'` field of the mult

356

357 Full example: [examples/offline_inference/audio_language.py](../../examples/offline_inference/audio_language.py)

358

359 +#### Automatic Audio Channel Normalization

360 +

361 +vLLM automatically normalizes audio channels for models that require specific audio formats. When loading audio with libraries like `torchaudio`, stereo files return shape `[channels, time]`, but many audio models (particularly Whisper-based models) expect mono audio with shape `[time]`.

362 +

363 +**Supported models with automatic mono conversion:**

364 +

365 +- **Whisper** and all Whisper-based models

366 +- **Qwen2-Audio**

367 +- **Qwen2.5-Omni** / **Qwen3-Omni** (inherits from Qwen2.5-Omni)

368 +- **Ultravox**

369 +

370 +For these models, vLLM automatically:

371 +

372 +1. Detects if the model requires mono audio via the feature extractor

373 +2. Converts multi-channel audio to mono using channel averaging

374 +3. Handles both `(channels, time)` format (torchaudio) and `(time, channels)` format (soundfile)

375 +

376 +**Example with stereo audio:**

377 +

378 +```python

379 +import torchaudio

380 +from vllm import LLM

381 +

382 +# Load stereo audio file - returns (channels, time) shape

383 +audio, sr = torchaudio.load("stereo_audio.wav")

384 +print(f"Original shape: {audio.shape}") # e.g., torch.Size([2, 16000])

385 +

386 +# vLLM automatically converts to mono for Whisper-based models

387 +llm = LLM(model="openai/whisper-large-v3")

388 +

389 +outputs = llm.generate({

390 + "prompt": "",

391 + "multi_modal_data": {"audio": (audio.numpy(), sr)},

392 +})

393 +```

394 +

395 +No manual conversion is needed - vLLM handles the channel normalization automatically based on the model's requirements.

396 +

397 ### Embedding Inputs

398

399 To input pre-computed embeddings belonging to a data type (i.e. image, video, or audio) directly to the language model,

tests/models/multimodal/processing/test_qwen3_omni.py

+4 -185

@@ -28,10 +28,9 @@ def test_processor_with_audio_sample_rate(

28 """

29 Test that vLLM's processor generates expected outputs with audio_sample_rate.

31 - This validates the reviewer's request that we test the actual processor

32 - can handle different audio_sample_rate values and generate audio tokens.

31 + This validates that the processor correctly handles audio_sample_rate

32 + passed via hf_processor_mm_kwargs and generates audio tokens.

33 """

34 - # Setup: Build model context and processor

34 ctx = build_model_context(

35 model_id,

36 limit_mm_per_prompt={"audio": 1, "image": 0, "video": 0},

@@ -48,18 +47,17 @@ def test_processor_with_audio_sample_rate(

47 prompt = "<|audio_start|><|audio_pad|><|audio_end|>"

48 mm_data = {"audio": [(audio_data, audio_sample_rate)]}

51 - # Execute: Apply processor with audio_sample_rate in mm_kwargs

50 + # Apply processor with audio_sample_rate in mm_kwargs

51 hf_processor_mm_kwargs: dict[str, Any] = {

52 "audio_sample_rate": audio_sample_rate,

53 }

54 processed_inputs = processor.apply(prompt, mm_data, hf_processor_mm_kwargs)

57 - # Assert: Verify audio tokens are generated

56 + # Verify audio tokens are generated

57 hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs)

58 audio_token_id = tokenizer.convert_tokens_to_ids(hf_processor.audio_token)

59 aud_tok_count = processed_inputs["prompt_token_ids"].count(audio_token_id)

62 - # Audio should generate at least 1 token

61 assert aud_tok_count >= 1, (

62 f"Expected at least 1 audio token but got {aud_tok_count}. "

63 f"sample_rate: {audio_sample_rate}Hz, duration: {audio_duration_sec}s"

@@ -97,189 +95,10 @@ def test_longer_audio_generates_more_tokens(model_id: str) -> None:

95 audio_token_id = tokenizer.convert_tokens_to_ids(hf_proc.audio_token)

96 return processed["prompt_token_ids"].count(audio_token_id)

100 - # Get token counts for different durations

98 short_tokens = get_token_count(1.0)

99 long_tokens = get_token_count(2.0)

100

104 - # Longer audio should produce more tokens

101 assert long_tokens > short_tokens, (

102 f"Expected longer audio (2s) to have more tokens than shorter (1s). "

103 f"Got short={short_tokens}, long={long_tokens}"

104 )

109 -

110 -

111 -class TestQwen3OmniAudioSampleRatePreservation:

112 - """Test that audio_sample_rate is preserved during kwargs restructuring.

113 -

114 - These tests validate the fix for the audio_sample_rate bug in Qwen3 Omni

115 - where the parameter was lost during kwargs restructuring.

116 - """

117 -

118 - @staticmethod

119 - def _process_kwargs(

120 - mm_kwargs: dict[str, Any],

121 - tok_kwargs: dict[str, Any],

122 - transformers_version: str = "4.57.0",

123 - ) -> dict[str, Any]:

124 - """

125 - Helper method to simulate kwargs processing logic from production code.

126 -

127 - This method simulates the kwargs restructuring that happens in the

128 - Qwen3 Omni model when transformers < 4.58.0. By centralizing this

129 - logic, we make tests easier to maintain if the production logic changes.

130 -

131 - Args:

132 - mm_kwargs: Multimodal kwargs (e.g., audio_sample_rate, truncation)

133 - tok_kwargs: Tokenizer kwargs (e.g., truncation)

134 - transformers_version: Version string to test against (default: "4.57.0")

135 -

136 - Returns:

137 - Processed kwargs dictionary with restructured audio_kwargs and text_kwargs

138 - """

139 - from packaging.version import Version

140 -

141 - mm_kwargs_copy = dict(mm_kwargs)

142 - tok_kwargs_copy = dict(tok_kwargs)

143 -

144 - if Version(transformers_version) < Version("4.58.0"):

145 - # Extract audio_sample_rate before restructuring (THE FIX)

146 - audio_sample_rate = mm_kwargs_copy.pop("audio_sample_rate", None)

147 -

148 - # Restructure kwargs

149 - mm_kwargs_copy["audio_kwargs"] = {

150 - "truncation": mm_kwargs_copy.pop("truncation", False)

151 - }

152 - mm_kwargs_copy["text_kwargs"] = {

153 - "truncation": tok_kwargs_copy.pop("truncation", False)

154 - }

155 -

156 - # Put audio_sample_rate into audio_kwargs (THE FIX)

157 - if audio_sample_rate is not None:

158 - mm_kwargs_copy["audio_kwargs"]["audio_sample_rate"] = audio_sample_rate

159 -

160 - return mm_kwargs_copy

161 -

162 - def test_audio_sample_rate_preserved_in_audio_kwargs(self) -> None:

163 - """

164 - Test that audio_sample_rate is moved from top-level mm_kwargs

165 - into audio_kwargs during kwargs restructuring.

166 -

167 - This is the core fix: when transformers < 4.58.0, the code

168 - restructures kwargs into audio_kwargs and text_kwargs, and

169 - audio_sample_rate must be preserved in audio_kwargs.

170 - """

171 - # Setup: Create mm_kwargs with audio_sample_rate at top level

172 - mm_kwargs: dict[str, Any] = {

173 - "audio_sample_rate": 16000,

174 - "truncation": True,

175 - }

176 - tok_kwargs: dict[str, Any] = {

177 - "truncation": False,

178 - }

179 -

180 - # Execute: Process kwargs using helper method

181 - result = self._process_kwargs(mm_kwargs, tok_kwargs)

182 -

183 - # Assert: Verify audio_sample_rate is in audio_kwargs

184 - assert "audio_kwargs" in result

185 - assert "audio_sample_rate" in result["audio_kwargs"]

186 - assert result["audio_kwargs"]["audio_sample_rate"] == 16000

187 -

188 - # Assert: Verify truncation is also in audio_kwargs

189 - assert result["audio_kwargs"]["truncation"] is True

190 -

191 - # Assert: Verify text_kwargs is created correctly

192 - assert "text_kwargs" in result

193 - assert result["text_kwargs"]["truncation"] is False

194 -

195 - def test_audio_sample_rate_absent_when_not_provided(self) -> None:

196 - """

197 - Test that when audio_sample_rate is not provided in mm_kwargs,

198 - the restructured audio_kwargs doesn't contain it.

199 - """

200 - # Setup: Create mm_kwargs WITHOUT audio_sample_rate

201 - mm_kwargs: dict[str, Any] = {

202 - "truncation": True,

203 - }

204 - tok_kwargs: dict[str, Any] = {

205 - "truncation": False,

206 - }

207 -

208 - # Execute: Process kwargs using helper method

209 - result = self._process_kwargs(mm_kwargs, tok_kwargs)

210 -

211 - # Assert: Verify audio_sample_rate is NOT in audio_kwargs

212 - assert "audio_kwargs" in result

213 - assert "audio_sample_rate" not in result["audio_kwargs"]

214 -

215 - # Assert: Verify truncation is still in audio_kwargs

216 - assert result["audio_kwargs"]["truncation"] is True

217 -

218 - @pytest.mark.parametrize("sample_rate", [8000, 16000, 22050, 24000, 44100, 48000])

219 - def test_various_audio_sample_rates_preserved(self, sample_rate: int) -> None:

220 - """

221 - Test that various common audio sample rates are preserved.

222 -

223 - Common sample rates:

224 - - 8000: Telephone quality

225 - - 16000: Wideband speech (Qwen3 Omni default)

226 - - 22050: Low-quality audio

227 - - 24000: High-quality speech

228 - - 44100: CD quality

229 - - 48000: Professional audio

230 - """

231 - # Setup: Create mm_kwargs with specific sample rate

232 - mm_kwargs: dict[str, Any] = {

233 - "audio_sample_rate": sample_rate,

234 - "truncation": True,

235 - }

236 - tok_kwargs: dict[str, Any] = {"truncation": False}

237 -

238 - # Execute: Process kwargs using helper method

239 - result = self._process_kwargs(mm_kwargs, tok_kwargs)

240 -

241 - # Assert: Verify the specific sample rate is preserved

242 - assert result["audio_kwargs"]["audio_sample_rate"] == sample_rate

243 -

244 - def test_kwargs_unchanged_for_newer_transformers_version(self) -> None:

245 - """

246 - Test that kwargs structure remains unchanged for transformers >= 4.58.0.

247 -

248 - This test ensures that when transformers version is 4.58.0 or higher,

249 - the kwargs restructuring is bypassed and audio_sample_rate remains

250 - at the top level as originally passed.

251 - """

252 - from packaging.version import Version

253 -

254 - # Setup: Create mm_kwargs with audio_sample_rate at top level

255 - mm_kwargs: dict[str, Any] = {

256 - "audio_sample_rate": 16000,

257 - "truncation": True,

258 - }

259 - tok_kwargs: dict[str, Any] = {

260 - "truncation": False,

261 - }

262 -

263 - # Execute: Simulate with transformers >= 4.58.0

264 - mm_kwargs_copy = dict(mm_kwargs)

265 - tok_kwargs_copy = dict(tok_kwargs)

266 -

267 - transformers_ver = "4.58.0" # Version that bypasses restructuring

268 - if Version(transformers_ver) < Version("4.58.0"):

269 - # This block should NOT execute for >= 4.58.0

270 - audio_sample_rate = mm_kwargs_copy.pop("audio_sample_rate", None)

271 - mm_kwargs_copy["audio_kwargs"] = {

272 - "truncation": mm_kwargs_copy.pop("truncation", False)

273 - }

274 - mm_kwargs_copy["text_kwargs"] = {

275 - "truncation": tok_kwargs_copy.pop("truncation", False)

276 - }

277 - if audio_sample_rate is not None:

278 - mm_kwargs_copy["audio_kwargs"]["audio_sample_rate"] = audio_sample_rate

279 -

280 - # Assert: Verify kwargs structure is unchanged

281 - assert "audio_kwargs" not in mm_kwargs_copy

282 - assert "text_kwargs" not in mm_kwargs_copy

283 - assert mm_kwargs_copy["audio_sample_rate"] == 16000

284 - assert mm_kwargs_copy["truncation"] is True

285 - assert tok_kwargs_copy["truncation"] is False

tests/multimodal/test_audio.py

+503

@@ -7,10 +7,16 @@ from unittest.mock import patch

8 import numpy as np

9 import pytest

10 +import torch

12 from vllm.multimodal.audio import (

13 + MONO_AUDIO_SPEC,

14 + PASSTHROUGH_AUDIO_SPEC,

15 AudioMediaIO,

16 AudioResampler,

17 + AudioSpec,

18 + ChannelReduction,

19 + normalize_audio,

20 resample_audio_librosa,

21 resample_audio_scipy,

22 )

@@ -137,3 +143,500 @@ def test_audio_media_io_encode_base64(dummy_audio):

143 decoded = base64.b64decode(out)

144 assert decoded == b"dummy_wav_data"

145 mock_write.assert_called_once()

146 +

147 +

148 +# ============================================================

149 +# Tests for normalize_audio function

150 +# ============================================================

151 +

152 +

153 +class TestNormalizeAudio:

154 + """Tests for normalize_audio function with different specs."""

155 +

156 + def test_passthrough_preserves_audio(self):

157 + """Passthrough spec should not modify audio."""

158 + stereo = np.array([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]], dtype=np.float32)

159 + result = normalize_audio(stereo, PASSTHROUGH_AUDIO_SPEC)

160 + np.testing.assert_array_equal(result, stereo)

161 +

162 + def test_mono_spec_with_numpy_stereo(self):

163 + """Mono spec should reduce stereo numpy array to 1D."""

164 + stereo = np.array([[1.0, 2.0], [-1.0, 0.0]], dtype=np.float32)

165 + result = normalize_audio(stereo, MONO_AUDIO_SPEC)

166 + assert result.ndim == 1

167 + np.testing.assert_array_almost_equal(result, [0.0, 1.0])

168 +

169 + def test_mono_spec_with_torch_stereo(self):

170 + """Mono spec should reduce stereo torch tensor to 1D."""

171 + stereo = torch.tensor([[1.0, 2.0], [-1.0, 0.0]])

172 + result = normalize_audio(stereo, MONO_AUDIO_SPEC)

173 + assert result.ndim == 1

174 + torch.testing.assert_close(result, torch.tensor([0.0, 1.0]))

175 +

176 + def test_mono_passthrough_for_1d_numpy(self):

177 + """1D numpy array should pass through unchanged with mono spec."""

178 + mono = np.array([1.0, 2.0, 3.0], dtype=np.float32)

179 + result = normalize_audio(mono, MONO_AUDIO_SPEC)

180 + assert result.ndim == 1

181 + np.testing.assert_array_equal(result, mono)

182 +

183 + def test_mono_passthrough_for_1d_torch(self):

184 + """1D torch tensor should pass through unchanged with mono spec."""

185 + mono = torch.tensor([1.0, 2.0, 3.0])

186 + result = normalize_audio(mono, MONO_AUDIO_SPEC)

187 + assert result.ndim == 1

188 + torch.testing.assert_close(result, mono)

189 +

190 + def test_first_channel_reduction(self):

191 + """FIRST reduction should take only the first channel."""

192 + spec = AudioSpec(target_channels=1, channel_reduction=ChannelReduction.FIRST)

193 + stereo = np.array([[1.0, 2.0], [3.0, 4.0]], dtype=np.float32)

194 + result = normalize_audio(stereo, spec)

195 + np.testing.assert_array_equal(result, [1.0, 2.0])

196 +

197 + def test_max_channel_reduction(self):

198 + """MAX reduction should take max across channels."""

199 + spec = AudioSpec(target_channels=1, channel_reduction=ChannelReduction.MAX)

200 + stereo = np.array([[1.0, 4.0], [3.0, 2.0]], dtype=np.float32)

201 + result = normalize_audio(stereo, spec)

202 + np.testing.assert_array_equal(result, [3.0, 4.0])

203 +

204 + def test_sum_channel_reduction(self):

205 + """SUM reduction should sum across channels."""

206 + spec = AudioSpec(target_channels=1, channel_reduction=ChannelReduction.SUM)

207 + stereo = np.array([[1.0, 2.0], [3.0, 4.0]], dtype=np.float32)

208 + result = normalize_audio(stereo, spec)

209 + np.testing.assert_array_equal(result, [4.0, 6.0])

210 +

211 + def test_invalid_3d_array_raises(self):

212 + """3D arrays should raise ValueError."""

213 + audio_3d = np.random.randn(2, 3, 4).astype(np.float32)

214 + with pytest.raises(ValueError, match="Unsupported audio"):

215 + normalize_audio(audio_3d, MONO_AUDIO_SPEC)

216 +

217 + def test_channel_expansion_raises(self):

218 + """Expanding from mono to stereo should raise ValueError."""

219 + mono = np.array([1.0, 2.0, 3.0], dtype=np.float32)

220 + spec = AudioSpec(target_channels=2)

221 + with pytest.raises(ValueError, match="Cannot expand"):

222 + normalize_audio(mono, spec)

223 +

224 + def test_time_channels_format_numpy(self):

225 + """Audio in (time, channels) format should be transposed to (channels, time).

226 +

227 + This handles the case where audio loaders like soundfile return

228 + (time, channels) format instead of (channels, time) like torchaudio.

229 + """

230 + # Create audio in (time, channels) format: 1000 samples, 2 channels

231 + audio_time_channels = np.array(

232 + [[1.0, -1.0]] * 1000, # 1000 time steps, 2 channels

233 + dtype=np.float32,

234 + )

235 + assert audio_time_channels.shape == (1000, 2) # (time, channels)

236 +

237 + result = normalize_audio(audio_time_channels, MONO_AUDIO_SPEC)

238 +

239 + # Should be reduced to mono 1D

240 + assert result.ndim == 1

241 + assert result.shape == (1000,)

242 + # Mean of [1.0, -1.0] at each time step should be 0.0

243 + np.testing.assert_array_almost_equal(result, np.zeros(1000))

244 +

245 + def test_time_channels_format_torch(self):

246 + """Torch tensor in (time, channels) format should be transposed."""

247 + # Create audio in (time, channels) format: 1000 samples, 2 channels

248 + audio_time_channels = torch.tensor(

249 + [[1.0, -1.0]] * 1000, # 1000 time steps, 2 channels

250 + )

251 + assert audio_time_channels.shape == (1000, 2) # (time, channels)

252 +

253 + result = normalize_audio(audio_time_channels, MONO_AUDIO_SPEC)

254 +

255 + # Should be reduced to mono 1D

256 + assert result.ndim == 1

257 + assert result.shape == (1000,)

258 + # Mean of [1.0, -1.0] at each time step should be 0.0

259 + torch.testing.assert_close(result, torch.zeros(1000))

260 +

261 + def test_channels_time_format_preserved(self):

262 + """Audio already in (channels, time) format should work correctly."""

263 + # Create audio in standard (channels, time) format: 2 channels, 1000 samples

264 + audio_channels_time = np.array(

265 + [[1.0] * 1000, [-1.0] * 1000], # 2 channels, 1000 time steps

266 + dtype=np.float32,

267 + )

268 + assert audio_channels_time.shape == (2, 1000) # (channels, time)

269 +

270 + result = normalize_audio(audio_channels_time, MONO_AUDIO_SPEC)

271 +

272 + # Should be reduced to mono 1D

273 + assert result.ndim == 1

274 + assert result.shape == (1000,)

275 + # Mean of [1.0, -1.0] at each time step should be 0.0

276 + np.testing.assert_array_almost_equal(result, np.zeros(1000))

277 +

278 + def test_ambiguous_square_audio_numpy(self):

279 + """Square audio arrays (N, N) should use shape[0] > shape[1] heuristic.

280 +

281 + For a square array, shape[0] == shape[1], so no transpose happens

282 + and we assume (channels, time) format.

283 + """

284 + # Create square audio: 4 channels, 4 samples

285 + audio_square = np.array(

286 + [

287 + [1.0, 2.0, 3.0, 4.0],

288 + [5.0, 6.0, 7.0, 8.0],

289 + [9.0, 10.0, 11.0, 12.0],

290 + [13.0, 14.0, 15.0, 16.0],

291 + ],

292 + dtype=np.float32,

293 + )

294 + assert audio_square.shape == (4, 4)

295 +

296 + result = normalize_audio(audio_square, MONO_AUDIO_SPEC)

297 +

298 + # Should be reduced to mono 1D with mean across channels (axis 0)

299 + assert result.ndim == 1

300 + assert result.shape == (4,)

301 + # Mean across 4 channels: [1+5+9+13, 2+6+10+14, ...] / 4

302 + expected = np.array([7.0, 8.0, 9.0, 10.0])

303 + np.testing.assert_array_almost_equal(result, expected)

304 +

305 +

306 +# ============================================================

307 +# Tests for MultiModalDataParser integration with target_channels

308 +# ============================================================

309 +

310 +

311 +class TestMultiModalDataParserChannelNormalization:

312 + """Tests for MultiModalDataParser.target_channels integration.

313 +

314 + These tests verify that the target_channels parameter is properly used

315 + in the _parse_audio_data method to normalize audio channels.

316 + """

317 +

318 + def test_parser_normalizes_stereo_to_mono(self):

319 + """Parser should normalize stereo to mono when target_channels=1."""

320 + from vllm.multimodal.parse import MultiModalDataParser

321 +

322 + # Create parser with mono normalization enabled

323 + parser = MultiModalDataParser(

324 + target_sr=16000,

325 + target_channels=1,

326 + )

327 +

328 + # Create stereo audio (simulating torchaudio output)

329 + stereo_audio = np.array(

330 + [[1.0, 1.0, 1.0], [-1.0, -1.0, -1.0]], # 2 channels, 3 samples

331 + dtype=np.float32,

332 + )

333 +

334 + # Parse audio data

335 + result = parser._parse_audio_data((stereo_audio, 16000))

336 +

337 + # Check that result is mono (1D)

338 + audio_item = result.get(0)

339 + assert audio_item.ndim == 1, f"Expected 1D mono audio, got {audio_item.ndim}D"

340 + assert audio_item.shape == (3,), f"Expected shape (3,), got {audio_item.shape}"

341 + # Channel average of [1, 1, 1] and [-1, -1, -1] should be [0, 0, 0]

342 + np.testing.assert_array_almost_equal(audio_item, np.zeros(3))

343 +

344 + def test_parser_preserves_stereo_when_target_channels_none(self):

345 + """Parser should preserve stereo when target_channels=None."""

346 + from vllm.multimodal.parse import MultiModalDataParser

347 +

348 + # Create parser without channel normalization

349 + parser = MultiModalDataParser(

350 + target_sr=16000,

351 + target_channels=None,

352 + )

353 +

354 + # Create stereo audio

355 + stereo_audio = np.array(

356 + [[1.0, 1.0, 1.0], [-1.0, -1.0, -1.0]],

357 + dtype=np.float32,

358 + )

359 +

360 + # Parse audio data

361 + result = parser._parse_audio_data((stereo_audio, 16000))

362 +

363 + # Check that result preserves original shape (after resampling)

364 + audio_item = result.get(0)

365 + # When target_channels=None, stereo audio should be preserved

366 + assert audio_item.ndim == 2, f"Expected 2D stereo audio, got {audio_item.ndim}D"

367 +

368 + def test_parser_mono_passthrough_when_target_channels_1(self):

369 + """Parser should pass through mono audio unchanged when target_channels=1."""

370 + from vllm.multimodal.parse import MultiModalDataParser

371 +

372 + # Create parser with mono normalization enabled

373 + parser = MultiModalDataParser(

374 + target_sr=16000,

375 + target_channels=1,

376 + )

377 +

378 + # Create mono audio (already 1D)

379 + mono_audio = np.random.randn(16000).astype(np.float32)

380 +

381 + # Parse audio data

382 + result = parser._parse_audio_data((mono_audio, 16000))

383 +

384 + # Check that result is still mono (1D)

385 + audio_item = result.get(0)

386 + assert audio_item.ndim == 1

387 + assert audio_item.shape == (16000,)

388 +

389 + def test_parser_with_target_channels_2(self):

390 + """Parser should reduce 6-channel to 2-channel when target_channels=2."""

391 + from vllm.multimodal.parse import MultiModalDataParser

392 +

393 + # Create parser with stereo target

394 + parser = MultiModalDataParser(

395 + target_sr=16000,

396 + target_channels=2,

397 + )

398 +

399 + # Create 6-channel audio (5.1 surround)

400 + surround_audio = np.random.randn(6, 1000).astype(np.float32)

401 +

402 + # Parse audio data

403 + result = parser._parse_audio_data((surround_audio, 16000))

404 +

405 + # Check that result is stereo (2 channels)

406 + audio_item = result.get(0)

407 + assert audio_item.ndim == 2

408 + assert audio_item.shape[0] == 2 # 2 channels

409 +

410 +

411 +# ============================================================

412 +# End-to-End Audio Pipeline Tests

413 +# ============================================================

414 +

415 +

416 +class TestAudioPipelineE2E:

417 + """End-to-end tests for audio normalization in the full pipeline.

418 +

419 + These tests verify the complete flow from raw audio input through

420 + the MultiModalDataParser, simulating different audio loader formats.

421 + """

422 +

423 + def test_stereo_audio_normalized_to_mono_e2e(self):

424 + """Full pipeline: stereo audio (torchaudio format) → mono output."""

425 + from vllm.multimodal.parse import MultiModalDataParser

426 +

427 + # Simulate torchaudio output: (channels, time) format

428 + # Stereo audio with left channel = 1.0, right channel = -1.0

429 + stereo_torchaudio = np.array(

430 + [[1.0] * 16000, [-1.0] * 16000], # 2 channels, 1 second at 16kHz

431 + dtype=np.float32,

432 + )

433 + assert stereo_torchaudio.shape == (2, 16000)

434 +

435 + # Create parser with mono normalization (like Whisper models)

436 + parser = MultiModalDataParser(

437 + target_sr=16000,

438 + target_channels=1,

439 + )

440 +

441 + # Process audio through the parser

442 + result = parser._parse_audio_data((stereo_torchaudio, 16000))

443 + audio_output = result.get(0)

444 +

445 + # Verify output is mono 1D

446 + assert audio_output.ndim == 1, f"Expected 1D, got {audio_output.ndim}D"

447 + assert audio_output.shape == (16000,)

448 +

449 + # Verify channel averaging: mean of [1.0, -1.0] = 0.0

450 + np.testing.assert_array_almost_equal(audio_output, np.zeros(16000), decimal=5)

451 +

452 + def test_soundfile_format_normalized_to_mono_e2e(self):

453 + """Full pipeline: soundfile format (time, channels) → mono output."""

454 + from vllm.multimodal.parse import MultiModalDataParser

455 +

456 + # Simulate soundfile output: (time, channels) format

457 + # 16000 samples, 2 channels

458 + stereo_soundfile = np.array(

459 + [[0.5, -0.5]] * 16000, # Each row is [left, right]

460 + dtype=np.float32,

461 + )

462 + assert stereo_soundfile.shape == (16000, 2)

463 +

464 + # Create parser with mono normalization

465 + parser = MultiModalDataParser(

466 + target_sr=16000,

467 + target_channels=1,

468 + )

469 +

470 + # Process audio through the parser

471 + result = parser._parse_audio_data((stereo_soundfile, 16000))

472 + audio_output = result.get(0)

473 +

474 + # Verify output is mono 1D

475 + assert audio_output.ndim == 1, f"Expected 1D, got {audio_output.ndim}D"

476 + assert audio_output.shape == (16000,)

477 +

478 + # Verify channel averaging: mean of [0.5, -0.5] = 0.0

479 + np.testing.assert_array_almost_equal(audio_output, np.zeros(16000), decimal=5)

480 +

481 + def test_librosa_mono_passthrough_e2e(self):

482 + """Full pipeline: librosa mono format → preserved as mono."""

483 + from vllm.multimodal.parse import MultiModalDataParser

484 +

485 + # Simulate librosa output: already mono (time,) format

486 + mono_librosa = np.random.randn(16000).astype(np.float32)

487 + assert mono_librosa.shape == (16000,)

488 +

489 + # Create parser with mono normalization

490 + parser = MultiModalDataParser(

491 + target_sr=16000,

492 + target_channels=1,

493 + )

494 +

495 + # Process audio through the parser

496 + result = parser._parse_audio_data((mono_librosa, 16000))

497 + audio_output = result.get(0)

498 +

499 + # Verify output is still mono 1D

500 + assert audio_output.ndim == 1

501 + assert audio_output.shape == (16000,)

502 +

503 + # Verify audio content is preserved

504 + np.testing.assert_array_almost_equal(audio_output, mono_librosa)

505 +

506 + def test_multichannel_5_1_surround_to_mono_e2e(self):

507 + """Full pipeline: 5.1 surround (6 channels) → mono output."""

508 + from vllm.multimodal.parse import MultiModalDataParser

509 +

510 + # Simulate 5.1 surround audio: 6 channels

511 + surround_audio = np.array(

512 + [

513 + [1.0] * 8000, # Front Left

514 + [2.0] * 8000, # Front Right

515 + [3.0] * 8000, # Center

516 + [4.0] * 8000, # LFE (subwoofer)

517 + [5.0] * 8000, # Rear Left

518 + [6.0] * 8000, # Rear Right

519 + ],

520 + dtype=np.float32,

521 + )

522 + assert surround_audio.shape == (6, 8000)

523 +

524 + # Create parser with mono normalization

525 + parser = MultiModalDataParser(

526 + target_sr=16000,

527 + target_channels=1,

528 + )

529 +

530 + # Process audio through the parser

531 + result = parser._parse_audio_data((surround_audio, 16000))

532 + audio_output = result.get(0)

533 +

534 + # Verify output is mono 1D

535 + assert audio_output.ndim == 1

536 +

537 + # Verify channel averaging: mean of [1,2,3,4,5,6] = 3.5

538 + expected_value = (1.0 + 2.0 + 3.0 + 4.0 + 5.0 + 6.0) / 6

539 + np.testing.assert_array_almost_equal(

540 + audio_output, np.full(8000, expected_value), decimal=5

541 + )

542 +

543 + def test_torch_tensor_input_e2e(self):

544 + """Full pipeline: torch.Tensor stereo input → mono numpy output."""

545 + from vllm.multimodal.parse import MultiModalDataParser

546 +

547 + # Simulate torch tensor input (from torchaudio)

548 + stereo_torch = torch.tensor(

549 + [[1.0] * 8000, [-1.0] * 8000], # 2 channels

550 + dtype=torch.float32,

551 + )

552 + assert stereo_torch.shape == (2, 8000)

553 +

554 + # Create parser with mono normalization

555 + parser = MultiModalDataParser(

556 + target_sr=16000,

557 + target_channels=1,

558 + )

559 +

560 + # Process audio through the parser

561 + # Note: Parser expects numpy, so we convert first (simulating real usage)

562 + result = parser._parse_audio_data((stereo_torch.numpy(), 16000))

563 + audio_output = result.get(0)

564 +

565 + # Verify output is mono 1D numpy array

566 + assert audio_output.ndim == 1

567 + assert isinstance(audio_output, np.ndarray)

568 +

569 + # Verify channel averaging

570 + np.testing.assert_array_almost_equal(audio_output, np.zeros(8000), decimal=5)

571 +

572 + def test_passthrough_preserves_stereo_e2e(self):

573 + """Full pipeline: stereo with target_channels=None → stereo preserved."""

574 + from vllm.multimodal.parse import MultiModalDataParser

575 +

576 + # Stereo audio

577 + stereo_audio = np.array(

578 + [[1.0] * 8000, [-1.0] * 8000],

579 + dtype=np.float32,

580 + )

581 +

582 + # Create parser WITHOUT mono normalization (passthrough)

583 + parser = MultiModalDataParser(

584 + target_sr=16000,

585 + target_channels=None, # Passthrough - no normalization

586 + )

587 +

588 + # Process audio through the parser

589 + result = parser._parse_audio_data((stereo_audio, 16000))

590 + audio_output = result.get(0)

591 +

592 + # Verify output preserves stereo (2D)

593 + assert audio_output.ndim == 2

594 + assert audio_output.shape == (2, 8000)

595 +

596 + def test_resampling_with_channel_normalization_e2e(self):

597 + """Full pipeline: resample + channel normalize in single pass."""

598 + from vllm.multimodal.parse import MultiModalDataParser

599 +

600 + # Stereo audio at 48kHz (common recording rate)

601 + stereo_48k = np.array(

602 + [[1.0] * 48000, [-1.0] * 48000], # 1 second at 48kHz

603 + dtype=np.float32,

604 + )

605 +

606 + # Create parser with both resampling and mono normalization

607 + parser = MultiModalDataParser(

608 + target_sr=16000, # Resample to 16kHz

609 + target_channels=1, # Normalize to mono

610 + )

611 +

612 + # Process audio through the parser

613 + result = parser._parse_audio_data((stereo_48k, 48000))

614 + audio_output = result.get(0)

615 +

616 + # Verify output is mono 1D at target sample rate

617 + assert audio_output.ndim == 1

618 + # After resampling from 48kHz to 16kHz, length should be ~16000

619 + assert audio_output.shape[0] == 16000

620 +

621 + def test_very_short_audio_e2e(self):

622 + """Full pipeline: very short audio (< 1 frame) handled correctly."""

623 + from vllm.multimodal.parse import MultiModalDataParser

624 +

625 + # Very short stereo audio (10 samples)

626 + short_stereo = np.array(

627 + [[1.0] * 10, [-1.0] * 10],

628 + dtype=np.float32,

629 + )

630 +

631 + parser = MultiModalDataParser(

632 + target_sr=16000,

633 + target_channels=1,

634 + )

635 +

636 + result = parser._parse_audio_data((short_stereo, 16000))

637 + audio_output = result.get(0)

638 +

639 + # Should still produce mono output

640 + assert audio_output.ndim == 1

641 + assert audio_output.shape == (10,)

642 + np.testing.assert_array_almost_equal(audio_output, np.zeros(10))

vllm/model_executor/models/qwen2_5_omni_thinker.py

@@ -226,6 +226,10 @@ class Qwen2_5OmniThinkerProcessingInfo(

226 assert isinstance(feature_extractor, WhisperFeatureExtractor)

227 return feature_extractor

228

229 + def get_target_channels(self) -> int:

230 + """Return target audio channels for Qwen2.5 Omni models (mono)."""

231 + return 1

232 +

233 def get_supported_mm_limits(self) -> Mapping[str, int | None]:

234 return {"audio": None, "image": None, "video": None}

235

@@ -310,6 +314,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor(

314 return Qwen2_5OmniThinkerMultiModalDataParser(

315 spatial_merge_size=self.info.get_hf_config().vision_config.spatial_merge_size,

316 target_sr=feature_extractor.sampling_rate,

317 + target_channels=self.info.get_target_channels(),

318 )

319

320 def _call_hf_processor(

vllm/model_executor/models/qwen2_audio.py

+8 -1

@@ -140,6 +140,10 @@ class Qwen2AudioProcessingInfo(BaseProcessingInfo):

140 assert isinstance(feature_extractor, WhisperFeatureExtractor)

141 return feature_extractor

142

143 + def get_target_channels(self) -> int:

144 + """Return target audio channels for Qwen2 Audio models (mono)."""

145 + return 1

146 +

147 def get_supported_mm_limits(self) -> Mapping[str, int | None]:

148 return {"audio": None}

149

@@ -201,7 +205,10 @@ class Qwen2AudioMultiModalDataParser(MultiModalDataParser):

205 class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor[Qwen2AudioProcessingInfo]):

206 def _get_data_parser(self) -> MultiModalDataParser:

207 feature_extractor = self.info.get_feature_extractor()

204 - return Qwen2AudioMultiModalDataParser(target_sr=feature_extractor.sampling_rate)

208 + return Qwen2AudioMultiModalDataParser(

209 + target_sr=feature_extractor.sampling_rate,

210 + target_channels=self.info.get_target_channels(),

211 + )

212

213 def _call_hf_processor(

214 self,

vllm/model_executor/models/ultravox.py

+8 -1

@@ -133,6 +133,10 @@ class UltravoxProcessingInfo(BaseProcessingInfo):

133 assert isinstance(feature_extractor, WhisperFeatureExtractor)

134 return feature_extractor

135

136 + def get_target_channels(self) -> int:

137 + """Return target audio channels for Ultravox models (mono)."""

138 + return 1

139 +

140 def get_supported_mm_limits(self) -> Mapping[str, int | None]:

141 return {"audio": None}

142

@@ -169,7 +173,10 @@ class UltravoxDummyInputsBuilder(BaseDummyInputsBuilder[UltravoxProcessingInfo])

173 class UltravoxMultiModalProcessor(BaseMultiModalProcessor[UltravoxProcessingInfo]):

174 def _get_data_parser(self) -> MultiModalDataParser:

175 feature_extractor = self.info.get_feature_extractor()

172 - return MultiModalDataParser(target_sr=feature_extractor.sampling_rate)

176 + return MultiModalDataParser(

177 + target_sr=feature_extractor.sampling_rate,

178 + target_channels=self.info.get_target_channels(),

179 + )

180

181 def _call_hf_processor(

182 self,

vllm/model_executor/models/whisper.py

+8 -1

@@ -690,6 +690,10 @@ class WhisperProcessingInfo(BaseProcessingInfo):

690 assert isinstance(feature_extractor, WhisperFeatureExtractor)

691 return feature_extractor

692

693 + def get_target_channels(self) -> int:

694 + """Return target audio channels for Whisper models (mono)."""

695 + return 1

696 +

697 def get_num_audio_tokens(self) -> int:

698 return self.get_hf_config().max_source_positions

699

@@ -724,7 +728,10 @@ class WhisperDummyInputsBuilder(BaseDummyInputsBuilder[WhisperProcessingInfo]):

728 class WhisperMultiModalProcessor(EncDecMultiModalProcessor[WhisperProcessingInfo]):

729 def _get_data_parser(self) -> MultiModalDataParser:

730 feature_extractor = self.info.get_feature_extractor()

727 - return MultiModalDataParser(target_sr=feature_extractor.sampling_rate)

731 + return MultiModalDataParser(

732 + target_sr=feature_extractor.sampling_rate,

733 + target_channels=self.info.get_target_channels(),

734 + )

735

736 @property

737 def pad_dummy_encoder_prompt(self) -> bool:

vllm/multimodal/audio.py

+132

@@ -1,6 +1,8 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 import base64

4 +from dataclasses import dataclass

5 +from enum import Enum

6 from io import BytesIO

7 from pathlib import Path

8 from typing import Literal

@@ -25,6 +27,136 @@ try:

27 except ImportError:

28 soundfile = PlaceholderModule("soundfile") # type: ignore[assignment]

30 +# ============================================================

31 +

32 +

33 +class ChannelReduction(str, Enum):

34 + """Method to reduce multi-channel audio to target channels."""

35 +

36 + MEAN = "mean" # Average across channels (default, preserves energy balance)

37 + FIRST = "first" # Take first channel only

38 + MAX = "max" # Take max value across channels

39 + SUM = "sum" # Sum across channels

40 +

41 +

42 +@dataclass

43 +class AudioSpec:

44 + """Specification for target audio format.

45 +

46 + This dataclass defines the expected audio format for a model's feature

47 + extractor. It is used to normalize audio data before processing.

48 +

49 + Attributes:

50 + target_channels: Number of output channels. None means passthrough

51 + (no normalization). 1 = mono, 2 = stereo, etc.

52 + channel_reduction: Method to reduce channels when input has more

53 + channels than target. Only used when reducing channels.

54 + """

55 +

56 + target_channels: int | None = 1

57 + channel_reduction: ChannelReduction = ChannelReduction.MEAN

58 +

59 + @property

60 + def needs_normalization(self) -> bool:

61 + """Whether audio normalization is needed."""

62 + return self.target_channels is not None

63 +

64 + def __repr__(self) -> str:

65 + if self.target_channels is None:

66 + return "AudioSpec(passthrough)"

67 + return (

68 + f"AudioSpec(channels={self.target_channels}, "

69 + f"reduction={self.channel_reduction.value})"

70 + )

71 +

72 +

73 +# Pre-defined specs for common use cases

74 +MONO_AUDIO_SPEC = AudioSpec(target_channels=1, channel_reduction=ChannelReduction.MEAN)

75 +PASSTHROUGH_AUDIO_SPEC = AudioSpec(target_channels=None)

76 +

77 +

78 +def normalize_audio(

79 + audio: npt.NDArray[np.floating] | torch.Tensor,

80 + spec: AudioSpec,

81 +) -> npt.NDArray[np.floating] | torch.Tensor:

82 + """Normalize audio to the specified format.

83 +

84 + This function handles channel reduction for multi-channel audio,

85 + supporting both numpy arrays and torch tensors.

86 +

87 + Args:

88 + audio: Input audio data. Can be:

89 + - 1D array/tensor: (time,) - already mono

90 + - 2D array/tensor: (channels, time) - standard format from torchaudio

91 + - 2D array/tensor: (time, channels) - format from soundfile

92 + (will be auto-detected and transposed if time > channels)

93 + spec: AudioSpec defining the target format.

94 +

95 + Returns:

96 + Normalized audio in the same type as input (numpy or torch).

97 + For mono output (target_channels=1), returns 1D array/tensor.

98 +

99 + Raises:

100 + ValueError: If audio has unsupported dimensions or channel expansion

101 + is requested (e.g., mono to stereo).

102 + """

103 + if not spec.needs_normalization:

104 + return audio

105 +

106 + # Handle 1D audio (already mono)

107 + if audio.ndim == 1:

108 + if spec.target_channels == 1:

109 + return audio

110 + raise ValueError(f"Cannot expand mono audio to {spec.target_channels} channels")

111 +

112 + # Handle 2D audio

113 + if audio.ndim != 2:

114 + raise ValueError(f"Unsupported audio shape: {audio.shape}. Expected 1D or 2D.")

115 +

116 + # Auto-detect format: if shape[0] > shape[1], assume (time, channels)

117 + # This handles soundfile format where time dimension is typically much larger

118 + if audio.shape[0] > audio.shape[1]:

119 + # Transpose from (time, channels) to (channels, time)

120 + audio = audio.T if isinstance(audio, np.ndarray) else audio.T

121 +

122 + num_channels = audio.shape[0]

123 +

124 + # No reduction needed if already at target

125 + if num_channels == spec.target_channels:

126 + return audio

127 +

128 + # Cannot expand channels

129 + if num_channels < spec.target_channels:

130 + raise ValueError(

131 + f"Cannot expand {num_channels} channels to {spec.target_channels}"

132 + )

133 +

134 + # Reduce channels

135 + is_numpy = isinstance(audio, np.ndarray)

136 +

137 + if spec.target_channels == 1:

138 + # Reduce to mono

139 + if spec.channel_reduction == ChannelReduction.MEAN:

140 + result = np.mean(audio, axis=0) if is_numpy else audio.mean(dim=0)

141 + elif spec.channel_reduction == ChannelReduction.FIRST:

142 + result = audio[0]

143 + elif spec.channel_reduction == ChannelReduction.MAX:

144 + result = np.max(audio, axis=0) if is_numpy else audio.max(dim=0).values

145 + elif spec.channel_reduction == ChannelReduction.SUM:

146 + result = np.sum(audio, axis=0) if is_numpy else audio.sum(dim=0)

147 + else:

148 + raise ValueError(f"Unknown reduction method: {spec.channel_reduction}")

149 + return result

150 + else:

151 + # Reduce to N channels (take first N and apply reduction if needed)

152 + # For now, just take first N channels

153 + return audio[: spec.target_channels]

154 +

155 +

156 +# ============================================================

157 +# Audio Resampling

158 +# ============================================================

159 +

160

161 def resample_audio_librosa(

162 audio: npt.NDArray[np.floating],

vllm/multimodal/parse.py

+11 -1

@@ -22,7 +22,7 @@ from typing_extensions import assert_never

22 from vllm.utils.collection_utils import is_list_of

23 from vllm.utils.import_utils import LazyLoader

25 -from .audio import AudioResampler

25 +from .audio import AudioResampler, AudioSpec, normalize_audio

26 from .base import MediaWithBytes

27 from .inputs import (

28 AudioItem,

@@ -456,6 +456,9 @@ class MultiModalDataParser:

456 Args:

457 target_sr (float, optional): Enables automatic resampling of audio

458 items to the model's expected sampling rate.

459 + target_channels (int, optional): Target number of audio channels.

460 + If provided, normalizes audio to this many channels (e.g., 1 for mono).

461 + If None, audio channels are passed through unchanged.

462 expected_hidden_size (int, optional): Expected hidden dimension for

463 embedding inputs. If provided, validates that user-supplied

464 embeddings have the correct hidden size to prevent crashes

@@ -466,6 +469,7 @@ class MultiModalDataParser:

469 self,

470 *,

471 target_sr: float | None = None,

472 + target_channels: int | None = None,

473 audio_resample_method: Literal["librosa", "scipy"] = "librosa",

474 video_needs_metadata: bool = False,

475 expected_hidden_size: int | None = None,

@@ -476,6 +480,7 @@ class MultiModalDataParser:

480 target_sr=target_sr,

481 method=audio_resample_method,

482 )

483 + self.target_channels = target_channels

484 self.video_needs_metadata = video_needs_metadata

485 self.expected_hidden_size = expected_hidden_size

486

@@ -565,6 +570,11 @@ class MultiModalDataParser:

570 else:

571 new_audio = self.audio_resampler.resample(audio, orig_sr=orig_sr)

572

573 + # Apply channel normalization if target_channels is set

574 + if self.target_channels is not None:

575 + spec = AudioSpec(target_channels=self.target_channels)

576 + new_audio = normalize_audio(new_audio, spec)

577 +

578 new_audios.append(new_audio)

579

580 return AudioProcessorItems(new_audios)