@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 226ba9fc9e

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

[ASR] Add Long Audio benchmark and correctness test (#44587)

Signed-off-by: Ekagra Ranjan <3116519+ekagra-ranjan@users.noreply.github.com>

Ekagra Ranjan committed Jun 12, 2026 at 00:11 UTC 226ba9fc9e285556e7269e4efe102530a4d9fedb

5 files changed +530 -46

docs/benchmarking/cli.md

+2 -2

@@ -37,7 +37,7 @@ th {

37 | HuggingFace-HumanEval | ✅ | ✅ | `openai/openai_humaneval` |

38 | HuggingFace-GSM8K | ✅ | ✅ | `openai/gsm8k` |

39 | HuggingFace-Blazedit | ✅ | ✅ | `vdaita/edit_5k_char`, `vdaita/edit_10k_char` |

40 -| HuggingFace-ASR | ✅ | ✅ | `openslr/librispeech_asr`, `facebook/voxpopuli`, `LIUM/tedlium`, `edinburghcstr/ami`, `speechcolab/gigaspeech`, `kensho/spgispeech` |

40 +| HuggingFace-ASR | ✅ | ✅ | `openslr/librispeech_asr`, `facebook/voxpopuli`, `LIUM/tedlium`, `edinburghcstr/ami`, `speechcolab/gigaspeech`, `kensho/spgispeech`, `ArtificialAnalysis/Earnings22-Cleaned-AA`, `D4nt3/esb-datasets-earnings22-validation-tiny-filtered` |

41 | Spec Bench | ✅ | ✅ | `wget https://raw.githubusercontent.com/hemingkx/Spec-Bench/refs/heads/main/data/spec_bench/question.jsonl` |

42 | SPEED-Bench | ✅ | ✅ | `curl -LsSf https://raw.githubusercontent.com/NVIDIA-NeMo/Skills/refs/heads/main/nemo_skills/dataset/speed-bench/prepare.py \| python3 -` |

43 | Custom | ✅ | ✅ | Local file: `data.jsonl` |

@@ -532,7 +532,7 @@ vllm bench serve \

532 --blazedit-max-distance 0.99

533 ```

534

535 -`openslr/librispeech_asr`, `facebook/voxpopuli`, `LIUM/tedlium`, `edinburghcstr/ami`, `speechcolab/gigaspeech`, `kensho/spgispeech`

535 +`openslr/librispeech_asr`, `facebook/voxpopuli`, `LIUM/tedlium`, `edinburghcstr/ami`, `speechcolab/gigaspeech`, `kensho/spgispeech`, `ArtificialAnalysis/Earnings22-Cleaned-AA`, `D4nt3/esb-datasets-earnings22-validation-tiny-filtered`

536

537 ```bash

538 vllm bench serve \

tests/benchmarks/test_audio_dataset.py new

+200

@@ -0,0 +1,200 @@

1 +#!/usr/bin/env python3

2 +# SPDX-License-Identifier: Apache-2.0

3 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

4 +

5 +import asyncio

6 +from pathlib import Path

7 +from typing import Protocol, cast

8 +

9 +import numpy as np

10 +import pytest

11 +import soundfile as sf

12 +

13 +import vllm.benchmarks.datasets.datasets as datasets_module

14 +import vllm.benchmarks.lib.endpoint_request_func as request_func_module

15 +from vllm.benchmarks.lib.endpoint_request_func import RequestFuncInput

16 +

17 +pytestmark = pytest.mark.skip_global_cleanup

18 +

19 +

20 +class _ReadableBinary(Protocol):

21 + def read(self, size: int = -1) -> bytes: ...

22 +

23 +

24 +class _TokenizedPrompt:

25 + def __init__(self, prompt: str) -> None:

26 + self.input_ids = prompt.split()

27 +

28 +

29 +class _Tokenizer:

30 + def __init__(self, name_or_path: str = "openai/whisper-large-v3") -> None:

31 + self.name_or_path = name_or_path

32 +

33 + def __call__(self, prompt: str) -> _TokenizedPrompt:

34 + return _TokenizedPrompt(prompt)

35 +

36 +

37 +def _write_wav(path: Path, duration_s: float = 0.1, sample_rate: int = 16_000) -> None:

38 + num_samples = int(duration_s * sample_rate)

39 + sf.write(path, np.zeros(num_samples, dtype=np.float32), sample_rate)

40 +

41 +

42 +class _FakeFormData:

43 + def __init__(self) -> None:

44 + self.fields: list[tuple[str, object, dict[str, str]]] = []

45 +

46 + def add_field(self, name: str, value: object, **kwargs: str) -> None:

47 + self.fields.append((name, value, kwargs))

48 +

49 +

50 +class _FakeContent:

51 + async def iter_any(self):

52 + yield b'data: {"choices":[{"delta":{"content":"hello"}}]}\n\n'

53 + yield b'data: {"usage":{"completion_tokens":1}}\n\n'

54 + yield b"data: [DONE]\n\n"

55 +

56 +

57 +class _FakeResponse:

58 + def __init__(self) -> None:

59 + self.status = 200

60 + self.reason = "OK"

61 + self.content = _FakeContent()

62 +

63 + async def __aenter__(self):

64 + return self

65 +

66 + async def __aexit__(self, exc_type, exc, tb):

67 + return False

68 +

69 +

70 +class _FakeSession:

71 + def __init__(self) -> None:

72 + self.uploaded_bytes: bytes | None = None

73 + self.upload_filename: str | None = None

74 + self.fields: list[tuple[str, object, dict[str, str]]] | None = None

75 +

76 + def post(self, *, url: str, data: _FakeFormData, headers: dict[str, str]):

77 + del url, headers

78 + self.fields = list(data.fields)

79 + _, file_obj, file_kwargs = self.fields[0]

80 + file_obj = cast(_ReadableBinary, file_obj)

81 + self.uploaded_bytes = file_obj.read()

82 + self.upload_filename = file_kwargs.get("filename")

83 + return _FakeResponse()

84 +

85 +

86 +def test_asr_dataset_sample_handles_local_audio_paths(tmp_path: Path) -> None:

87 + audio_path = tmp_path / "earnings.wav"

88 + _write_wav(audio_path, duration_s=0.1)

89 +

90 + dataset = object.__new__(datasets_module.ASRDataset)

91 + dataset.data = [

92 + {

93 + "audio": {

94 + "path": str(audio_path),

95 + "bytes": None,

96 + },

97 + "text": "quarterly earnings call",

98 + }

99 + ]

100 +

101 + samples = dataset.sample(

102 + tokenizer=_Tokenizer(),

103 + num_requests=1,

104 + output_len=32,

105 + asr_min_audio_len_sec=0.0,

106 + asr_max_audio_len_sec=1.0,

107 + )

108 +

109 + assert len(samples) == 1

110 + assert samples[0].multi_modal_data == {"audio_path": str(audio_path)}

111 + assert (

112 + samples[0].prompt == "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>"

113 + )

114 +

115 +

116 +def test_asr_dataset_sample_handles_embedded_audio_bytes(tmp_path: Path) -> None:

117 + audio_path = tmp_path / "earnings.wav"

118 + _write_wav(audio_path, duration_s=0.1)

119 +

120 + dataset = object.__new__(datasets_module.ASRDataset)

121 + dataset.data = [

122 + {

123 + "audio": {

124 + "path": None,

125 + "bytes": audio_path.read_bytes(),

126 + },

127 + "text": "quarterly earnings call",

128 + }

129 + ]

130 +

131 + samples = dataset.sample(

132 + tokenizer=_Tokenizer(),

133 + num_requests=1,

134 + output_len=32,

135 + asr_min_audio_len_sec=0.0,

136 + asr_max_audio_len_sec=1.0,

137 + )

138 +

139 + assert len(samples) == 1

140 + assert isinstance(samples[0].multi_modal_data, dict)

141 + audio, sample_rate = samples[0].multi_modal_data["audio"]

142 + assert sample_rate == 16_000

143 + assert isinstance(audio, np.ndarray)

144 + assert audio.size > 0

145 +

146 +

147 +def test_async_request_openai_audio_handles_local_audio_paths(

148 + monkeypatch: pytest.MonkeyPatch,

149 + tmp_path: Path,

150 +) -> None:

151 + audio_path = tmp_path / "earnings.wav"

152 + _write_wav(audio_path, duration_s=0.25)

153 +

154 + monkeypatch.setattr(request_func_module.aiohttp, "FormData", _FakeFormData)

155 + session = _FakeSession()

156 + request_input = RequestFuncInput(

157 + prompt="",

158 + api_url="http://localhost:8000/v1/audio/transcriptions",

159 + prompt_len=1,

160 + output_len=32,

161 + model="openai/whisper-large-v3",

162 + multi_modal_content={"audio_path": str(audio_path)},

163 + )

164 +

165 + output = asyncio.run(

166 + request_func_module.async_request_openai_audio(request_input, session)

167 + )

168 +

169 + assert session.upload_filename == audio_path.name

170 + assert session.uploaded_bytes == audio_path.read_bytes()

171 + assert output.success is True

172 + assert output.generated_text == "hello"

173 + assert output.output_tokens == 1

174 + assert output.input_audio_duration == pytest.approx(0.25, abs=1e-2)

175 +

176 +

177 +def test_async_request_openai_audio_handles_decoded_audio_arrays(

178 + monkeypatch: pytest.MonkeyPatch,

179 +) -> None:

180 + monkeypatch.setattr(request_func_module.aiohttp, "FormData", _FakeFormData)

181 + session = _FakeSession()

182 + request_input = RequestFuncInput(

183 + prompt="",

184 + api_url="http://localhost:8000/v1/audio/transcriptions",

185 + prompt_len=1,

186 + output_len=32,

187 + model="openai/whisper-large-v3",

188 + multi_modal_content={

189 + "audio": (np.zeros(1_600, dtype=np.float32), 16_000),

190 + },

191 + )

192 +

193 + output = asyncio.run(

194 + request_func_module.async_request_openai_audio(request_input, session)

195 + )

196 +

197 + assert session.upload_filename == "audio.wav"

198 + assert session.uploaded_bytes is not None

199 + assert output.success is True

200 + assert output.generated_text == "hello"

tests/entrypoints/speech_to_text/correctness/test_transcription_api_correctness.py

+201 -19

@@ -16,10 +16,11 @@ from statistics import mean, median

16 import pytest

17 import soundfile

18 import torch

19 -from datasets import load_dataset

19 +from datasets import Audio, load_dataset

20 from evaluate import load

21 from transformers.models.whisper.english_normalizer import EnglishTextNormalizer

23 +from vllm.benchmarks.datasets.datasets import ASRDataset

24 from vllm.multimodal.audio import get_audio_duration

25 from vllm.tokenizers import get_tokenizer

@@ -38,6 +39,20 @@ def to_bytes(y, sr):

39 return buffer

42 +def load_audio_sample(audio):

43 + # Avoid torchcodec in CI by decoding dataset audio with soundfile.

44 + if "array" in audio and "sampling_rate" in audio:

45 + return audio["array"], audio["sampling_rate"]

46 +

47 + if audio.get("path"):

48 + return soundfile.read(audio["path"], dtype="float32")

49 +

50 + if audio.get("bytes") is not None:

51 + return soundfile.read(io.BytesIO(audio["bytes"]), dtype="float32")

52 +

53 + raise ValueError("Audio sample did not contain array, path, or bytes data")

54 +

55 +

56 # not all models have a normalizer so use the one from whisper as a standard option

57 normalizer_model_info = HF_EXAMPLE_MODELS.find_hf_info("openai/whisper-large-v3")

58 normalizer_tokenizer = get_tokenizer(

@@ -48,7 +63,7 @@ normalizer_tokenizer = get_tokenizer(

63 normalizer = EnglishTextNormalizer(normalizer_tokenizer.english_spelling_normalizer)

51 -async def transcribe_audio(client, tokenizer, y, sr):

66 +async def transcribe_audio(client, tokenizer, y, sr, extra_body=None):

67 # Send loaded audio directly instead of loading from disk,

68 # don't account for that time though

69 with to_bytes(y, sr) as f:

@@ -58,6 +73,7 @@ async def transcribe_audio(client, tokenizer, y, sr):

73 model=tokenizer.name_or_path,

74 language="en",

75 temperature=0.0,

76 + extra_body=extra_body,

77 )

78 end_time = time.perf_counter()

79 # NOTE there's no streaming in transcriptions, can't measure ttft

@@ -68,17 +84,21 @@ async def transcribe_audio(client, tokenizer, y, sr):

84 return latency, num_output_tokens, transcription.text

71 -async def bound_transcribe(sem, client, tokenizer, audio, reference):

87 +async def bound_transcribe(

88 + sem, client, tokenizer, audio, sr, reference, extra_body=None

89 +):

90 # Use semaphore to limit concurrent requests.

91 async with sem:

74 - result = await transcribe_audio(client, tokenizer, *audio)

92 + result = await transcribe_audio(

93 + client, tokenizer, audio, sr, extra_body=extra_body

94 + )

95 # Normalize *english* output/reference for evaluation.

96 out = normalizer(result[2])

97 ref = normalizer(reference)

98 return result[:2] + (out, ref)

100

81 -async def process_dataset(model, client, data, concurrent_request):

101 +async def process_dataset(model, client, data, concurrent_request, extra_body=None):

102 sem = asyncio.Semaphore(concurrent_request)

103

104 model_info = HF_EXAMPLE_MODELS.find_hf_info(model)

@@ -89,14 +109,16 @@ async def process_dataset(model, client, data, concurrent_request):

109 )

110

111 # Warmup call as the first `load_audio` server-side is quite slow.

92 - audio, sr = data[0]["audio"]["array"], data[0]["audio"]["sampling_rate"]

93 - _ = await bound_transcribe(sem, client, tokenizer, (audio, sr), "")

112 + audio, sr = load_audio_sample(data[0]["audio"])

113 + _ = await bound_transcribe(sem, client, tokenizer, audio, sr, "", extra_body)

114

115 tasks: list[asyncio.Task] = []

116 for sample in data:

97 - audio, sr = sample["audio"]["array"], sample["audio"]["sampling_rate"]

117 + audio, sr = load_audio_sample(sample["audio"])

118 task = asyncio.create_task(

99 - bound_transcribe(sem, client, tokenizer, (audio, sr), sample["text"])

119 + bound_transcribe(

120 + sem, client, tokenizer, audio, sr, sample["text"], extra_body

121 + )

122 )

123 tasks.append(task)

124 return await asyncio.gather(*tasks)

@@ -121,19 +143,36 @@ def print_performance_metrics(results, total_time):

143

144

145 def add_duration(sample):

124 - y, sr = sample["audio"]["array"], sample["audio"]["sampling_rate"]

146 + y, sr = load_audio_sample(sample["audio"])

147 sample["duration_ms"] = get_audio_duration(y=y, sr=sr) * 1000

148 return sample

149

150

129 -def load_hf_dataset(dataset_repo: str, split="validation", **hf_kwargs):

130 - ## Load and filter the dataset

131 - dataset = load_dataset(dataset_repo, split=split, **hf_kwargs)

132 - if "duration_ms" not in dataset[0]:

133 - # compute duration to filter

151 +def load_asr_dataset_rows(dataset_repo: str, split="validation", **hf_kwargs):

152 + if dataset_repo in ASRDataset.SUPPORTED_DATASET_PATHS:

153 + asr_dataset_kwargs = {

154 + "dataset_path": dataset_repo,

155 + "dataset_split": split,

156 + "disable_shuffle": True,

157 + "no_stream": True,

158 + }

159 + for key in ("dataset_subset", "hf_name", "trust_remote_code"):

160 + if key in hf_kwargs:

161 + asr_dataset_kwargs[key] = hf_kwargs[key]

162 + return ASRDataset(**asr_dataset_kwargs).data

163 +

164 + return load_dataset(dataset_repo, split=split, **hf_kwargs)

165 +

166 +

167 +def load_shortform_eval_dataset(dataset_repo: str, split="validation", **hf_kwargs):

168 + ## Load and filter the dataset.

169 + dataset = load_asr_dataset_rows(dataset_repo, split=split, **hf_kwargs)

170 + dataset = dataset.cast_column("audio", Audio(decode=False))

171 + if "duration_ms" not in dataset.column_names:

172 + # Compute duration to filter.

173 dataset = dataset.map(add_duration)

174

136 - # Whisper max supported duration

175 + # Whisper max supported duration.

176 dataset = dataset.filter(lambda example: example["duration_ms"] < 30000)

177 return dataset

178

@@ -145,11 +184,16 @@ def run_evaluation(

184 max_concurrent_reqs: int,

185 n_examples: int = -1,

186 print_metrics: bool = True,

187 + extra_body=None,

188 ):

189 if n_examples > 0:

190 dataset = dataset.select(range(n_examples))

191 start = time.perf_counter()

152 - results = asyncio.run(process_dataset(model, client, dataset, max_concurrent_reqs))

192 + results = asyncio.run(

193 + process_dataset(

194 + model, client, dataset, max_concurrent_reqs, extra_body=extra_body

195 + )

196 + )

197 end = time.perf_counter()

198 total_time = end - start

199 print(f"Total Test Time: {total_time:.4f} seconds")

@@ -164,6 +208,106 @@ def run_evaluation(

208 return wer_score

209

210

211 +LONGFORM_DATASET_REPO = ASRDataset.EARNINGS22_CLEANED_DATASET

212 +LONGFORM_DATASET_SPLIT = "test"

213 +LONGFORM_NUM_SAMPLES = 6

214 +

215 +

216 +def load_longform_dataset():

217 + dataset = load_asr_dataset_rows(

218 + LONGFORM_DATASET_REPO,

219 + split=LONGFORM_DATASET_SPLIT,

220 + )

221 + assert len(dataset) >= LONGFORM_NUM_SAMPLES

222 + return dataset.select(range(LONGFORM_NUM_SAMPLES))

223 +

224 +

225 +async def transcribe_audio_path(client, tokenizer, audio_path: str, extra_body=None):

226 + with open(audio_path, "rb") as f:

227 + start_time = time.perf_counter()

228 + transcription = await client.audio.transcriptions.create(

229 + file=f,

230 + model=tokenizer.name_or_path,

231 + language="en",

232 + temperature=0.0,

233 + extra_body=extra_body,

234 + )

235 + end_time = time.perf_counter()

236 +

237 + latency = end_time - start_time

238 + num_output_tokens = len(

239 + tokenizer(transcription.text, add_special_tokens=False).input_ids

240 + )

241 + return latency, num_output_tokens, transcription.text

242 +

243 +

244 +async def bound_transcribe_path(

245 + sem, client, tokenizer, audio_path, reference, extra_body=None

246 +):

247 + async with sem:

248 + result = await transcribe_audio_path(

249 + client, tokenizer, audio_path, extra_body=extra_body

250 + )

251 + out = normalizer(result[2])

252 + ref = normalizer(reference)

253 + return result[:2] + (out, ref)

254 +

255 +

256 +async def process_longform_dataset(

257 + model, client, data, concurrent_request, extra_body=None

258 +):

259 + sem = asyncio.Semaphore(concurrent_request)

260 +

261 + model_info = HF_EXAMPLE_MODELS.find_hf_info(model)

262 + tokenizer = get_tokenizer(

263 + model,

264 + tokenizer_mode=model_info.tokenizer_mode,

265 + trust_remote_code=model_info.trust_remote_code,

266 + )

267 +

268 + warmup_path = data[0]["audio"]["path"]

269 + _ = await bound_transcribe_path(sem, client, tokenizer, warmup_path, "", extra_body)

270 +

271 + tasks: list[asyncio.Task] = []

272 + for sample in data:

273 + audio_path = sample["audio"]["path"]

274 + task = asyncio.create_task(

275 + bound_transcribe_path(

276 + sem, client, tokenizer, audio_path, sample["text"], extra_body

277 + )

278 + )

279 + tasks.append(task)

280 + return await asyncio.gather(*tasks)

281 +

282 +

283 +def run_longform_evaluation(

284 + model: str,

285 + client,

286 + dataset,

287 + max_concurrent_reqs: int,

288 + print_metrics: bool = True,

289 + extra_body=None,

290 +):

291 + start = time.perf_counter()

292 + results = asyncio.run(

293 + process_longform_dataset(

294 + model, client, dataset, max_concurrent_reqs, extra_body=extra_body

295 + )

296 + )

297 + end = time.perf_counter()

298 + total_time = end - start

299 + print(f"Total Test Time: {total_time:.4f} seconds")

300 + if print_metrics:

301 + print_performance_metrics(results, total_time)

302 +

303 + predictions = [res[2] for res in results]

304 + references = [res[3] for res in results]

305 + wer = load("wer")

306 + wer_score = 100 * wer.compute(references=references, predictions=predictions)

307 + print("WER:", wer_score)

308 + return wer_score

309 +

310 +

311 # alternatives "openai/whisper-large-v2", "openai/whisper-large-v3-turbo"..

312 # NOTE: Expected WER measured with equivalent hf.transformers args:

313 # whisper-large-v3 + esb-datasets-earnings22-validation-tiny-filtered.

@@ -184,7 +328,6 @@ def test_wer_correctness(

328 ):

329 model_name, expected_wer = model_config

330 model_info = HF_EXAMPLE_MODELS.find_hf_info(model_name)

187 - # TODO refactor to use `ASRDataset`

331 server_args = [

332 "--enforce-eager",

333 f"--tokenizer_mode={model_info.tokenizer_mode}",

@@ -197,7 +340,7 @@ def test_wer_correctness(

340 model_name,

341 server_args,

342 ) as remote_server:

200 - dataset = load_hf_dataset(dataset_repo)

343 + dataset = load_shortform_eval_dataset(dataset_repo)

344

345 if not max_concurrent_request:

346 # No max concurrency

@@ -216,3 +359,42 @@ def test_wer_correctness(

359

360 if expected_wer:

361 torch.testing.assert_close(wer, expected_wer, atol=1e-1, rtol=1e-2)

362 +

363 +

364 +# 14-22mins of 6 audio samples of total ~115 mins and just 37MB.

365 +# checks for long audio transcription correctness and RMS split.

366 +@pytest.mark.parametrize(

367 + "model_config",

368 + [("openai/whisper-large-v3", 9.5)],

369 +)

370 +def test_long_audio_wer_correctness(model_config):

371 + model_name, expected_wer = model_config

372 + model_info = HF_EXAMPLE_MODELS.find_hf_info(model_name)

373 + server_args = [

374 + f"--tokenizer_mode={model_info.tokenizer_mode}",

375 + ]

376 +

377 + if model_info.trust_remote_code:

378 + server_args.append("--trust-remote-code")

379 +

380 + # 1800 seconds is 30 minutes

381 + env_dict = {

382 + "VLLM_MAX_AUDIO_DECODE_DURATION_S": "1800",

383 + }

384 +

385 + with RemoteOpenAIServer(

386 + model_name,

387 + server_args,

388 + env_dict=env_dict,

389 + ) as remote_server:

390 + dataset = load_longform_dataset()

391 + client = remote_server.get_async_client()

392 + wer = run_longform_evaluation(

393 + model=model_name,

394 + client=client,

395 + dataset=dataset,

396 + max_concurrent_reqs=LONGFORM_NUM_SAMPLES,

397 + )

398 +

399 + print(f"Expected WER: {expected_wer}, Actual WER: {wer}")

400 + torch.testing.assert_close(wer, expected_wer, atol=1e-1, rtol=1e-2)

vllm/benchmarks/datasets/datasets.py

+83 -14

@@ -4001,20 +4001,27 @@ class ASRDataset(HuggingFaceDataset):

4001 Dataset class for processing a ASR dataset for transcription.

4002 Tested on the following set:

4003

4004 - +----------------+----------------------------------------+--------------------------+-----------------------------+

4006 - +----------------+----------------------------------------+--------------------------+-----------------------------+

4008 - | | | | release3-speaker-adaptation |

4014 - +----------------+----------------------------------------+--------------------------+-----------------------------+

4004 + +---------------------------+----------------------------------------+--------------------------+-----------------------------+

4006 + +---------------------------+----------------------------------------+--------------------------+-----------------------------+

4008 + | | | | release3-speaker-adaptation |

4016 + +---------------------------+----------------------------------------+--------------------------+-----------------------------+

4017

4018 """ # noqa: E501

4019

4020 + EARNINGS22_CLEANED_DATASET = "ArtificialAnalysis/Earnings22-Cleaned-AA"

4021 + EARNINGS22_TINY_FILTERED_DATASET = (

4022 + "D4nt3/esb-datasets-earnings22-validation-tiny-filtered"

4023 + )

4024 +

4025 SUPPORTED_DATASET_PATHS = {

4026 "openslr/librispeech_asr",

4027 "facebook/voxpopuli",

@@ -4022,11 +4029,52 @@ class ASRDataset(HuggingFaceDataset):

4029 "edinburghcstr/ami",

4030 "speechcolab/gigaspeech",

4031 "kensho/spgispeech",

4032 + EARNINGS22_CLEANED_DATASET,

4033 + EARNINGS22_TINY_FILTERED_DATASET,

4034 }

4035

4036 DEFAULT_OUTPUT_LEN = 1024

4037 IS_MULTIMODAL = True

4038

4039 + def load_data(self) -> None:

4040 + if self.hf_name == self.EARNINGS22_CLEANED_DATASET:

4041 + # This subset stores repo-local MP3 paths instead of a HF `Audio`

4042 + # column, so eagerly materialize it back into the common schema.

4043 + self.data = load_dataset(

4044 + self.dataset_path,

4045 + name=self.dataset_subset,

4046 + split=self.dataset_split,

4047 + streaming=False,

4048 + trust_remote_code=self.trust_remote_code,

4049 + )

4050 + if not getattr(self, "disable_shuffle", False):

4051 + self.data = self.data.shuffle(seed=self.random_seed)

4052 + self._materialize_local_audio_column()

4053 + return

4054 + if self.hf_name == self.EARNINGS22_TINY_FILTERED_DATASET:

4055 + super().load_data()

4056 + self._disable_audio_decode()

4057 + return

4058 +

4059 + super().load_data()

4060 +

4061 + def _disable_audio_decode(self) -> None:

4062 + from datasets import Audio

4063 +

4064 + self.data = self.data.cast_column("audio", Audio(decode=False))

4065 +

4066 + def _materialize_local_audio_column(self) -> None:

4067 + local_path_root = Path(

4068 + hf_api().snapshot_download(self.hf_name, repo_type="dataset")

4069 + )

4070 + self.data = self.data.map(

4071 + lambda item: {

4072 + "audio": str(local_path_root / item["url"]),

4073 + "text": item["transcript"],

4074 + }

4075 + )

4076 + self._disable_audio_decode()

4077 +

4078 def sample(

4079 self,

4080 tokenizer: TokenizerLike,

@@ -4052,14 +4100,35 @@ class ASRDataset(HuggingFaceDataset):

4100 if len(sampled_requests) >= num_requests:

4101 break

4102 audio = item["audio"]

4055 - y, sr = audio["array"], audio["sampling_rate"]

4056 - duration_s = get_audio_duration(y=y, sr=sr)

4103 + if (

4104 + isinstance(audio, dict)

4105 + and "array" in audio

4106 + and "sampling_rate" in audio

4107 + ):

4108 + y, sr = audio["array"], audio["sampling_rate"]

4109 + duration_s = get_audio_duration(y=y, sr=sr)

4110 + mm_content = {"audio": (y, sr)}

4111 + elif isinstance(audio, str):

4112 + duration_s = sf.info(audio).duration

4113 + mm_content = {"audio_path": audio}

4114 + elif isinstance(audio, dict) and audio.get("path"):

4115 + duration_s = sf.info(audio["path"]).duration

4116 + mm_content = {"audio_path": audio["path"]}

4117 + elif isinstance(audio, dict) and audio.get("bytes") is not None:

4118 + with BytesIO(audio["bytes"]) as audio_buffer:

4119 + y, sr = sf.read(audio_buffer, dtype="float32")

4120 + duration_s = get_audio_duration(y=y, sr=sr)

4121 + mm_content = {"audio": (y, sr)}

4122 + else:

4123 + raise ValueError(

4124 + "ASR samples must provide decoded audio arrays, "

4125 + "embedded audio bytes, or a local audio path."

4126 + )

4127 if duration_s < asr_min_audio_len_sec or duration_s > asr_max_audio_len_sec:

4128 skipped += 1

4129 continue

4130

4131 durations.append(duration_s)

4062 - mm_content = {"audio": (y, sr)}

4132 sampled_requests.append(

4133 SampleRequest(

4134 prompt=prompt,

vllm/benchmarks/lib/endpoint_request_func.py

+44 -11

@@ -445,7 +445,6 @@ async def async_request_openai_audio(

445 api_url = request_func_input.api_url

446 _validate_api_url(api_url, "OpenAI Audio API", {"transcriptions", "translations"})

447

448 - content = [{"type": "text", "text": request_func_input.prompt}]

448 payload = {

449 "model": request_func_input.model_name

450 if request_func_input.model_name

@@ -469,19 +468,26 @@ async def async_request_openai_audio(

468 buffer.seek(0)

469 return buffer

470

472 - mm_audio = request_func_input.multi_modal_content

473 - if not isinstance(mm_audio, dict) or "audio" not in mm_audio:

474 - raise TypeError("multi_modal_content must be a dict containing 'audio'")

475 - with to_bytes(*mm_audio["audio"]) as f:

471 + async def send_audio_file(

472 + audio_file: io.BytesIO | Any,

473 + *,

474 + input_audio_duration: float,

475 + filename: str | None = None,

476 + content_type: str | None = None,

477 + ) -> RequestFuncOutput:

478 form = aiohttp.FormData()

477 - form.add_field("file", f, content_type="audio/wav")

479 + add_field_kwargs: dict[str, str] = {}

480 + if filename is not None:

481 + add_field_kwargs["filename"] = filename

482 + if content_type is not None:

483 + add_field_kwargs["content_type"] = content_type

484 + form.add_field("file", audio_file, **add_field_kwargs)

485 for key, value in payload.items():

486 form.add_field(key, str(value))

487

488 output = RequestFuncOutput()

489 output.prompt_len = request_func_input.prompt_len

483 - output.input_audio_duration = soundfile.info(f).duration

484 - f.seek(0)

490 + output.input_audio_duration = input_audio_duration

491

492 generated_text = ""

493 ttft = 0.0

@@ -541,9 +547,36 @@ async def async_request_openai_audio(

547 exc_info = sys.exc_info()

548 output.error = "".join(traceback.format_exception(*exc_info))

549

544 - if pbar:

545 - pbar.update(1)

546 - return output

550 + if pbar:

551 + pbar.update(1)

552 + return output

553 +

554 + mm_audio = request_func_input.multi_modal_content

555 + if not isinstance(mm_audio, dict):

556 + raise TypeError(

557 + "multi_modal_content must be a dict containing 'audio' or 'audio_path'"

558 + )

559 + if "audio" in mm_audio:

560 + with to_bytes(*mm_audio["audio"]) as f:

561 + input_audio_duration = soundfile.info(f).duration

562 + f.seek(0)

563 + return await send_audio_file(

564 + f,

565 + input_audio_duration=input_audio_duration,

566 + filename="audio.wav",

567 + content_type="audio/wav",

568 + )

569 + if "audio_path" in mm_audio:

570 + audio_path = mm_audio["audio_path"]

571 + with open(audio_path, "rb") as f:

572 + return await send_audio_file(

573 + f,

574 + input_audio_duration=soundfile.info(audio_path).duration,

575 + filename=os.path.basename(audio_path),

576 + )

577 + raise TypeError(

578 + "multi_modal_content must be a dict containing 'audio' or 'audio_path'"

579 + )

580

581

582 async def _run_pooling_request(