@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1d5922fade

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 8, 16:30 UTC

[ASR] Fix audio benchmark and add RTFx metric (#32300)

Signed-off-by: Ekagra Ranjan <3116519+ekagra-ranjan@users.noreply.github.com> Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@gmail.com>

Ekagra Ranjan committed Feb 9, 2026 at 05:02 UTC 1d5922fadeebc5ec133dc1c88eb1e85605a5510c

4 files changed +90 -19

docs/benchmarking/cli.md

+17

@@ -30,6 +30,7 @@ th {

30 | HuggingFace-Other | ✅ | ✅ | `lmms-lab/LLaVA-OneVision-Data`, `Aeala/ShareGPT_Vicuna_unfiltered` |

31 | HuggingFace-MTBench | ✅ | ✅ | `philschmid/mt-bench` |

32 | HuggingFace-Blazedit | ✅ | ✅ | `vdaita/edit_5k_char`, `vdaita/edit_10k_char` |

33 +| HuggingFace-ASR | ✅ | ✅ | `openslr/librispeech_asr`, `facebook/voxpopuli`, `LIUM/tedlium`, `edinburghcstr/ami`, `speechcolab/gigaspeech`, `kensho/spgispeech` |

34 | Spec Bench | ✅ | ✅ | `wget https://raw.githubusercontent.com/hemingkx/Spec-Bench/refs/heads/main/data/spec_bench/question.jsonl` |

35 | Custom | ✅ | ✅ | Local file: `data.jsonl` |

36 | Custom MM | ✅ | ✅ | Local file: `mm_data.jsonl` |

@@ -299,6 +300,22 @@ vllm bench serve \

300 --blazedit-max-distance 0.99

301 ```

302

303 +`openslr/librispeech_asr`, `facebook/voxpopuli`, `LIUM/tedlium`, `edinburghcstr/ami`, `speechcolab/gigaspeech`, `kensho/spgispeech`

304 +

305 +```bash

306 +vllm bench serve \

307 + --model openai/whisper-large-v3-turbo \

308 + --backend openai-audio \

309 + --dataset-name hf \

310 + --dataset-path facebook/voxpopuli --hf-subset en --hf-split test --no-stream --trust-remote-code \

311 + --num-prompts 99999999 \

312 + --no-oversample \

313 + --endpoint /v1/audio/transcriptions \

314 + --ready-check-timeout-sec 600 \

315 + --save-result \

316 + --max-concurrency 512

317 +```

318 +

319 #### Running With Sampling Parameters

320

321 When using OpenAI-compatible backends such as `vllm`, optional sampling

vllm/benchmarks/datasets.py

+56 -18

@@ -1443,6 +1443,20 @@ def add_dataset_parser(parser: FlexibleArgumentParser):

1443 help="Maximum distance for blazedit dataset. Min: 0, Max: 1.0",

1444 )

1445

1446 + asr_group = parser.add_argument_group("asr dataset options")

1447 + asr_group.add_argument(

1448 + "--asr-max-audio-len-sec",

1449 + type=float,

1450 + default=float("inf"),

1451 + help="Maximum audio length in seconds for ASR dataset.",

1452 + )

1453 + asr_group.add_argument(

1454 + "--asr-min-audio-len-sec",

1455 + type=float,

1456 + default=0.0,

1457 + help="Minimum audio length in seconds for ASR dataset.",

1458 + )

1459 +

1460 random_group = parser.add_argument_group("random dataset options")

1461 add_random_dataset_base_args(random_group)

1462

@@ -1744,27 +1758,27 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:

1758 or args.hf_name in VisionArenaDataset.SUPPORTED_DATASET_PATHS

1759 ):

1760 dataset_class = VisionArenaDataset

1747 - args.hf_split = "train"

1761 + args.hf_split = args.hf_split if args.hf_split else "train"

1762 args.hf_subset = None

1763 elif (

1764 args.dataset_path in MMVUDataset.SUPPORTED_DATASET_PATHS

1765 or args.hf_name in MMVUDataset.SUPPORTED_DATASET_PATHS

1766 ):

1767 dataset_class = MMVUDataset

1754 - args.hf_split = "validation"

1768 + args.hf_split = args.hf_split if args.hf_split else "validation"

1769 args.hf_subset = None

1770 elif (

1771 args.dataset_path in InstructCoderDataset.SUPPORTED_DATASET_PATHS

1772 or args.hf_name in InstructCoderDataset.SUPPORTED_DATASET_PATHS

1773 ):

1774 dataset_class = InstructCoderDataset

1761 - args.hf_split = "train"

1775 + args.hf_split = args.hf_split if args.hf_split else "train"

1776 elif (

1777 args.dataset_path in MTBenchDataset.SUPPORTED_DATASET_PATHS

1778 or args.hf_name in MTBenchDataset.SUPPORTED_DATASET_PATHS

1779 ):

1780 dataset_class = MTBenchDataset

1767 - args.hf_split = "train"

1781 + args.hf_split = args.hf_split if args.hf_split else "train"

1782 elif (

1783 args.dataset_path in MultiModalConversationDataset.SUPPORTED_DATASET_PATHS

1784 or args.hf_name in MultiModalConversationDataset.SUPPORTED_DATASET_PATHS

@@ -1780,22 +1794,26 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:

1794 or args.hf_name in AIMODataset.SUPPORTED_DATASET_PATHS

1795 ):

1796 dataset_class = AIMODataset

1783 - args.hf_split = "train"

1797 + args.hf_split = args.hf_split if args.hf_split else "train"

1798 elif (

1799 args.dataset_path in NextEditPredictionDataset.SUPPORTED_DATASET_PATHS # noqa: E501

1800 or args.hf_name in NextEditPredictionDataset.SUPPORTED_DATASET_PATHS

1801 ):

1802 dataset_class = NextEditPredictionDataset

1789 - args.hf_split = "train"

1803 + args.hf_split = args.hf_split if args.hf_split else "train"

1804 elif (

1805 args.dataset_path in ASRDataset.SUPPORTED_DATASET_PATHS

1806 or args.hf_name in ASRDataset.SUPPORTED_DATASET_PATHS

1807 ):

1808 dataset_class = ASRDataset

1795 - args.hf_split = "train"

1809 + args.hf_split = args.hf_split if args.hf_split else "train"

1810 + hf_kwargs = {

1811 + "asr_min_audio_len_sec": args.asr_min_audio_len_sec,

1812 + "asr_max_audio_len_sec": args.asr_max_audio_len_sec,

1813 + }

1814 elif args.dataset_path in BlazeditDataset.SUPPORTED_DATASET_PATHS:

1815 dataset_class = BlazeditDataset

1798 - args.hf_split = "train"

1816 + args.hf_split = args.hf_split if args.hf_split else "train"

1817 hf_kwargs = {

1818 "min_distance": args.blazedit_min_distance,

1819 "max_distance": args.blazedit_max_distance,

@@ -1805,13 +1823,13 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:

1823 or args.hf_name in MLPerfDataset.SUPPORTED_DATASET_PATHS

1824 ):

1825 dataset_class = MLPerfDataset

1808 - args.hf_split = "train"

1826 + args.hf_split = args.hf_split if args.hf_split else "train"

1827 elif (

1828 args.dataset_path in MMStarDataset.SUPPORTED_DATASET_PATHS

1829 or args.hf_name in MMStarDataset.SUPPORTED_DATASET_PATHS

1830 ):

1831 dataset_class = MMStarDataset

1814 - args.hf_split = "val"

1832 + args.hf_split = args.hf_split if args.hf_split else "val"

1833 args.hf_subset = None

1834 else:

1835 supported_datasets = set(

@@ -1847,6 +1865,7 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:

1865 no_stream=args.no_stream,

1866 hf_name=args.hf_name,

1867 disable_shuffle=args.disable_shuffle,

1868 + trust_remote_code=args.trust_remote_code,

1869 ).sample(

1870 num_requests=args.num_prompts,

1871 tokenizer=tokenizer,

@@ -2405,6 +2424,7 @@ class HuggingFaceDataset(BenchmarkDataset):

2424 no_stream: bool = False,

2425 dataset_subset: str | None = None,

2426 hf_name: str | None = None,

2427 + trust_remote_code: bool = False,

2428 **kwargs,

2429 ) -> None:

2430 super().__init__(dataset_path=dataset_path, **kwargs)

@@ -2413,6 +2433,7 @@ class HuggingFaceDataset(BenchmarkDataset):

2433 self.dataset_subset = dataset_subset

2434 self.load_stream = not no_stream

2435 self.hf_name = hf_name or dataset_path

2436 + self.trust_remote_code = trust_remote_code

2437 self.load_data()

2438

2439 def load_data(self) -> None:

@@ -2422,6 +2443,7 @@ class HuggingFaceDataset(BenchmarkDataset):

2443 name=self.dataset_subset,

2444 split=self.dataset_split,

2445 streaming=self.load_stream,

2446 + trust_remote_code=self.trust_remote_code,

2447 )

2448 if not getattr(self, "disable_shuffle", False):

2449 self.data = self.data.shuffle(seed=self.random_seed)

@@ -3071,13 +3093,9 @@ class ASRDataset(HuggingFaceDataset):

3093 "kensho/spgispeech",

3094 }

3095

3074 - DEFAULT_OUTPUT_LEN = 128

3096 + DEFAULT_OUTPUT_LEN = 1024

3097 IS_MULTIMODAL = True

3098

3077 - # TODO Whisper-specific. Abstract interface when more models are supported.

3078 - TRANSCRIPTION_PREAMBLE = "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>"

3079 - skip_long_audios: bool = True

3080 -

3099 def sample(

3100 self,

3101 tokenizer: TokenizerLike,

@@ -3088,22 +3106,28 @@ class ASRDataset(HuggingFaceDataset):

3106 **kwargs,

3107 ) -> list:

3108 output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN

3091 - prompt = ASRDataset.TRANSCRIPTION_PREAMBLE

3109 + if "openai" in tokenizer.name_or_path:

3110 + prompt = "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>"

3111 + else:

3112 + prompt = ""

3113 prompt_len = len(tokenizer(prompt).input_ids)

3114 sampled_requests = []

3115 ind = 0

3116 skipped = 0

3117 + asr_min_audio_len_sec = kwargs.get("asr_min_audio_len_sec")

3118 + asr_max_audio_len_sec = kwargs.get("asr_max_audio_len_sec")

3119 + durations = []

3120 for item in self.data:

3121 if len(sampled_requests) >= num_requests:

3122 break

3123 audio = item["audio"]

3124 y, sr = audio["array"], audio["sampling_rate"]

3125 duration_s = librosa.get_duration(y=y, sr=sr)

3102 - # Whisper max supported duration

3103 - if self.skip_long_audios and duration_s > 30:

3126 + if duration_s < asr_min_audio_len_sec or duration_s > asr_max_audio_len_sec:

3127 skipped += 1

3128 continue

3129

3130 + durations.append(duration_s)

3131 mm_content = {"audio": (y, sr)}

3132 sampled_requests.append(

3133 SampleRequest(

@@ -3122,6 +3146,20 @@ class ASRDataset(HuggingFaceDataset):

3146 " what Whisper supports.",

3147 skipped,

3148 )

3149 +

3150 + logger.info("Number of audio samples: %d", len(durations))

3151 + avg_duration = sum(durations) / len(durations) if durations else 0

3152 + min_duration = min(durations) if durations else 0

3153 + max_duration = max(durations) if durations else 0

3154 + median_duration = np.median(durations) if durations else 0

3155 + logger.info(

3156 + "Audio duration statistics (s): avg=%.2f, min=%.2f, max=%.2f, median=%.2f",

3157 + avg_duration,

3158 + min_duration,

3159 + max_duration,

3160 + median_duration,

3161 + )

3162 +

3163 self.maybe_oversample_requests(

3164 sampled_requests, num_requests, request_id_prefix, no_oversample

3165 )

vllm/benchmarks/lib/endpoint_request_func.py

+6 -1

@@ -93,6 +93,7 @@ class RequestFuncOutput:

93 prompt_len: int = 0

94 error: str = ""

95 start_time: float = 0.0

96 + input_audio_duration: float = 0.0 # in seconds

99 class RequestFunc(Protocol):

@@ -422,6 +423,8 @@ async def async_request_openai_audio(

423

424 output = RequestFuncOutput()

425 output.prompt_len = request_func_input.prompt_len

426 + output.input_audio_duration = soundfile.info(f).duration

427 + f.seek(0)

428

429 generated_text = ""

430 ttft = 0.0

@@ -442,7 +445,9 @@ async def async_request_openai_audio(

445

446 messages = handler.add_chunk(chunk_bytes)

447 for message in messages:

445 - chunk = message.decode("utf-8").removeprefix("data: ")

448 + if type(message) is bytes:

449 + message = message.decode("utf-8")

450 + chunk = message.removeprefix("data: ")

451 if chunk != "[DONE]":

452 timestamp = time.perf_counter()

453 data = json.loads(chunk)

vllm/benchmarks/serve.py

+11

@@ -193,6 +193,7 @@ class BenchmarkMetrics:

193 # Max output tokens per second and concurrent requests at that peak

194 max_output_tokens_per_s: float

195 max_concurrent_requests: int

196 + rtfx: float = 0.0 # Inverse Real-Time Factor for ASR benchmarks

197

198

199 @dataclass

@@ -412,6 +413,7 @@ def calculate_metrics(

413 all_tpots: list[float] = []

414 ttfts: list[float] = []

415 e2els: list[float] = []

416 + input_audio_duration = 0.0

417 for i in range(len(outputs)):

418 if outputs[i].success:

419 output_len = outputs[i].output_tokens

@@ -439,6 +441,7 @@ def calculate_metrics(

441 itls += outputs[i].itl

442 ttfts.append(outputs[i].ttft)

443 e2els.append(outputs[i].latency)

444 + input_audio_duration += outputs[i].input_audio_duration

445 completed += 1

446 else:

447 actual_output_lens.append(0)

@@ -583,6 +586,7 @@ def calculate_metrics(

586 ],

587 max_output_tokens_per_s=max_output_tokens_per_s,

588 max_concurrent_requests=max_concurrent_requests,

589 + rtfx=input_audio_duration / dur_s,

590 )

591

592 return metrics, actual_output_lens

@@ -937,6 +941,12 @@ async def benchmark(

941 "Peak concurrent requests:", metrics.max_concurrent_requests

942 )

943 )

944 + if metrics.rtfx > 0.0:

945 + print(

946 + "{:<40} {:<10.2f}".format(

947 + "RTFx (Inverse Real-Time Factor):", metrics.rtfx

948 + )

949 + )

950 print(

951 "{:<40} {:<10.2f}".format(

952 "Total token throughput (tok/s):", metrics.total_token_throughput

@@ -963,6 +973,7 @@ async def benchmark(

973 "errors": [output.error for output in outputs],

974 "max_output_tokens_per_s": metrics.max_output_tokens_per_s,

975 "max_concurrent_requests": metrics.max_concurrent_requests,

976 + "rtfx": metrics.rtfx,

977 }

978 else:

979 result = {