@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c83c4ff815

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

[Benchmark] Add support for local hf dataset path in benchmark (#23999)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>

Jiangyun Zhu committed Sep 3, 2025 at 01:49 UTC c83c4ff815f57f57194b99828368f5785ca4e1cc

2 files changed +56 -15

benchmarks/README.md

+6 -1

@@ -110,7 +110,12 @@ become available.

110

111 🚧: to be supported

112

113 -**Note**: HuggingFace dataset's `dataset-name` should be set to `hf`

113 +**Note**: HuggingFace dataset's `dataset-name` should be set to `hf`.

114 +For local `dataset-path`, please set `hf-name` to its Hugging Face ID like

115 +

116 +```bash

117 +--dataset-path /datasets/VisionArena-Chat/ --hf-name lmarena-ai/VisionArena-Chat

118 +```

119

120 ## 🚀 Example - Online Benchmark

121

vllm/benchmarks/datasets.py

+50 -14

@@ -1227,6 +1227,16 @@ def add_dataset_parser(parser: FlexibleArgumentParser):

1227 type=str,

1228 default=None,

1229 help="Split of the HF dataset.")

1230 + hf_group.add_argument(

1231 + "--hf-name",

1232 + type=str,

1233 + default=None,

1234 + help=(

1235 + "Name of the dataset on HuggingFace "

1236 + "(e.g., 'lmarena-ai/VisionArena-Chat'). "

1237 + "Specify this if your dataset-path is a local path."

1238 + ),

1239 + )

1240 hf_group.add_argument(

1241 "--hf-output-len",

1242 type=int,

@@ -1307,28 +1317,53 @@ def get_samples(args, tokenizer) -> list[SampleRequest]:

1317 elif args.dataset_name == "hf":

1318 # all following datasets are implemented from the

1319 # HuggingFaceDataset base class

1310 - if args.dataset_path in VisionArenaDataset.SUPPORTED_DATASET_PATHS:

1320 + if (

1321 + args.dataset_path in VisionArenaDataset.SUPPORTED_DATASET_PATHS

1322 + or args.hf_name in VisionArenaDataset.SUPPORTED_DATASET_PATHS

1323 + ):

1324 dataset_class = VisionArenaDataset

1325 args.hf_split = "train"

1326 args.hf_subset = None

1314 - elif args.dataset_path in InstructCoderDataset.SUPPORTED_DATASET_PATHS:

1327 + elif (

1328 + args.dataset_path in InstructCoderDataset.SUPPORTED_DATASET_PATHS

1329 + or args.hf_name in InstructCoderDataset.SUPPORTED_DATASET_PATHS

1330 + ):

1331 dataset_class = InstructCoderDataset

1332 args.hf_split = "train"

1317 - elif args.dataset_path in MTBenchDataset.SUPPORTED_DATASET_PATHS:

1333 + elif (

1334 + args.dataset_path in MTBenchDataset.SUPPORTED_DATASET_PATHS

1335 + or args.hf_name in MTBenchDataset.SUPPORTED_DATASET_PATHS

1336 + ):

1337 dataset_class = MTBenchDataset

1338 args.hf_split = "train"

1320 - elif args.dataset_path in ConversationDataset.SUPPORTED_DATASET_PATHS:

1339 + elif (

1340 + args.dataset_path in ConversationDataset.SUPPORTED_DATASET_PATHS

1341 + or args.hf_name in ConversationDataset.SUPPORTED_DATASET_PATHS

1342 + ):

1343 dataset_class = ConversationDataset

1322 - elif args.dataset_path in AIMODataset.SUPPORTED_DATASET_PATHS:

1344 + elif (

1345 + args.dataset_path in AIMODataset.SUPPORTED_DATASET_PATHS

1346 + or args.hf_name in AIMODataset.SUPPORTED_DATASET_PATHS

1347 + ):

1348 dataset_class = AIMODataset

1349 args.hf_split = "train"

1325 - elif args.dataset_path in NextEditPredictionDataset.SUPPORTED_DATASET_PATHS: # noqa: E501

1350 + elif (

1351 + args.dataset_path

1352 + in NextEditPredictionDataset.SUPPORTED_DATASET_PATHS # noqa: E501

1353 + or args.hf_name in NextEditPredictionDataset.SUPPORTED_DATASET_PATHS

1354 + ):

1355 dataset_class = NextEditPredictionDataset

1356 args.hf_split = "train"

1328 - elif args.dataset_path in ASRDataset.SUPPORTED_DATASET_PATHS:

1357 + elif (

1358 + args.dataset_path in ASRDataset.SUPPORTED_DATASET_PATHS

1359 + or args.hf_name in ASRDataset.SUPPORTED_DATASET_PATHS

1360 + ):

1361 dataset_class = ASRDataset

1362 args.hf_split = "train"

1331 - elif args.dataset_path in MLPerfDataset.SUPPORTED_DATASET_PATHS:

1363 + elif (

1364 + args.dataset_path in MLPerfDataset.SUPPORTED_DATASET_PATHS

1365 + or args.hf_name in MLPerfDataset.SUPPORTED_DATASET_PATHS

1366 + ):

1367 dataset_class = MLPerfDataset

1368 args.hf_split = "train"

1369 else:

@@ -1358,6 +1393,7 @@ def get_samples(args, tokenizer) -> list[SampleRequest]:

1393 dataset_split=args.hf_split,

1394 random_seed=args.seed,

1395 no_stream=args.no_stream,

1396 + hf_name=args.hf_name,

1397 ).sample(

1398 num_requests=args.num_prompts,

1399 tokenizer=tokenizer,

@@ -1710,6 +1746,7 @@ class HuggingFaceDataset(BenchmarkDataset):

1746 dataset_split: str,

1747 no_stream: bool = False,

1748 dataset_subset: Optional[str] = None,

1749 + hf_name: Optional[str] = None,

1750 **kwargs,

1751 ) -> None:

1752 super().__init__(dataset_path=dataset_path, **kwargs)

@@ -1717,6 +1754,7 @@ class HuggingFaceDataset(BenchmarkDataset):

1754 self.dataset_split = dataset_split

1755 self.dataset_subset = dataset_subset

1756 self.load_stream = not no_stream

1757 + self.hf_name = hf_name or dataset_path

1758 self.load_data()

1759

1760 def load_data(self) -> None:

@@ -1827,10 +1865,9 @@ class VisionArenaDataset(HuggingFaceDataset):

1865 for i, item in enumerate(self.data):

1866 if len(sampled_requests) >= num_requests:

1867 break

1830 - parser_fn = self.SUPPORTED_DATASET_PATHS.get(self.dataset_path)

1868 + parser_fn = self.SUPPORTED_DATASET_PATHS.get(self.hf_name)

1869 if parser_fn is None:

1832 - raise ValueError(

1833 - f"Unsupported dataset path: {self.dataset_path}")

1870 + raise ValueError(f"Unsupported dataset path: {self.hf_name}")

1871 prompt = parser_fn(item)

1872 mm_content = process_image(item["images"][0])

1873 prompt_len = len(tokenizer(prompt).input_ids)

@@ -2099,10 +2136,9 @@ class NextEditPredictionDataset(HuggingFaceDataset):

2136 def sample(self, tokenizer: PreTrainedTokenizerBase, num_requests: int,

2137 request_id_prefix: str = "",

2138 **kwargs):

2102 - formatting_prompt_func = self.MAPPING_PROMPT_FUNCS.get(

2103 - self.dataset_path)

2139 + formatting_prompt_func = self.MAPPING_PROMPT_FUNCS.get(self.hf_name)

2140 if formatting_prompt_func is None:

2105 - raise ValueError(f"Unsupported dataset path: {self.dataset_path}")

2141 + raise ValueError(f"Unsupported dataset path: {self.hf_name}")

2142 samples = []

2143 for i, sample in enumerate(self.data):

2144 sample = formatting_prompt_func(sample)