A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Benchmarks] Add MMVU video dataset support and clean up deprecated datasets (#24719)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Isotr0py committed
Sep 17, 2025 at 11:29 UTC
5a411ef6c446f8fb08311a385e16e13bacf44bc5
3 files changed
+65
-1290
benchmarks/benchmark_dataset.py
deleted
-1288
@@ -1,1288 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-"""
4
-This module defines a framework for sampling benchmark requests from various
5
-datasets. Each dataset subclass of BenchmarkDataset must implement sample
6
-generation. Supported dataset types include:
7
- - ShareGPT
8
- - Random (synthetic)
9
- - Sonnet
10
- - BurstGPT
11
- - HuggingFace
12
- - VisionArena
13
-"""
14
-
15
-import base64
16
-import io
17
-import json
18
-import logging
19
-import random
20
-from abc import ABC, abstractmethod
21
-from collections.abc import Mapping
22
-from copy import deepcopy
23
-from dataclasses import dataclass
24
-from functools import cache
25
-from io import BytesIO
26
-from typing import Any, Callable, Optional, Union
27
-
28
-import numpy as np
29
-import pandas as pd
30
-from datasets import load_dataset
31
-from PIL import Image
32
-from transformers import PreTrainedTokenizerBase
33
-
34
-from vllm.lora.request import LoRARequest
35
-from vllm.lora.utils import get_adapter_absolute_path
36
-from vllm.multimodal import MultiModalDataDict
37
-from vllm.multimodal.image import convert_image_mode
38
-from vllm.transformers_utils.tokenizer import AnyTokenizer, get_lora_tokenizer
39
-
40
-logger = logging.getLogger(__name__)
41
-
42
-# -----------------------------------------------------------------------------
43
-# Data Classes
44
-# -----------------------------------------------------------------------------
45
-
46
-
47
-@dataclass
48
-class SampleRequest:
49
- """
50
- Represents a single inference request for benchmarking.
51
- """
52
-
53
- prompt: Union[str, Any]
54
- prompt_len: int
55
- expected_output_len: int
56
- multi_modal_data: Optional[Union[MultiModalDataDict, dict, list[dict]]] = None
57
- lora_request: Optional[LoRARequest] = None
58
- request_id: Optional[str] = None
59
-
60
-
61
-# -----------------------------------------------------------------------------
62
-# Benchmark Dataset Base Class
63
-# -----------------------------------------------------------------------------
64
-
65
-
66
-class BenchmarkDataset(ABC):
67
- DEFAULT_SEED = 0
68
- IS_MULTIMODAL = False
69
-
70
- def __init__(
71
- self,
72
- dataset_path: Optional[str] = None,
73
- random_seed: int = DEFAULT_SEED,
74
- ) -> None:
75
- """
76
- Initialize the BenchmarkDataset with an optional dataset path and random
77
- seed. Args:
78
- dataset_path (Optional[str]): Path to the dataset. If None, it
79
- indicates that a default or random dataset might be used.
80
- random_seed (int): Seed value for reproducible shuffling or
81
- sampling. Defaults to DEFAULT_SEED.
82
- """
83
- self.dataset_path = dataset_path
84
- # Set the random seed, ensuring that a None value is replaced with the
85
- # default seed.
86
- self.random_seed = random_seed if random_seed is not None else self.DEFAULT_SEED
87
- self.data = None
88
-
89
- def apply_multimodal_chat_transformation(
90
- self, prompt: str, mm_content: Optional[MultiModalDataDict] = None
91
- ) -> list[dict]:
92
- """
93
- Transform a prompt and optional multimodal content into a chat format.
94
- This method is used for chat models that expect a specific conversation
95
- format.
96
- """
97
- content = [{"text": prompt, "type": "text"}]
98
- if mm_content is not None:
99
- content.append(mm_content)
100
- return [{"role": "user", "content": content}]
101
-
102
- def load_data(self) -> None:
103
- """
104
- Load data from the dataset path into self.data.
105
-
106
- This method must be overridden by subclasses since the method to load
107
- data will vary depending on the dataset format and source.
108
-
109
- Raises:
110
- NotImplementedError: If a subclass does not implement this method.
111
- """
112
- # TODO (jenniferzhao): add support for downloading data
113
- raise NotImplementedError("load_data must be implemented in subclasses.")
114
-
115
- def get_random_lora_request(
116
- self,
117
- tokenizer: PreTrainedTokenizerBase,
118
- max_loras: Optional[int] = None,
119
- lora_path: Optional[str] = None,
120
- ) -> tuple[Optional[LoRARequest], AnyTokenizer]:
121
- """
122
- Optionally select a random LoRA request and return its associated
123
- tokenizer.
124
-
125
- This method is used when LoRA parameters are provided. It randomly
126
- selects a LoRA based on max_loras and retrieves a cached tokenizer for
127
- that LoRA if available. Otherwise, it returns the base tokenizer.
128
-
129
- Args:
130
- tokenizer (PreTrainedTokenizerBase): The base tokenizer to use if no
131
- LoRA is selected. max_loras (Optional[int]): The maximum number of
132
- LoRAs available. If None, LoRA is not used. lora_path
133
- (Optional[str]): Path to the LoRA parameters on disk. If None, LoRA
134
- is not used.
135
-
136
- Returns:
137
- tuple[Optional[LoRARequest], AnyTokenizer]: A tuple where the first
138
- element is a LoRARequest (or None if not applicable) and the second
139
- element is the tokenizer associated with the LoRA request (or the
140
- base tokenizer).
141
- """
142
- if max_loras is None or lora_path is None:
143
- return None, tokenizer
144
-
145
- # Generate a random LoRA ID in the range [1, max_loras].
146
- lora_id = random.randint(1, max_loras)
147
- lora_request = LoRARequest(
148
- lora_name=str(lora_id),
149
- lora_int_id=lora_id,
150
- lora_path=lora_path_on_disk(lora_path),
151
- )
152
- if lora_id not in lora_tokenizer_cache:
153
- lora_tokenizer_cache[lora_id] = get_lora_tokenizer(lora_request)
154
- # Return lora_request and the cached tokenizer if available; otherwise,
155
- # return the base tokenizer
156
- return lora_request, lora_tokenizer_cache[lora_id] or tokenizer
157
-
158
- @abstractmethod
159
- def sample(
160
- self,
161
- tokenizer: PreTrainedTokenizerBase,
162
- num_requests: int,
163
- request_id_prefix: str = "",
164
- ) -> list[SampleRequest]:
165
- """
166
- Abstract method to generate sample requests from the dataset.
167
-
168
- Subclasses must override this method to implement dataset-specific logic
169
- for generating a list of SampleRequest objects.
170
-
171
- Args:
172
- tokenizer (PreTrainedTokenizerBase): The tokenizer to be used
173
- for processing the dataset's text.
174
- num_requests (int): The number of sample requests to generate.
175
- request_id_prefix (str) The prefix of request_id.
176
-
177
- Returns:
178
- list[SampleRequest]: A list of sample requests generated from the
179
- dataset.
180
- """
181
- raise NotImplementedError("sample must be implemented in subclasses.")
182
-
183
- def maybe_oversample_requests(
184
- self,
185
- requests: list[SampleRequest],
186
- num_requests: int,
187
- request_id_prefix: str = "",
188
- ) -> None:
189
- """
190
- Oversamples the list of requests if its size is less than the desired
191
- number.
192
-
193
- Args:
194
- requests (List[SampleRequest]): The current list of sampled
195
- requests.
196
- num_requests (int): The target number of requests.
197
- request_id_prefix (str) The prefix of the request ids.
198
- """
199
- if len(requests) < num_requests:
200
- random.seed(self.random_seed)
201
- additional = deepcopy(
202
- random.choices(requests, k=num_requests - len(requests))
203
- )
204
- for i in range(len(additional)):
205
- req = additional[i]
206
- req.request_id = request_id_prefix + str(len(requests) + i)
207
- requests.extend(additional)
208
- logger.info("Oversampled requests to reach %d total samples.", num_requests)
209
-
210
-
211
-# -----------------------------------------------------------------------------
212
-# Utility Functions and Global Caches
213
-# -----------------------------------------------------------------------------
214
-
215
-
216
-def is_valid_sequence(
217
- prompt_len: int,
218
- output_len: int,
219
- min_len: int = 4,
220
- max_prompt_len: int = 1024,
221
- max_total_len: int = 2048,
222
- skip_min_output_len_check: bool = False,
223
-) -> bool:
224
- """
225
- Validate a sequence based on prompt and output lengths.
226
-
227
- Default pruning criteria are copied from the original `sample_hf_requests`
228
- and `sample_sharegpt_requests` functions in benchmark_serving.py, as well as
229
- from `sample_requests` in benchmark_throughput.py.
230
- """
231
- # Check for invalid conditions
232
- prompt_too_short = prompt_len < min_len
233
- output_too_short = (not skip_min_output_len_check) and (output_len < min_len)
234
- prompt_too_long = prompt_len > max_prompt_len
235
- combined_too_long = (prompt_len + output_len) > max_total_len
236
-
237
- # Return True if none of the invalid conditions are met
238
- return not (
239
- prompt_too_short or output_too_short or prompt_too_long or combined_too_long
240
- )
241
-
242
-
243
-@cache
244
-def lora_path_on_disk(lora_path: str) -> str:
245
- return get_adapter_absolute_path(lora_path)
246
-
247
-
248
-# Global cache for LoRA tokenizers.
249
-lora_tokenizer_cache: dict[int, AnyTokenizer] = {}
250
-
251
-
252
-def process_image(image: Any) -> Mapping[str, Any]:
253
- """
254
- Process a single image input and return a multimedia content dictionary.
255
-
256
- Supports three input types:
257
-
258
- 1. Dictionary with raw image bytes: - Expects a dict with a 'bytes' key
259
- containing raw image data. - Loads the bytes as a PIL.Image.Image.
260
-
261
- 2. PIL.Image.Image input: - Converts the image to RGB. - Saves the image as
262
- a JPEG in memory. - Encodes the JPEG data as a base64 string. - Returns
263
- a dictionary with the image as a base64 data URL.
264
-
265
- 3. String input: - Treats the string as a URL or local file path. -
266
- Prepends "file://" if the string doesn't start with "http://" or
267
- "file://". - Returns a dictionary with the image URL.
268
-
269
- Raises:
270
- ValueError: If the input is not a supported type.
271
- """
272
- if isinstance(image, dict) and "bytes" in image:
273
- image = Image.open(BytesIO(image["bytes"]))
274
- if isinstance(image, Image.Image):
275
- image = convert_image_mode(image, "RGB")
276
- with io.BytesIO() as image_data:
277
- image.save(image_data, format="JPEG")
278
- image_base64 = base64.b64encode(image_data.getvalue()).decode("utf-8")
279
- return {
280
- "type": "image_url",
281
- "image_url": {"url": f"data:image/jpeg;base64,{image_base64}"},
282
- }
283
-
284
- if isinstance(image, str):
285
- image_url = (
286
- image if image.startswith(("http://", "file://")) else f"file://{image}"
287
- )
288
- return {"type": "image_url", "image_url": {"url": image_url}}
289
-
290
- raise ValueError(
291
- f"Invalid image input {image}. Must be a PIL.Image.Image"
292
- " or str or dictionary with raw image bytes."
293
- )
294
-
295
-
296
-def process_video(video: Any) -> Mapping[str, Any]:
297
- """
298
- Process a single video input and return a multimedia content dictionary.
299
-
300
- Supports the following input types:
301
-
302
- 1. Dictionary with raw video bytes: - Expects a dict with a 'bytes' key
303
- containing raw video data.
304
-
305
- 2. String input: - Treats the string as a URL or local file path. -
306
- Prepends "file://" if the string doesn't start with "http://" or
307
- "file://". - Returns a dictionary with the image URL.
308
-
309
- Raises:
310
- ValueError: If the input is not a supported type.
311
- """
312
- if isinstance(video, dict) and "bytes" in video:
313
- video_bytes = video["bytes"]
314
- video_base64 = base64.b64encode(video_bytes).decode("utf-8")
315
- return {
316
- "type": "video_url",
317
- "video_url": {"url": f"data:video/mp4;base64,{video_base64}"},
318
- }
319
-
320
- if isinstance(video, str):
321
- video_url = (
322
- video if video.startswith(("http://", "file://")) else f"file://{video}"
323
- )
324
- return {"type": "video_url", "video_url": {"url": video_url}}
325
-
326
- raise ValueError(
327
- f"Invalid video input {video}. Must be a string of local path/remote url, or a dictionary with raw video bytes in the form of `{{'bytes': raw_video_bytes}}`." # noqa: E501
328
- )
329
-
330
-
331
-# -----------------------------------------------------------------------------
332
-# Random Dataset Implementation (Synthetic Data)
333
-# -----------------------------------------------------------------------------
334
-
335
-
336
-class RandomDataset(BenchmarkDataset):
337
- # Default values copied from benchmark_serving.py for the random dataset.
338
- DEFAULT_PREFIX_LEN = 0
339
- DEFAULT_RANGE_RATIO = 0.0
340
- DEFAULT_INPUT_LEN = 1024
341
- DEFAULT_OUTPUT_LEN = 128
342
-
343
- def __init__(
344
- self,
345
- **kwargs,
346
- ) -> None:
347
- super().__init__(**kwargs)
348
-
349
- def sample(
350
- self,
351
- tokenizer: PreTrainedTokenizerBase,
352
- num_requests: int,
353
- prefix_len: int = DEFAULT_PREFIX_LEN,
354
- range_ratio: float = DEFAULT_RANGE_RATIO,
355
- input_len: int = DEFAULT_INPUT_LEN,
356
- output_len: int = DEFAULT_OUTPUT_LEN,
357
- request_id_prefix: str = "",
358
- **kwargs,
359
- ) -> list[SampleRequest]:
360
- # Enforce range_ratio < 1
361
- assert range_ratio < 1.0, (
362
- "random_range_ratio must be < 1.0 to ensure a valid sampling range"
363
- )
364
-
365
- vocab_size = tokenizer.vocab_size
366
- num_special_tokens = tokenizer.num_special_tokens_to_add()
367
- real_input_len = input_len - num_special_tokens
368
-
369
- prefix_token_ids = (
370
- np.random.randint(0, vocab_size, size=prefix_len).tolist()
371
- if prefix_len > 0
372
- else []
373
- )
374
-
375
- # New sampling logic: [X * (1 - b), X * (1 + b)]
376
- input_low = int(real_input_len * (1 - range_ratio))
377
- input_high = int(real_input_len * (1 + range_ratio))
378
- output_low = int(output_len * (1 - range_ratio))
379
- # Ensure the lower bound for output length is at least 1 to prevent
380
- # sampling 0 tokens, which can cause request failures.
381
- output_low = max(output_low, 1)
382
- output_high = int(output_len * (1 + range_ratio))
383
-
384
- # Add logging for debugging
385
- logger.info("Sampling input_len from [%s, %s]", input_low, input_high)
386
- logger.info("Sampling output_len from [%s, %s]", output_low, output_high)
387
-
388
- input_lens = np.random.randint(input_low, input_high + 1, size=num_requests)
389
- output_lens = np.random.randint(output_low, output_high + 1, size=num_requests)
390
- offsets = np.random.randint(0, vocab_size, size=num_requests)
391
-
392
- requests = []
393
- for i in range(num_requests):
394
- inner_seq = (
395
- (offsets[i] + i + np.arange(input_lens[i])) % vocab_size
396
- ).tolist()
397
- token_sequence = prefix_token_ids + inner_seq
398
- prompt = tokenizer.decode(token_sequence)
399
- # After decoding the prompt we have to encode and decode it again.
400
- # This is done because in some cases N consecutive tokens
401
- # give a string tokenized into != N number of tokens.
402
- # For example for GPT2Tokenizer:
403
- # [6880, 6881] -> ['Ġcalls', 'here'] ->
404
- # [1650, 939, 486] -> ['Ġcall', 'sh', 'ere']
405
- # To avoid uncontrolled change of the prompt length,
406
- # the encoded sequence is truncated before being decoded again.
407
- total_input_len = prefix_len + int(input_lens[i])
408
- re_encoded_sequence = tokenizer.encode(prompt, add_special_tokens=False)[
409
- :total_input_len
410
- ]
411
- prompt = tokenizer.decode(re_encoded_sequence)
412
- total_input_len = len(re_encoded_sequence)
413
- requests.append(
414
- SampleRequest(
415
- prompt=prompt,
416
- prompt_len=total_input_len,
417
- expected_output_len=int(output_lens[i]),
418
- request_id=request_id_prefix + str(i),
419
- )
420
- )
421
-
422
- return requests
423
-
424
-
425
-# -----------------------------------------------------------------------------
426
-# ShareGPT Dataset Implementation
427
-# -----------------------------------------------------------------------------
428
-
429
-
430
-class ShareGPTDataset(BenchmarkDataset):
431
- """
432
- Implements the ShareGPT dataset. Loads data from a JSON file and generates
433
- sample requests based on conversation turns.
434
- """
435
-
436
- def __init__(self, **kwargs) -> None:
437
- super().__init__(**kwargs)
438
- self.load_data()
439
-
440
- def load_data(self) -> None:
441
- if self.dataset_path is None:
442
- raise ValueError("dataset_path must be provided for loading data.")
443
-
444
- with open(self.dataset_path, encoding="utf-8") as f:
445
- self.data = json.load(f)
446
- # Filter entries with at least two conversation turns.
447
- self.data = [
448
- entry
449
- for entry in self.data
450
- if "conversations" in entry and len(entry["conversations"]) >= 2
451
- ]
452
- random.seed(self.random_seed)
453
- random.shuffle(self.data)
454
-
455
- def sample(
456
- self,
457
- tokenizer: PreTrainedTokenizerBase,
458
- num_requests: int,
459
- lora_path: Optional[str] = None,
460
- max_loras: Optional[int] = None,
461
- output_len: Optional[int] = None,
462
- enable_multimodal_chat: bool = False,
463
- request_id_prefix: str = "",
464
- **kwargs,
465
- ) -> list:
466
- samples: list = []
467
- ind = 0
468
- for entry in self.data:
469
- if len(samples) >= num_requests:
470
- break
471
- prompt, completion = (
472
- entry["conversations"][0]["value"],
473
- entry["conversations"][1]["value"],
474
- )
475
-
476
- lora_request, tokenizer = self.get_random_lora_request(
477
- tokenizer=tokenizer, max_loras=max_loras, lora_path=lora_path
478
- )
479
- prompt_ids = tokenizer(prompt).input_ids
480
- completion_ids = tokenizer(completion).input_ids
481
- prompt_len = len(prompt_ids)
482
- new_output_len = len(completion_ids) if output_len is None else output_len
483
- if not is_valid_sequence(
484
- prompt_len,
485
- new_output_len,
486
- skip_min_output_len_check=output_len is not None,
487
- ):
488
- continue
489
- if image_path := entry.get("image"):
490
- mm_content = process_image(image_path)
491
- elif video_path := entry.get("video"):
492
- mm_content = process_video(video_path)
493
- else:
494
- mm_content = None
495
- if enable_multimodal_chat:
496
- prompt = self.apply_multimodal_chat_transformation(prompt, mm_content)
497
- samples.append(
498
- SampleRequest(
499
- prompt=prompt,
500
- prompt_len=prompt_len,
501
- expected_output_len=new_output_len,
502
- lora_request=lora_request,
503
- multi_modal_data=mm_content,
504
- request_id=request_id_prefix + str(ind),
505
- )
506
- )
507
- ind += 1
508
- self.maybe_oversample_requests(samples, num_requests, request_id_prefix)
509
- return samples
510
-
511
-
512
-# -----------------------------------------------------------------------------
513
-# Custom Dataset Implementation
514
-# -----------------------------------------------------------------------------
515
-
516
-
517
-class CustomDataset(BenchmarkDataset):
518
- """
519
- Implements the Custom dataset. Loads data from a JSONL file and generates
520
- sample requests based on conversation turns. E.g.,
521
- ```
522
- {"prompt": "What is the capital of India?"}
523
- {"prompt": "What is the capital of Iran?"}
524
- {"prompt": "What is the capital of China?"}
525
- ```
526
- """
527
-
528
- def __init__(self, **kwargs) -> None:
529
- super().__init__(**kwargs)
530
- self.load_data()
531
-
532
- def load_data(self) -> None:
533
- if self.dataset_path is None:
534
- raise ValueError("dataset_path must be provided for loading data.")
535
-
536
- # self.data will be a list of dictionaries
537
- # e.g., [{"prompt": "What is the capital of India?"}, ...]
538
- # This will be the standardized format which load_data()
539
- # has to convert into depending on the filetype of dataset_path.
540
- # sample() will assume this standardized format of self.data
541
- self.data = []
542
-
543
- # Load the JSONL file
544
- if self.dataset_path.endswith(".jsonl"):
545
- jsonl_data = pd.read_json(path_or_buf=self.dataset_path, lines=True)
546
-
547
- # check if the JSONL file has a 'prompt' column
548
- if "prompt" not in jsonl_data.columns:
549
- raise ValueError("JSONL file must contain a 'prompt' column.")
550
-
551
- # Convert each row to a dictionary and append to self.data
552
- # This will convert the DataFrame to a list of dictionaries
553
- # where each dictionary corresponds to a row in the DataFrame.
554
- # This is the standardized format we want for self.data
555
- for _, row in jsonl_data.iterrows():
556
- self.data.append(row.to_dict())
557
- else:
558
- raise NotImplementedError(
559
- "Only JSONL format is supported for CustomDataset."
560
- )
561
-
562
- random.seed(self.random_seed)
563
- random.shuffle(self.data)
564
-
565
- def sample(
566
- self,
567
- tokenizer: PreTrainedTokenizerBase,
568
- num_requests: int,
569
- lora_path: Optional[str] = None,
570
- max_loras: Optional[int] = None,
571
- output_len: Optional[int] = None,
572
- enable_multimodal_chat: bool = False,
573
- skip_chat_template: bool = False,
574
- request_id_prefix: str = "",
575
- **kwargs,
576
- ) -> list:
577
- sampled_requests = []
578
- for i, item in enumerate(self.data):
579
- if len(sampled_requests) >= num_requests:
580
- break
581
- prompt = item["prompt"]
582
-
583
- # apply template
584
- if not skip_chat_template:
585
- prompt = tokenizer.apply_chat_template(
586
- [{"role": "user", "content": prompt}],
587
- add_generation_prompt=True,
588
- tokenize=False,
589
- )
590
-
591
- prompt_len = len(tokenizer(prompt).input_ids)
592
- sampled_requests.append(
593
- SampleRequest(
594
- prompt=prompt,
595
- prompt_len=prompt_len,
596
- expected_output_len=output_len,
597
- request_id=request_id_prefix + str(i),
598
- )
599
- )
600
- self.maybe_oversample_requests(
601
- sampled_requests, num_requests, request_id_prefix
602
- )
603
-
604
- return sampled_requests
605
-
606
-
607
-# -----------------------------------------------------------------------------
608
-# Sonnet Dataset Implementation
609
-# -----------------------------------------------------------------------------
610
-
611
-
612
-class SonnetDataset(BenchmarkDataset):
613
- """
614
- Simplified implementation of the Sonnet dataset. Loads poem lines from a
615
- text file and generates sample requests. Default values here copied from
616
- `benchmark_serving.py` for the sonnet dataset.
617
- """
618
-
619
- DEFAULT_PREFIX_LEN = 200
620
- DEFAULT_INPUT_LEN = 550
621
- DEFAULT_OUTPUT_LEN = 150
622
-
623
- def __init__(
624
- self,
625
- **kwargs,
626
- ) -> None:
627
- super().__init__(**kwargs)
628
- self.load_data()
629
-
630
- def load_data(self) -> None:
631
- if not self.dataset_path:
632
- raise ValueError("dataset_path must be provided.")
633
- with open(self.dataset_path, encoding="utf-8") as f:
634
- self.data = f.readlines()
635
-
636
- def sample(
637
- self,
638
- tokenizer,
639
- num_requests: int,
640
- prefix_len: int = DEFAULT_PREFIX_LEN,
641
- input_len: int = DEFAULT_INPUT_LEN,
642
- output_len: int = DEFAULT_OUTPUT_LEN,
643
- return_prompt_formatted: bool = False,
644
- request_id_prefix: str = "",
645
- **kwargs,
646
- ) -> list:
647
- # Calculate average token length for a poem line.
648
- tokenized_lines = [tokenizer(line).input_ids for line in self.data]
649
- avg_len = sum(len(tokens) for tokens in tokenized_lines) / len(tokenized_lines)
650
-
651
- # Build the base prompt.
652
- base_prompt = "Pick as many lines as you can from these poem lines:\n"
653
- base_msg = [{"role": "user", "content": base_prompt}]
654
- base_fmt = tokenizer.apply_chat_template(
655
- base_msg, add_generation_prompt=True, tokenize=False
656
- )
657
- base_offset = len(tokenizer(base_fmt).input_ids)
658
- if input_len <= base_offset:
659
- raise ValueError(
660
- f"'input_len' must be higher than the base prompt length "
661
- f"({base_offset})."
662
- )
663
-
664
- # Determine how many poem lines to use.
665
- num_input_lines = round((input_len - base_offset) / avg_len)
666
- num_prefix_lines = max(round((prefix_len - base_offset) / avg_len), 0)
667
- prefix_lines = self.data[:num_prefix_lines]
668
-
669
- samples = []
670
- ind = 0
671
- while len(samples) < num_requests:
672
- extra_lines = random.choices(
673
- self.data, k=num_input_lines - num_prefix_lines
674
- )
675
- prompt = f"{base_prompt}{''.join(prefix_lines + extra_lines)}"
676
- msg = [{"role": "user", "content": prompt}]
677
- prompt_formatted = tokenizer.apply_chat_template(
678
- msg, add_generation_prompt=True, tokenize=False
679
- )
680
- prompt_len = len(tokenizer(prompt_formatted).input_ids)
681
-
682
- if prompt_len <= input_len:
683
- samples.append(
684
- SampleRequest(
685
- prompt=prompt_formatted if return_prompt_formatted else prompt,
686
- prompt_len=prompt_len,
687
- expected_output_len=output_len,
688
- request_id=request_id_prefix + str(ind),
689
- )
690
- )
691
- ind += 1
692
- return samples
693
-
694
-
695
-# -----------------------------------------------------------------------------
696
-# BurstGPT Dataset Implementation
697
-# -----------------------------------------------------------------------------
698
-
699
-
700
-class BurstGPTDataset(BenchmarkDataset):
701
- """
702
- Implements the BurstGPT dataset. Loads data from a CSV file and generates
703
- sample requests based on synthetic prompt generation. Only rows with Model
704
- "GPT-4" and positive response tokens are used.
705
- """
706
-
707
- def __init__(self, **kwargs) -> None:
708
- super().__init__(**kwargs)
709
- self.load_data()
710
-
711
- def load_data(
712
- self,
713
- ):
714
- if self.dataset_path is None:
715
- raise ValueError("dataset_path must be provided for loading data.")
716
-
717
- df = pd.read_csv(self.dataset_path)
718
- # Filter to keep only GPT-4 rows.
719
- gpt4_df = df[df["Model"] == "GPT-4"]
720
- # Remove failed requests (where Response tokens is 0 or less).
721
- gpt4_df = gpt4_df[gpt4_df["Response tokens"] > 0]
722
- # Sample the desired number of rows.
723
- self.data = gpt4_df
724
-
725
- def _sample_loaded_data(self, num_requests: int) -> list:
726
- if num_requests <= len(self.data):
727
- data = self.data.sample(n=num_requests, random_state=self.random_seed)
728
- else:
729
- data = self.data.sample(
730
- n=num_requests,
731
- random_state=self.random_seed,
732
- replace=True,
733
- )
734
- # Convert the dataframe to a list of lists.
735
- return data.values.tolist()
736
-
737
- def sample(
738
- self,
739
- tokenizer: PreTrainedTokenizerBase,
740
- num_requests: int,
741
- max_loras: Optional[int] = None,
742
- lora_path: Optional[str] = None,
743
- request_id_prefix: str = "",
744
- **kwargs,
745
- ) -> list[SampleRequest]:
746
- samples = []
747
- data = self._sample_loaded_data(num_requests=num_requests)
748
- for i in range(num_requests):
749
- input_len = int(data[i][2])
750
- output_len = int(data[i][3])
751
- lora_req, tokenizer = self.get_random_lora_request(
752
- tokenizer=tokenizer, max_loras=max_loras, lora_path=lora_path
753
- )
754
- vocab_size = tokenizer.vocab_size
755
- # Generate a synthetic prompt: a list of token IDs computed as (i +
756
- # j) modulo vocab_size.
757
- token_ids = [(i + j) % vocab_size for j in range(input_len)]
758
- prompt = tokenizer.decode(token_ids)
759
- samples.append(
760
- SampleRequest(
761
- prompt=prompt,
762
- prompt_len=input_len,
763
- expected_output_len=output_len,
764
- lora_request=lora_req,
765
- request_id=request_id_prefix + str(i),
766
- )
767
- )
768
- return samples
769
-
770
-
771
-# -----------------------------------------------------------------------------
772
-# HuggingFace Dataset Base Implementation
773
-# -----------------------------------------------------------------------------
774
-class HuggingFaceDataset(BenchmarkDataset):
775
- """Base class for datasets hosted on HuggingFace."""
776
-
777
- SUPPORTED_DATASET_PATHS: Union[set[str], dict[str, Callable]] = set()
778
-
779
- def __init__(
780
- self,
781
- dataset_path: str,
782
- dataset_split: str,
783
- no_stream: bool = False,
784
- dataset_subset: Optional[str] = None,
785
- **kwargs,
786
- ) -> None:
787
- super().__init__(dataset_path=dataset_path, **kwargs)
788
-
789
- self.dataset_split = dataset_split
790
- self.dataset_subset = dataset_subset
791
- self.load_stream = not no_stream
792
- self.load_data()
793
-
794
- def load_data(self) -> None:
795
- """Load data from HuggingFace datasets."""
796
- self.data = load_dataset(
797
- self.dataset_path,
798
- name=self.dataset_subset,
799
- split=self.dataset_split,
800
- streaming=self.load_stream,
801
- )
802
- self.data = self.data.shuffle(seed=self.random_seed)
803
-
804
-
805
-# -----------------------------------------------------------------------------
806
-# Conversation Dataset Implementation
807
-# -----------------------------------------------------------------------------
808
-
809
-
810
-class ConversationDataset(HuggingFaceDataset):
811
- """Dataset for conversation data with multimodal support."""
812
-
813
- SUPPORTED_DATASET_PATHS = {
814
- "lmms-lab/LLaVA-OneVision-Data",
815
- "Aeala/ShareGPT_Vicuna_unfiltered",
816
- }
817
- IS_MULTIMODAL = True
818
-
819
- def sample(
820
- self,
821
- tokenizer: PreTrainedTokenizerBase,
822
- num_requests: int,
823
- output_len: Optional[int] = None,
824
- enable_multimodal_chat: bool = False,
825
- request_id_prefix: str = "",
826
- **kwargs,
827
- ) -> list:
828
- # Filter examples with at least 2 conversations
829
- filtered_data = self.data.filter(lambda x: len(x["conversations"]) >= 2)
830
- sampled_requests = []
831
- dynamic_output = output_len is None
832
- ind = 0
833
-
834
- for item in filtered_data:
835
- if len(sampled_requests) >= num_requests:
836
- break
837
- conv = item["conversations"]
838
- prompt, completion = conv[0]["value"], conv[1]["value"]
839
-
840
- prompt_ids = tokenizer(prompt).input_ids
841
- completion_ids = tokenizer(completion).input_ids
842
- prompt_len = len(prompt_ids)
843
- completion_len = len(completion_ids)
844
- output_len = completion_len if dynamic_output else output_len
845
- assert isinstance(output_len, int) and output_len > 0
846
- if dynamic_output and not is_valid_sequence(prompt_len, completion_len):
847
- continue
848
- mm_content = process_image(item["image"]) if "image" in item else None
849
- if enable_multimodal_chat:
850
- # Note: when chat is enabled the request prompt_len is no longer
851
- # accurate and we will be using request output to count the
852
- # actual prompt len and output len
853
- prompt = self.apply_multimodal_chat_transformation(prompt, mm_content)
854
- sampled_requests.append(
855
- SampleRequest(
856
- prompt=prompt,
857
- prompt_len=prompt_len,
858
- expected_output_len=output_len,
859
- multi_modal_data=mm_content,
860
- request_id=request_id_prefix + str(ind),
861
- )
862
- )
863
- ind += 1
864
- self.maybe_oversample_requests(
865
- sampled_requests, num_requests, request_id_prefix
866
- )
867
- return sampled_requests
868
-
869
-
870
-# -----------------------------------------------------------------------------
871
-# Vision Arena Dataset Implementation
872
-# -----------------------------------------------------------------------------
873
-
874
-
875
-class VisionArenaDataset(HuggingFaceDataset):
876
- """
877
- Vision Arena Dataset.
878
- """
879
-
880
- DEFAULT_OUTPUT_LEN = 128
881
- SUPPORTED_DATASET_PATHS = {
882
- "lmarena-ai/VisionArena-Chat": lambda x: x["conversation"][0][0]["content"],
883
- "lmarena-ai/vision-arena-bench-v0.1": lambda x: x["turns"][0][0]["content"],
884
- }
885
- IS_MULTIMODAL = True
886
-
887
- def sample(
888
- self,
889
- tokenizer: PreTrainedTokenizerBase,
890
- num_requests: int,
891
- output_len: Optional[int] = None,
892
- enable_multimodal_chat: bool = False,
893
- request_id_prefix: str = "",
894
- **kwargs,
895
- ) -> list:
896
- output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
897
- sampled_requests = []
898
- for i, item in enumerate(self.data):
899
- if len(sampled_requests) >= num_requests:
900
- break
901
- parser_fn = self.SUPPORTED_DATASET_PATHS.get(self.dataset_path)
902
- if parser_fn is None:
903
- raise ValueError(f"Unsupported dataset path: {self.dataset_path}")
904
- prompt = parser_fn(item)
905
- mm_content = process_image(item["images"][0])
906
- prompt_len = len(tokenizer(prompt).input_ids)
907
- if enable_multimodal_chat:
908
- # Note: when chat is enabled the request prompt_len is no longer
909
- # accurate and we will be using request output to count the
910
- # actual prompt len
911
- prompt = self.apply_multimodal_chat_transformation(prompt, mm_content)
912
- sampled_requests.append(
913
- SampleRequest(
914
- prompt=prompt,
915
- prompt_len=prompt_len,
916
- expected_output_len=output_len,
917
- multi_modal_data=mm_content,
918
- request_id=request_id_prefix + str(i),
919
- )
920
- )
921
- self.maybe_oversample_requests(
922
- sampled_requests, num_requests, request_id_prefix
923
- )
924
- return sampled_requests
925
-
926
-
927
-# -----------------------------------------------------------------------------
928
-# Instruct Coder Dataset Implementation
929
-# -----------------------------------------------------------------------------
930
-
931
-
932
-class InstructCoderDataset(HuggingFaceDataset):
933
- """
934
- InstructCoder Dataset.
935
- https://huggingface.co/datasets/likaixin/InstructCoder
936
-
937
- InstructCoder is the dataset designed for general code editing. It consists
938
- of 114,239 instruction-input-output triplets, and covers multiple distinct
939
- code editing scenario.
940
- """
941
-
942
- DEFAULT_OUTPUT_LEN = 200 # this is the average default output length
943
- SUPPORTED_DATASET_PATHS = {
944
- "likaixin/InstructCoder",
945
- }
946
-
947
- def sample(
948
- self,
949
- tokenizer: PreTrainedTokenizerBase,
950
- num_requests: int,
951
- output_len: Optional[int] = None,
952
- enable_multimodal_chat: bool = False,
953
- request_id_prefix: str = "",
954
- **kwargs,
955
- ) -> list:
956
- output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
957
- sampled_requests = []
958
- for i, item in enumerate(self.data):
959
- if len(sampled_requests) >= num_requests:
960
- break
961
- prompt = (
962
- f"{item['input']}\n\n{item['instruction']} Just output "
963
- "the code, do not include any explanation."
964
- )
965
-
966
- # apply template
967
- prompt = tokenizer.apply_chat_template(
968
- [{"role": "user", "content": prompt}],
969
- add_generation_prompt=True,
970
- tokenize=False,
971
- )
972
- prompt_len = len(tokenizer(prompt).input_ids)
973
- sampled_requests.append(
974
- SampleRequest(
975
- prompt=prompt,
976
- prompt_len=prompt_len,
977
- expected_output_len=output_len,
978
- request_id=request_id_prefix + str(i),
979
- )
980
- )
981
- self.maybe_oversample_requests(
982
- sampled_requests, num_requests, request_id_prefix
983
- )
984
- return sampled_requests
985
-
986
-
987
-# -----------------------------------------------------------------------------
988
-# MT-Bench Dataset Implementation
989
-# -----------------------------------------------------------------------------
990
-
991
-
992
-class MTBenchDataset(HuggingFaceDataset):
993
- """
994
- MT-Bench Dataset.
995
- https://huggingface.co/datasets/philschmid/mt-bench
996
-
997
- We create a single turn dataset for MT-Bench.
998
- This is similar to Spec decoding benchmark setup in vLLM
999
- https://github.com/vllm-project/vllm/blob/9d98ab5ec/examples/offline_inference/eagle.py#L14-L18
1000
- """ # noqa: E501
1001
-
1002
- DEFAULT_OUTPUT_LEN = 256 # avg len used in SD bench in vLLM
1003
- SUPPORTED_DATASET_PATHS = {
1004
- "philschmid/mt-bench",
1005
- }
1006
-
1007
- def sample(
1008
- self,
1009
- tokenizer: PreTrainedTokenizerBase,
1010
- num_requests: int,
1011
- output_len: Optional[int] = None,
1012
- enable_multimodal_chat: bool = False,
1013
- request_id_prefix: str = "",
1014
- **kwargs,
1015
- ) -> list:
1016
- output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
1017
- sampled_requests = []
1018
-
1019
- for i, item in enumerate(self.data):
1020
- if len(sampled_requests) >= num_requests:
1021
- break
1022
- prompt = item["turns"][0]
1023
-
1024
- # apply template
1025
- prompt = tokenizer.apply_chat_template(
1026
- [{"role": "user", "content": prompt}],
1027
- add_generation_prompt=True,
1028
- tokenize=False,
1029
- )
1030
-
1031
- prompt_len = len(tokenizer(prompt).input_ids)
1032
- sampled_requests.append(
1033
- SampleRequest(
1034
- prompt=prompt,
1035
- prompt_len=prompt_len,
1036
- expected_output_len=output_len,
1037
- request_id=request_id_prefix + str(i),
1038
- )
1039
- )
1040
- self.maybe_oversample_requests(
1041
- sampled_requests, num_requests, request_id_prefix
1042
- )
1043
- return sampled_requests
1044
-
1045
-
1046
-# -----------------------------------------------------------------------------
1047
-# AIMO Dataset Implementation
1048
-# -----------------------------------------------------------------------------
1049
-
1050
-
1051
-class AIMODataset(HuggingFaceDataset):
1052
- """
1053
- Dataset class for processing a AIMO dataset with reasoning questions.
1054
- """
1055
-
1056
- SUPPORTED_DATASET_PATHS = {
1057
- "AI-MO/aimo-validation-aime",
1058
- "AI-MO/NuminaMath-1.5",
1059
- "AI-MO/NuminaMath-CoT",
1060
- }
1061
-
1062
- def sample(
1063
- self,
1064
- tokenizer: PreTrainedTokenizerBase,
1065
- num_requests: int,
1066
- output_len: Optional[int] = None,
1067
- request_id_prefix: str = "",
1068
- **kwargs,
1069
- ) -> list:
1070
- sampled_requests = []
1071
- dynamic_output = output_len is None
1072
- ind = 0
1073
-
1074
- for item in self.data:
1075
- if len(sampled_requests) >= num_requests:
1076
- break
1077
- prompt, completion = item["problem"], item["solution"]
1078
-
1079
- prompt_ids = tokenizer(prompt).input_ids
1080
- completion_ids = tokenizer(completion).input_ids
1081
- prompt_len = len(prompt_ids)
1082
- completion_len = len(completion_ids)
1083
- output_len = completion_len if dynamic_output else output_len
1084
- assert isinstance(output_len, int) and output_len > 0
1085
- if dynamic_output and not is_valid_sequence(
1086
- prompt_len, completion_len, max_prompt_len=2048, max_total_len=32000
1087
- ):
1088
- continue
1089
- sampled_requests.append(
1090
- SampleRequest(
1091
- prompt=prompt,
1092
- prompt_len=prompt_len,
1093
- expected_output_len=output_len,
1094
- multi_modal_data=None,
1095
- request_id=request_id_prefix + str(ind),
1096
- )
1097
- )
1098
- ind += 1
1099
- self.maybe_oversample_requests(
1100
- sampled_requests, num_requests, request_id_prefix
1101
- )
1102
- return sampled_requests
1103
-
1104
-
1105
-# -----------------------------------------------------------------------------
1106
-# Next Edit Prediction Dataset Implementation
1107
-# -----------------------------------------------------------------------------
1108
-
1109
-
1110
-zeta_prompt = """### Instruction:
1111
-You are a code completion assistant and your task is to analyze user edits and then rewrite an excerpt that the user provides, suggesting the appropriate edits within the excerpt, taking into account the cursor location.
1112
-
1113
-### User Edits:
1114
-
1115
-{}
1116
-
1117
-### User Excerpt:
1118
-
1119
-{}
1120
-
1121
-### Response:
1122
-
1123
-""" # noqa: E501
1124
-
1125
-
1126
-def _format_zeta_prompt(
1127
- sample: dict, original_start_marker: str = "<|editable_region_start|>"
1128
-) -> dict:
1129
- """Format the zeta prompt for the Next Edit Prediction (NEP) dataset.
1130
-
1131
- This function formats examples from the NEP dataset
1132
- into prompts and expected outputs. It could be
1133
- further extended to support more NEP datasets.
1134
-
1135
- Args:
1136
- sample: The dataset sample containing events,
1137
- inputs, and outputs.
1138
- original_start_marker: The marker indicating the
1139
- start of the editable region. Defaults to
1140
- "<|editable_region_start|>".
1141
-
1142
- Returns:
1143
- A dictionary with the formatted prompts and expected outputs.
1144
- """
1145
- events = sample["events"]
1146
- input = sample["input"]
1147
- output = sample["output"]
1148
- prompt = zeta_prompt.format(events, input)
1149
-
1150
- # following the original implementation, extract the focused region
1151
- # from the raw output
1152
- output_start_index = output.find(original_start_marker)
1153
- output_focused_region = output[output_start_index:]
1154
- expected_output = output_focused_region
1155
-
1156
- return {"prompt": prompt, "expected_output": expected_output}
1157
-
1158
-
1159
-class NextEditPredictionDataset(HuggingFaceDataset):
1160
- """
1161
- Dataset class for processing a Next Edit Prediction dataset.
1162
- """
1163
-
1164
- SUPPORTED_DATASET_PATHS = {
1165
- "zed-industries/zeta",
1166
- }
1167
- MAPPING_PROMPT_FUNCS = {
1168
- "zed-industries/zeta": _format_zeta_prompt,
1169
- }
1170
-
1171
- def sample(
1172
- self,
1173
- tokenizer: PreTrainedTokenizerBase,
1174
- num_requests: int,
1175
- request_id_prefix: str = "",
1176
- **kwargs,
1177
- ):
1178
- formatting_prompt_func = self.MAPPING_PROMPT_FUNCS.get(self.dataset_path)
1179
- if formatting_prompt_func is None:
1180
- raise ValueError(f"Unsupported dataset path: {self.dataset_path}")
1181
- samples = []
1182
- for i, sample in enumerate(self.data):
1183
- sample = formatting_prompt_func(sample)
1184
- samples.append(
1185
- SampleRequest(
1186
- prompt=sample["prompt"],
1187
- prompt_len=len(tokenizer(sample["prompt"]).input_ids),
1188
- expected_output_len=len(
1189
- tokenizer(sample["expected_output"]).input_ids
1190
- ),
1191
- request_id=request_id_prefix + str(i),
1192
- )
1193
- )
1194
- if len(samples) >= num_requests:
1195
- break
1196
- self.maybe_oversample_requests(samples, num_requests, request_id_prefix)
1197
- return samples
1198
-
1199
-
1200
-# -----------------------------------------------------------------------------
1201
-# ASR Dataset Implementation
1202
-# -----------------------------------------------------------------------------
1203
-
1204
-
1205
-class ASRDataset(HuggingFaceDataset):
1206
- """
1207
- Dataset class for processing a ASR dataset for transcription.
1208
- Tested on the following set:
1209
-
1210
- +----------------+----------------------------------------+--------------------------+-----------------------------+
1211
- | Dataset | Domain | Speaking Style | hf-subset |
1212
- +----------------+----------------------------------------+--------------------------+-----------------------------+
1213
- | TED-LIUM | TED talks | Oratory | release1, release2, release3|
1214
- | | | | release3-speaker-adaptation |
1215
- | VoxPopuli | European Parliament | Oratory | en, de, it, fr, ... |
1216
- | LibriSpeech | Audiobook | Narrated | "LIUM/tedlium" |
1217
- | GigaSpeech | Audiobook, podcast, YouTube | Narrated, spontaneous | xs, s, m, l, xl, dev, test |
1218
- | SPGISpeech | Financial meetings | Oratory, spontaneous | S, M, L, dev, test |
1219
- | AMI | Meetings | Spontaneous | ihm, sdm |
1220
- +----------------+----------------------------------------+--------------------------+-----------------------------+
1221
-
1222
- """ # noqa: E501
1223
-
1224
- SUPPORTED_DATASET_PATHS = {
1225
- "openslr/librispeech_asr",
1226
- "facebook/voxpopuli",
1227
- "LIUM/tedlium",
1228
- "edinburghcstr/ami",
1229
- "speechcolab/gigaspeech",
1230
- "kensho/spgispeech",
1231
- }
1232
-
1233
- DEFAULT_OUTPUT_LEN = 128
1234
- IS_MULTIMODAL = True
1235
-
1236
- # TODO Whisper-specific. Abstract interface when more models are supported.
1237
- TRANSCRIPTION_PREAMBLE = "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>"
1238
- skip_long_audios: bool = True
1239
-
1240
- def sample(
1241
- self,
1242
- tokenizer: PreTrainedTokenizerBase,
1243
- num_requests: int,
1244
- output_len: Optional[int] = None,
1245
- request_id_prefix: str = "",
1246
- **kwargs,
1247
- ) -> list:
1248
- import librosa
1249
-
1250
- output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
1251
- prompt = ASRDataset.TRANSCRIPTION_PREAMBLE
1252
- prompt_len = len(tokenizer(prompt).input_ids)
1253
- sampled_requests = []
1254
- skipped = 0
1255
- ind = 0
1256
- for item in self.data:
1257
- if len(sampled_requests) >= num_requests:
1258
- break
1259
- audio = item["audio"]
1260
- y, sr = audio["array"], audio["sampling_rate"]
1261
- duration_s = librosa.get_duration(y=y, sr=sr)
1262
- # Whisper max supported duration
1263
- if self.skip_long_audios and duration_s > 30:
1264
- skipped += 1
1265
- continue
1266
-
1267
- mm_content = {"audio": (y, sr)}
1268
- sampled_requests.append(
1269
- SampleRequest(
1270
- prompt=prompt,
1271
- prompt_len=prompt_len,
1272
- expected_output_len=output_len,
1273
- multi_modal_data=mm_content,
1274
- request_id=request_id_prefix + str(ind),
1275
- )
1276
- )
1277
- ind += 1
1278
- if skipped:
1279
- logger.warning(
1280
- "%d samples discarded from dataset due to"
1281
- " their length being greater than"
1282
- " what Whisper supports.",
1283
- skipped,
1284
- )
1285
- self.maybe_oversample_requests(
1286
- sampled_requests, num_requests, request_id_prefix
1287
- )
1288
- return sampled_requests
docs/contributing/benchmarks.md
+1
@@ -37,6 +37,7 @@ th {
37
| RandomMultiModal (Image/Video) | 🟡 | 🚧 | `synthetic` |
38
| Prefix Repetition | ✅ | ✅ | `synthetic` |
39
| HuggingFace-VisionArena | ✅ | ✅ | `lmarena-ai/VisionArena-Chat` |
40
+| HuggingFace-MMVU | ✅ | ✅ | `yale-nlp/MMVU` |
41
| HuggingFace-InstructCoder | ✅ | ✅ | `likaixin/InstructCoder` |
42
| HuggingFace-AIMO | ✅ | ✅ | `AI-MO/aimo-validation-aime`, `AI-MO/NuminaMath-1.5`, `AI-MO/NuminaMath-CoT` |
43
| HuggingFace-Other | ✅ | ✅ | `lmms-lab/LLaVA-OneVision-Data`, `Aeala/ShareGPT_Vicuna_unfiltered` |
vllm/benchmarks/datasets.py
+64
-2
@@ -335,7 +335,7 @@ def process_image(image: Any) -> Mapping[str, Any]:
335
336
if isinstance(image, str):
337
image_url = (image if image.startswith(
338
- ("http://", "file://")) else f"file://{image}")
338
+ ("http://", "https://", "file://")) else f"file://{image}")
339
return {"type": "image_url", "image_url": {"url": image_url}}
340
341
raise ValueError(f"Invalid image input {image}. Must be a PIL.Image.Image"
@@ -370,7 +370,7 @@ def process_video(video: Any) -> Mapping[str, Any]:
370
371
if isinstance(video, str):
372
video_url = (video if video.startswith(
373
- ("http://", "file://")) else f"file://{video}")
373
+ ("http://", "https://", "file://")) else f"file://{video}")
374
return {"type": "video_url", "video_url": {"url": video_url}}
375
376
raise ValueError(
@@ -1405,6 +1405,13 @@ def get_samples(args, tokenizer) -> list[SampleRequest]:
1405
dataset_class = VisionArenaDataset
1406
args.hf_split = "train"
1407
args.hf_subset = None
1408
+ elif (
1409
+ args.dataset_path in MMVUDataset.SUPPORTED_DATASET_PATHS
1410
+ or args.hf_name in MMVUDataset.SUPPORTED_DATASET_PATHS
1411
+ ):
1412
+ dataset_class = MMVUDataset
1413
+ args.hf_split = "validation"
1414
+ args.hf_subset = None
1415
elif (
1416
args.dataset_path in InstructCoderDataset.SUPPORTED_DATASET_PATHS
1417
or args.hf_name in InstructCoderDataset.SUPPORTED_DATASET_PATHS
@@ -2053,6 +2060,61 @@ class VisionArenaDataset(HuggingFaceDataset):
2060
return sampled_requests
2061
2062
2063
+class MMVUDataset(HuggingFaceDataset):
2064
+ """
2065
+ MMVU Dataset.
2066
+ https://huggingface.co/datasets/yale-nlp/MMVU
2067
+ """
2068
+
2069
+ DEFAULT_OUTPUT_LEN = 128
2070
+ SUPPORTED_DATASET_PATHS = {
2071
+ "yale-nlp/MMVU":
2072
+ lambda x: x["question"] + " " + (
2073
+ " ".join(f"{k}.{v}" for k, v in x["choices"].items())
2074
+ ),
2075
+ }
2076
+
2077
+ def sample(
2078
+ self,
2079
+ tokenizer: PreTrainedTokenizerBase,
2080
+ num_requests: int,
2081
+ output_len: Optional[int] = None,
2082
+ enable_multimodal_chat: bool = False,
2083
+ request_id_prefix: str = "",
2084
+ no_oversample: bool = False,
2085
+ **kwargs,
2086
+ ) -> list:
2087
+ output_len = (output_len
2088
+ if output_len is not None else self.DEFAULT_OUTPUT_LEN)
2089
+ sampled_requests = []
2090
+ for i, item in enumerate(self.data):
2091
+ if len(sampled_requests) >= num_requests:
2092
+ break
2093
+ parser_fn = self.SUPPORTED_DATASET_PATHS.get(self.hf_name)
2094
+ if parser_fn is None:
2095
+ raise ValueError(f"Unsupported dataset path: {self.hf_name}")
2096
+ prompt = parser_fn(item)
2097
+ mm_content = process_video(item["video"])
2098
+ prompt_len = len(tokenizer(prompt).input_ids)
2099
+ if enable_multimodal_chat:
2100
+ # Note: when chat is enabled the request prompt_len is no longer
2101
+ # accurate and we will be using request output to count the
2102
+ # actual prompt len
2103
+ prompt = self.apply_multimodal_chat_transformation(
2104
+ prompt, mm_content)
2105
+ sampled_requests.append(
2106
+ SampleRequest(
2107
+ prompt=prompt,
2108
+ prompt_len=prompt_len,
2109
+ expected_output_len=output_len,
2110
+ multi_modal_data=mm_content,
2111
+ request_id=request_id_prefix + str(i),
2112
+ ))
2113
+ self.maybe_oversample_requests(sampled_requests, num_requests,
2114
+ request_id_prefix, no_oversample)
2115
+ return sampled_requests
2116
+
2117
+
2118
# -----------------------------------------------------------------------------
2119
# Instruct Coder Dataset Implementation
2120
# -----------------------------------------------------------------------------