A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[Misc] Use VisionArena Dataset for VLM Benchmarking (#12389)
Signed-off-by: Roger Wang <ywang@roblox.com>
Roger Wang committed
Jan 24, 2025 at 00:22 UTC
3c818bdb42f2b0ed1250568abbabf45909ff308e
1 file changed
+12
-20
benchmarks/benchmark_serving.py
+12
-20
@@ -200,7 +200,7 @@ def sample_sonnet_requests(
200
return sampled_requests
201
202
203
-def sample_mmmu_pro_vision_requests(
203
+def sample_vision_arena_requests(
204
dataset,
205
num_requests: int,
206
tokenizer: PreTrainedTokenizerBase,
@@ -212,13 +212,7 @@ def sample_mmmu_pro_vision_requests(
212
if len(sampled_requests) == num_requests:
213
break
214
215
- # MMMU-Pro vision direct prompt
216
- # Ref: https://github.com/MMMU-Benchmark/MMMU/blob/6ce42f4d8f70c1841c67867152648974415b5cac/mmmu-pro/prompts.yaml#L5
217
- prompt = (
218
- "Answer with the option letter from the given choices directly. "
219
- "The last line of your response should be of the following "
220
- "format: 'Answer: $LETTER' (without quotes) where LETTER is one of "
221
- "options.")
215
+ prompt = data["turns"][0][0]['content']
216
217
prompt_token_ids = tokenizer(prompt).input_ids
218
if fixed_output_len is None:
@@ -230,10 +224,10 @@ def sample_mmmu_pro_vision_requests(
224
output_len = fixed_output_len
225
226
assert isinstance(
233
- data["image"],
227
+ data["images"][0],
228
Image), ("Input image format must be `PIL.Image.Image`, "
229
f"given {type(data['image'])}.")
236
- image: Image = data["image"]
230
+ image: Image = data["images"][0]
231
image = image.convert("RGB")
232
image_data = io.BytesIO()
233
image.save(image_data, format='JPEG')
@@ -252,7 +246,7 @@ def sample_mmmu_pro_vision_requests(
246
247
def sample_hf_requests(
248
dataset_path: str,
255
- dataset_subset: str,
249
+ dataset_subset: Optional[str],
250
dataset_split: str,
251
num_requests: int,
252
tokenizer: PreTrainedTokenizerBase,
@@ -260,19 +254,17 @@ def sample_hf_requests(
254
fixed_output_len: Optional[int] = None,
255
) -> List[Tuple[str, str, int, Optional[Dict[str, Collection[str]]]]]:
256
263
- # Special case for MMMU-Pro vision dataset
264
- if dataset_path == 'MMMU/MMMU_Pro' and dataset_subset == 'vision':
265
- assert dataset_split == "test"
257
+ # Special case for vision_arena dataset
258
+ if dataset_path == 'lmarena-ai/vision-arena-bench-v0.1' \
259
+ and dataset_subset is None:
260
+ assert dataset_split == "train"
261
dataset = load_dataset(dataset_path,
262
name=dataset_subset,
263
split=dataset_split,
264
streaming=True)
270
- assert "image" in dataset.features, (
271
- "MMMU/MMMU_Pro vision dataset must have 'image' column.")
272
- filter_func = lambda x: isinstance(x["image"], Image)
273
- dataset = dataset.shuffle(seed=random_seed).filter(filter_func)
274
- return sample_mmmu_pro_vision_requests(dataset, num_requests,
275
- tokenizer, fixed_output_len)
265
+ dataset = dataset.shuffle(seed=random_seed)
266
+ return sample_vision_arena_requests(dataset, num_requests, tokenizer,
267
+ fixed_output_len)
268
269
dataset = load_dataset(dataset_path,
270
name=dataset_subset,