@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3c818bdb42
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[Misc] Use VisionArena Dataset for VLM Benchmarking (#12389)

Signed-off-by: Roger Wang <ywang@roblox.com>

Roger Wang committed Jan 24, 2025 at 00:22 UTC 3c818bdb42f2b0ed1250568abbabf45909ff308e
1 file changed +12 -20
benchmarks/benchmark_serving.py
+12 -20
@@ -200,7 +200,7 @@ def sample_sonnet_requests(
200 return sampled_requests
201
202
203 -def sample_mmmu_pro_vision_requests(
203 +def sample_vision_arena_requests(
204 dataset,
205 num_requests: int,
206 tokenizer: PreTrainedTokenizerBase,
@@ -212,13 +212,7 @@ def sample_mmmu_pro_vision_requests(
212 if len(sampled_requests) == num_requests:
213 break
214
215 - # MMMU-Pro vision direct prompt
216 - # Ref: https://github.com/MMMU-Benchmark/MMMU/blob/6ce42f4d8f70c1841c67867152648974415b5cac/mmmu-pro/prompts.yaml#L5
217 - prompt = (
218 - "Answer with the option letter from the given choices directly. "
219 - "The last line of your response should be of the following "
220 - "format: 'Answer: $LETTER' (without quotes) where LETTER is one of "
221 - "options.")
215 + prompt = data["turns"][0][0]['content']
216
217 prompt_token_ids = tokenizer(prompt).input_ids
218 if fixed_output_len is None:
@@ -230,10 +224,10 @@ def sample_mmmu_pro_vision_requests(
224 output_len = fixed_output_len
225
226 assert isinstance(
233 - data["image"],
227 + data["images"][0],
228 Image), ("Input image format must be `PIL.Image.Image`, "
229 f"given {type(data['image'])}.")
236 - image: Image = data["image"]
230 + image: Image = data["images"][0]
231 image = image.convert("RGB")
232 image_data = io.BytesIO()
233 image.save(image_data, format='JPEG')
@@ -252,7 +246,7 @@ def sample_mmmu_pro_vision_requests(
246
247 def sample_hf_requests(
248 dataset_path: str,
255 - dataset_subset: str,
249 + dataset_subset: Optional[str],
250 dataset_split: str,
251 num_requests: int,
252 tokenizer: PreTrainedTokenizerBase,
@@ -260,19 +254,17 @@ def sample_hf_requests(
254 fixed_output_len: Optional[int] = None,
255 ) -> List[Tuple[str, str, int, Optional[Dict[str, Collection[str]]]]]:
256
263 - # Special case for MMMU-Pro vision dataset
264 - if dataset_path == 'MMMU/MMMU_Pro' and dataset_subset == 'vision':
265 - assert dataset_split == "test"
257 + # Special case for vision_arena dataset
258 + if dataset_path == 'lmarena-ai/vision-arena-bench-v0.1' \
259 + and dataset_subset is None:
260 + assert dataset_split == "train"
261 dataset = load_dataset(dataset_path,
262 name=dataset_subset,
263 split=dataset_split,
264 streaming=True)
270 - assert "image" in dataset.features, (
271 - "MMMU/MMMU_Pro vision dataset must have 'image' column.")
272 - filter_func = lambda x: isinstance(x["image"], Image)
273 - dataset = dataset.shuffle(seed=random_seed).filter(filter_func)
274 - return sample_mmmu_pro_vision_requests(dataset, num_requests,
275 - tokenizer, fixed_output_len)
265 + dataset = dataset.shuffle(seed=random_seed)
266 + return sample_vision_arena_requests(dataset, num_requests, tokenizer,
267 + fixed_output_len)
268
269 dataset = load_dataset(dataset_path,
270 name=dataset_subset,