@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c11f172187
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Misc] Adding `MMMU-Pro` vision dataset to serving benchmark (#10804)

Signed-off-by: Roger Wang <ywang@roblox.com> Co-authored-by: Chen Zhang <zhangch99@outlook.com> Co-authored-by: Isotr0py <2037008807@qq.com>

Roger Wang committed Dec 1, 2024 at 00:47 UTC c11f172187b6f44710e1f011ca8bff923ce49a7f
1 file changed +65
benchmarks/benchmark_serving.py
+65
@@ -199,6 +199,56 @@ def sample_sonnet_requests(
199 return sampled_requests
200
201
202 +def sample_mmmu_pro_vision_requests(
203 + dataset,
204 + num_requests: int,
205 + tokenizer: PreTrainedTokenizerBase,
206 + fixed_output_len: Optional[int] = None,
207 +) -> List[Tuple[str, str, int, Optional[Dict[str, Collection[str]]]]]:
208 + sampled_requests: List[Tuple[str, int, int, Dict[str,
209 + Collection[str]]]] = []
210 + for data in dataset:
211 + if len(sampled_requests) == num_requests:
212 + break
213 +
214 + # MMMU-Pro vision direct prompt
215 + # Ref: https://github.com/MMMU-Benchmark/MMMU/blob/6ce42f4d8f70c1841c67867152648974415b5cac/mmmu-pro/prompts.yaml#L5
216 + prompt = (
217 + "Answer with the option letter from the given choices directly. "
218 + "The last line of your response should be of the following "
219 + "format: 'Answer: $LETTER' (without quotes) where LETTER is one of "
220 + "options.")
221 +
222 + prompt_token_ids = tokenizer(prompt).input_ids
223 + if fixed_output_len is None:
224 + # Default max output len is set to 128
225 + print("--hf-output-len is not provided. Using default value 128.")
226 + fixed_output_len = 128
227 +
228 + prompt_len = len(prompt_token_ids)
229 + output_len = fixed_output_len
230 +
231 + assert isinstance(
232 + data["image"],
233 + Image), ("Input image format must be `PIL.Image.Image`, "
234 + f"given {type(data['image'])}.")
235 + image: Image = data["image"]
236 + image = image.convert("RGB")
237 + image_data = io.BytesIO()
238 + image.save(image_data, format='JPEG')
239 + image_base64 = base64.b64encode(image_data.getvalue()).decode("utf-8")
240 + mm_content = {
241 + "type": "image_url",
242 + "image_url": {
243 + "url": f"data:image/jpeg;base64,{image_base64}"
244 + },
245 + }
246 +
247 + sampled_requests.append((prompt, prompt_len, output_len, mm_content))
248 +
249 + return sampled_requests
250 +
251 +
252 def sample_hf_requests(
253 dataset_path: str,
254 dataset_subset: str,
@@ -208,6 +258,21 @@ def sample_hf_requests(
258 random_seed: int,
259 fixed_output_len: Optional[int] = None,
260 ) -> List[Tuple[str, str, int, Optional[Dict[str, Collection[str]]]]]:
261 +
262 + # Special case for MMMU-Pro vision dataset
263 + if dataset_path == 'MMMU/MMMU_Pro' and dataset_subset == 'vision':
264 + assert dataset_split == "test"
265 + dataset = load_dataset(dataset_path,
266 + name=dataset_subset,
267 + split=dataset_split,
268 + streaming=True)
269 + assert "image" in dataset.features, (
270 + "MMMU/MMMU_Pro vision dataset must have 'image' column.")
271 + filter_func = lambda x: isinstance(x["image"], Image)
272 + dataset = dataset.shuffle(seed=random_seed).filter(filter_func)
273 + return sample_mmmu_pro_vision_requests(dataset, num_requests,
274 + tokenizer, fixed_output_len)
275 +
276 dataset = load_dataset(dataset_path,
277 name=dataset_subset,
278 split=dataset_split,