| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | import json |
| 4 | from argparse import Namespace |
| 5 | from io import BytesIO |
| 6 | from pathlib import Path |
| 7 | from typing import Any, cast |
| 8 | |
| 9 | import pybase64 as base64 |
| 10 | import pytest |
| 11 | from PIL import Image |
| 12 | |
| 13 | import vllm.benchmarks.datasets.datasets as datasets_module |
| 14 | from vllm.benchmarks.datasets import CustomImageDataset, get_samples |
| 15 | from vllm.benchmarks.lib.endpoint_request_func import ( |
| 16 | RequestFuncInput, |
| 17 | _get_chat_content, |
| 18 | _get_chat_messages, |
| 19 | ) |
| 20 | from vllm.tokenizers import TokenizerLike |
| 21 | |
| 22 | pytestmark = pytest.mark.skip_global_cleanup |
| 23 | |
| 24 | |
| 25 | class _TokenizedPrompt: |
| 26 | def __init__(self, prompt: str) -> None: |
| 27 | self.input_ids = prompt.split() |
| 28 | |
| 29 | |
| 30 | class _Tokenizer: |
| 31 | def __call__(self, prompt: str) -> _TokenizedPrompt: |
| 32 | return _TokenizedPrompt(prompt) |
| 33 | |
| 34 | |
| 35 | def _tokenizer() -> TokenizerLike: |
| 36 | return cast(TokenizerLike, _Tokenizer()) |
| 37 | |
| 38 | |
| 39 | def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None: |
| 40 | with path.open("w") as f: |
| 41 | for row in rows: |
| 42 | f.write(json.dumps(row) + "\n") |
| 43 | |
| 44 | |
| 45 | def _write_png(path: Path, color: tuple[int, int, int] = (255, 0, 0)) -> None: |
| 46 | Image.new("RGB", (1, 1), color=color).save(path) |
| 47 | |
| 48 | |
| 49 | def _decode_data_url(data_url: str) -> tuple[str, bytes]: |
| 50 | prefix, image_base64 = data_url.split(",", 1) |
| 51 | return prefix, base64.b64decode(image_base64) |
| 52 | |
| 53 | |
| 54 | def _assert_png_data_url(data_url: str) -> None: |
| 55 | prefix, image_bytes = _decode_data_url(data_url) |
| 56 | assert prefix == "data:image/png;base64" |
| 57 | with Image.open(BytesIO(image_bytes)) as image: |
| 58 | image.verify() |
| 59 | |
| 60 | |
| 61 | def _args_for_custom_image(dataset_path: Path) -> Namespace: |
| 62 | return Namespace( |
| 63 | dataset_name="custom_image", |
| 64 | dataset_path=str(dataset_path), |
| 65 | disable_shuffle=True, |
| 66 | seed=0, |
| 67 | num_prompts=2, |
| 68 | custom_output_len=32, |
| 69 | enable_multimodal_chat=False, |
| 70 | custom_ensure_client_side_data=False, |
| 71 | request_id_prefix="req-", |
| 72 | no_oversample=False, |
| 73 | ) |
| 74 | |
| 75 | |
| 76 | @pytest.mark.benchmark |
| 77 | def test_get_samples_custom_image_cli_path_supports_multi_image_and_content( |
| 78 | tmp_path: Path, |
| 79 | ) -> None: |
| 80 | image_a = tmp_path / "chart_a.png" |
| 81 | image_b = tmp_path / "chart_b.png" |
| 82 | image_c = tmp_path / "chart_c.png" |
| 83 | jsonl = tmp_path / "images.jsonl" |
| 84 | _write_jsonl( |
| 85 | jsonl, |
| 86 | [ |
| 87 | { |
| 88 | "prompt": "Compare the first two charts.", |
| 89 | "image_files": [str(image_a), str(image_b)], |
| 90 | }, |
| 91 | { |
| 92 | "content": [ |
| 93 | {"type": "text", "text": "Now compare "}, |
| 94 | {"type": "image", "image": str(image_c)}, |
| 95 | ], |
| 96 | }, |
| 97 | ], |
| 98 | ) |
| 99 | |
| 100 | samples = get_samples(_args_for_custom_image(jsonl), _tokenizer()) |
| 101 | |
| 102 | assert len(samples) == 2 |
| 103 | assert samples[0].request_id == "req-0" |
| 104 | assert isinstance(samples[0].multi_modal_data, list) |
| 105 | assert [part["image_url"]["url"] for part in samples[0].multi_modal_data] == [ |
| 106 | f"file://{image_a}", |
| 107 | f"file://{image_b}", |
| 108 | ] |
| 109 | |
| 110 | assert samples[1].request_id == "req-1" |
| 111 | assert samples[1].multi_modal_data is None |
| 112 | assert isinstance(samples[1].prompt, list) |
| 113 | assert samples[1].prompt[0] == {"type": "text", "text": "Now compare "} |
| 114 | assert isinstance(samples[1].prompt[1], dict) |
| 115 | assert samples[1].prompt[1]["image_url"]["url"] == f"file://{image_c}" |
| 116 | |
| 117 | |
| 118 | @pytest.mark.benchmark |
| 119 | def test_custom_image_dataset_uses_all_image_files(tmp_path: Path) -> None: |
| 120 | image_a = tmp_path / "chart_a.png" |
| 121 | image_b = tmp_path / "chart_b.png" |
| 122 | jsonl = tmp_path / "images.jsonl" |
| 123 | _write_jsonl( |
| 124 | jsonl, |
| 125 | [ |
| 126 | { |
| 127 | "prompt": "Compare the charts.", |
| 128 | "image_files": [str(image_a), str(image_b)], |
| 129 | } |
| 130 | ], |
| 131 | ) |
| 132 | |
| 133 | dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True) |
| 134 | samples = dataset.sample( |
| 135 | tokenizer=_tokenizer(), |
| 136 | num_requests=1, |
| 137 | output_len=32, |
| 138 | ) |
| 139 | |
| 140 | assert len(samples) == 1 |
| 141 | sample = samples[0] |
| 142 | assert sample.prompt == "Compare the charts." |
| 143 | assert sample.prompt_len == 3 |
| 144 | assert isinstance(sample.multi_modal_data, list) |
| 145 | assert [part["image_url"]["url"] for part in sample.multi_modal_data] == [ |
| 146 | f"file://{image_a}", |
| 147 | f"file://{image_b}", |
| 148 | ] |
| 149 | |
| 150 | |
| 151 | @pytest.mark.benchmark |
| 152 | def test_custom_image_dataset_preserves_interleaved_content_order( |
| 153 | tmp_path: Path, |
| 154 | ) -> None: |
| 155 | image_a = tmp_path / "chart_a.png" |
| 156 | image_b = tmp_path / "chart_b.png" |
| 157 | jsonl = tmp_path / "images.jsonl" |
| 158 | _write_jsonl( |
| 159 | jsonl, |
| 160 | [ |
| 161 | { |
| 162 | "content": [ |
| 163 | {"type": "text", "text": "Compare "}, |
| 164 | {"type": "image", "image": str(image_a)}, |
| 165 | {"type": "text", "text": " with "}, |
| 166 | { |
| 167 | "type": "image_url", |
| 168 | "image_url": { |
| 169 | "url": str(image_b), |
| 170 | "detail": "low", |
| 171 | }, |
| 172 | }, |
| 173 | ], |
| 174 | } |
| 175 | ], |
| 176 | ) |
| 177 | |
| 178 | dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True) |
| 179 | samples = dataset.sample( |
| 180 | tokenizer=_tokenizer(), |
| 181 | num_requests=1, |
| 182 | output_len=32, |
| 183 | ) |
| 184 | |
| 185 | assert len(samples) == 1 |
| 186 | sample = samples[0] |
| 187 | assert sample.multi_modal_data is None |
| 188 | assert sample.prompt_len == 2 |
| 189 | assert isinstance(sample.prompt, list) |
| 190 | parts = [] |
| 191 | for part in sample.prompt: |
| 192 | assert isinstance(part, dict) |
| 193 | parts.append(part) |
| 194 | assert [part["type"] for part in parts] == [ |
| 195 | "text", |
| 196 | "image_url", |
| 197 | "text", |
| 198 | "image_url", |
| 199 | ] |
| 200 | assert parts[1]["image_url"]["url"] == f"file://{image_a}" |
| 201 | assert parts[3]["image_url"] == { |
| 202 | "url": f"file://{image_b}", |
| 203 | "detail": "low", |
| 204 | } |
| 205 | |
| 206 | request_input = RequestFuncInput( |
| 207 | prompt=sample.prompt, |
| 208 | api_url="http://localhost:8000/v1/chat/completions", |
| 209 | prompt_len=sample.prompt_len, |
| 210 | output_len=32, |
| 211 | model="test-model", |
| 212 | ) |
| 213 | assert _get_chat_content(request_input) == sample.prompt |
| 214 | |
| 215 | |
| 216 | @pytest.mark.benchmark |
| 217 | def test_custom_image_dataset_wraps_interleaved_content_for_multimodal_chat( |
| 218 | tmp_path: Path, |
| 219 | ) -> None: |
| 220 | image = tmp_path / "chart.png" |
| 221 | jsonl = tmp_path / "images.jsonl" |
| 222 | _write_jsonl( |
| 223 | jsonl, |
| 224 | [ |
| 225 | { |
| 226 | "content": [ |
| 227 | {"type": "text", "text": "Describe "}, |
| 228 | {"type": "image", "image": str(image)}, |
| 229 | ], |
| 230 | } |
| 231 | ], |
| 232 | ) |
| 233 | |
| 234 | dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True) |
| 235 | samples = dataset.sample( |
| 236 | tokenizer=_tokenizer(), |
| 237 | num_requests=1, |
| 238 | output_len=32, |
| 239 | enable_multimodal_chat=True, |
| 240 | ) |
| 241 | |
| 242 | sample = samples[0] |
| 243 | assert sample.multi_modal_data is None |
| 244 | assert sample.prompt == [ |
| 245 | { |
| 246 | "role": "user", |
| 247 | "content": [ |
| 248 | {"type": "text", "text": "Describe "}, |
| 249 | {"type": "image_url", "image_url": {"url": f"file://{image}"}}, |
| 250 | ], |
| 251 | } |
| 252 | ] |
| 253 | |
| 254 | request_input = RequestFuncInput( |
| 255 | prompt=sample.prompt, |
| 256 | api_url="http://localhost:8000/v1/chat/completions", |
| 257 | prompt_len=sample.prompt_len, |
| 258 | output_len=32, |
| 259 | model="test-model", |
| 260 | ) |
| 261 | assert _get_chat_messages(request_input) == sample.prompt |
| 262 | |
| 263 | |
| 264 | @pytest.mark.benchmark |
| 265 | def test_custom_image_dataset_encodes_image_media_when_requested( |
| 266 | tmp_path: Path, |
| 267 | monkeypatch: pytest.MonkeyPatch, |
| 268 | ) -> None: |
| 269 | image_a = tmp_path / "chart_a.png" |
| 270 | image_b = tmp_path / "chart b.png" |
| 271 | _write_png(image_a, color=(255, 0, 0)) |
| 272 | _write_png(image_b, color=(0, 255, 0)) |
| 273 | data_url = "data:image/png;base64,Zm9v" |
| 274 | remote_url = "https://example.com/chart.png" |
| 275 | original_fetch_image = datasets_module.fetch_image |
| 276 | |
| 277 | def fake_fetch_image(image_url: str) -> Image.Image: |
| 278 | if image_url == remote_url: |
| 279 | return Image.new("RGB", (1, 1), color=(0, 0, 255)) |
| 280 | return original_fetch_image(image_url) |
| 281 | |
| 282 | monkeypatch.setattr(datasets_module, "fetch_image", fake_fetch_image) |
| 283 | |
| 284 | jsonl = tmp_path / "images.jsonl" |
| 285 | _write_jsonl( |
| 286 | jsonl, |
| 287 | [ |
| 288 | { |
| 289 | "prompt": "Compare the charts.", |
| 290 | "image_files": [ |
| 291 | str(image_a), |
| 292 | image_b.as_uri(), |
| 293 | remote_url, |
| 294 | data_url, |
| 295 | ], |
| 296 | } |
| 297 | ], |
| 298 | ) |
| 299 | |
| 300 | dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True) |
| 301 | samples = dataset.sample( |
| 302 | tokenizer=_tokenizer(), |
| 303 | num_requests=1, |
| 304 | output_len=32, |
| 305 | ensure_client_side_data=True, |
| 306 | ) |
| 307 | |
| 308 | assert len(samples) == 1 |
| 309 | assert isinstance(samples[0].multi_modal_data, list) |
| 310 | image_urls = [part["image_url"]["url"] for part in samples[0].multi_modal_data] |
| 311 | |
| 312 | _assert_png_data_url(image_urls[0]) |
| 313 | _assert_png_data_url(image_urls[1]) |
| 314 | _assert_png_data_url(image_urls[2]) |
| 315 | assert image_urls[3] == data_url |
| 316 | |
| 317 | |
| 318 | @pytest.mark.benchmark |
| 319 | def test_custom_image_dataset_encodes_interleaved_image_media( |
| 320 | tmp_path: Path, |
| 321 | ) -> None: |
| 322 | image_a = tmp_path / "chart_a.png" |
| 323 | image_b = tmp_path / "chart_b.png" |
| 324 | _write_png(image_a, color=(255, 0, 0)) |
| 325 | _write_png(image_b, color=(0, 255, 0)) |
| 326 | jsonl = tmp_path / "images.jsonl" |
| 327 | _write_jsonl( |
| 328 | jsonl, |
| 329 | [ |
| 330 | { |
| 331 | "content": [ |
| 332 | {"type": "text", "text": "Compare "}, |
| 333 | {"type": "image", "image": str(image_a)}, |
| 334 | { |
| 335 | "type": "image_url", |
| 336 | "image_url": { |
| 337 | "url": image_b.as_uri(), |
| 338 | "detail": "low", |
| 339 | }, |
| 340 | }, |
| 341 | ], |
| 342 | } |
| 343 | ], |
| 344 | ) |
| 345 | |
| 346 | dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True) |
| 347 | samples = dataset.sample( |
| 348 | tokenizer=_tokenizer(), |
| 349 | num_requests=1, |
| 350 | output_len=32, |
| 351 | ensure_client_side_data=True, |
| 352 | ) |
| 353 | |
| 354 | sample = samples[0] |
| 355 | assert isinstance(sample.prompt, list) |
| 356 | parts = [] |
| 357 | for part in sample.prompt: |
| 358 | assert isinstance(part, dict) |
| 359 | parts.append(part) |
| 360 | _assert_png_data_url(parts[1]["image_url"]["url"]) |
| 361 | _assert_png_data_url(parts[2]["image_url"]["url"]) |
| 362 | assert parts[2]["image_url"]["detail"] == "low" |
| 363 | |
| 364 | |
| 365 | @pytest.mark.benchmark |
| 366 | def test_custom_image_dataset_rejects_invalid_image_media( |
| 367 | tmp_path: Path, |
| 368 | ) -> None: |
| 369 | invalid_image = tmp_path / "not_an_image.png" |
| 370 | invalid_image.write_text("not an image") |
| 371 | jsonl = tmp_path / "images.jsonl" |
| 372 | _write_jsonl( |
| 373 | jsonl, |
| 374 | [{"prompt": "Describe the image.", "image_files": [str(invalid_image)]}], |
| 375 | ) |
| 376 | |
| 377 | dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True) |
| 378 | with pytest.raises(ValueError, match="Invalid image URL"): |
| 379 | dataset.sample( |
| 380 | tokenizer=_tokenizer(), |
| 381 | num_requests=1, |
| 382 | output_len=32, |
| 383 | ensure_client_side_data=True, |
| 384 | ) |
| 385 | |
| 386 | |
| 387 | @pytest.mark.benchmark |
| 388 | def test_custom_image_dataset_rejects_invalid_content_part( |
| 389 | tmp_path: Path, |
| 390 | ) -> None: |
| 391 | jsonl = tmp_path / "images.jsonl" |
| 392 | _write_jsonl(jsonl, [{"content": [{"type": "audio", "audio": "clip.wav"}]}]) |
| 393 | |
| 394 | dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True) |
| 395 | with pytest.raises(ValueError, match="type 'text', 'image', or 'image_url'"): |
| 396 | dataset.sample( |
| 397 | tokenizer=_tokenizer(), |
| 398 | num_requests=1, |
| 399 | output_len=32, |
| 400 | ) |