| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import os |
| 5 | import tempfile |
| 6 | from pathlib import Path |
| 7 | from tempfile import NamedTemporaryFile |
| 8 | from typing import Any, cast |
| 9 | from unittest.mock import Mock |
| 10 | |
| 11 | import cv2 |
| 12 | import pybase64 as base64 |
| 13 | import pytest |
| 14 | from transformers import AutoTokenizer, PreTrainedTokenizerBase |
| 15 | |
| 16 | from vllm.benchmarks.datasets import RandomMultiModalDataset, SampleRequest |
| 17 | |
| 18 | |
| 19 | @pytest.fixture(scope="session") |
| 20 | def hf_tokenizer() -> PreTrainedTokenizerBase: |
| 21 | """Use a small, commonly available tokenizer.""" |
| 22 | return AutoTokenizer.from_pretrained("openai-community/gpt2") |
| 23 | |
| 24 | |
| 25 | @pytest.fixture |
| 26 | def video_dataset() -> RandomMultiModalDataset: |
| 27 | """Create a RandomMultiModalDataset instance for testing.""" |
| 28 | return RandomMultiModalDataset(random_seed=42) |
| 29 | |
| 30 | |
| 31 | @pytest.mark.benchmark |
| 32 | def test_generate_synthetic_video_different_seeds(): |
| 33 | """Test that different seeds produce different videos.""" |
| 34 | dataset1 = RandomMultiModalDataset(random_seed=123) |
| 35 | dataset2 = RandomMultiModalDataset(random_seed=456) |
| 36 | |
| 37 | width, height, num_frames = 64, 48, 8 |
| 38 | |
| 39 | video1 = dataset1.generate_synthetic_video(width, height, num_frames) |
| 40 | video2 = dataset2.generate_synthetic_video(width, height, num_frames) |
| 41 | |
| 42 | # Videos should be different due to different seeds |
| 43 | assert video1["bytes"] != video2["bytes"] |
| 44 | |
| 45 | |
| 46 | @pytest.mark.benchmark |
| 47 | def test_map_config_to_modality(video_dataset: RandomMultiModalDataset): |
| 48 | """Test modality mapping for different configurations.""" |
| 49 | # Test image configuration (num_frames = 1) |
| 50 | assert video_dataset.map_config_to_modality((256, 256, 1)) == "image" |
| 51 | assert video_dataset.map_config_to_modality((720, 1280, 1)) == "image" |
| 52 | |
| 53 | # Test video configurations (num_frames > 1) |
| 54 | assert video_dataset.map_config_to_modality((256, 256, 8)) == "video" |
| 55 | assert video_dataset.map_config_to_modality((720, 1280, 16)) == "video" |
| 56 | assert video_dataset.map_config_to_modality((64, 64, 32)) == "video" |
| 57 | |
| 58 | # Test invalid configurations |
| 59 | with pytest.raises(ValueError, match="Invalid multimodal item configuration"): |
| 60 | video_dataset.map_config_to_modality((256, 256, 0)) |
| 61 | |
| 62 | with pytest.raises(ValueError, match="Invalid multimodal item configuration"): |
| 63 | video_dataset.map_config_to_modality((256, 256, -1)) |
| 64 | |
| 65 | |
| 66 | @pytest.mark.benchmark |
| 67 | def test_generate_mm_item_video( |
| 68 | video_dataset: RandomMultiModalDataset, |
| 69 | tmp_path: Path, |
| 70 | monkeypatch: pytest.MonkeyPatch, |
| 71 | ): |
| 72 | """Generate a decodable video without leaving temporary files behind.""" |
| 73 | monkeypatch.setattr(tempfile, "tempdir", str(tmp_path)) |
| 74 | # Test video item generation |
| 75 | video_config = (64, 48, 8) # height, width, num_frames |
| 76 | result = video_dataset.generate_mm_item(video_config) |
| 77 | assert not list(tmp_path.iterdir()) |
| 78 | |
| 79 | # Check the result structure matches OpenAI API format |
| 80 | assert isinstance(result, dict) |
| 81 | assert result["type"] == "video_url" |
| 82 | assert "video_url" in result |
| 83 | assert "url" in result["video_url"] |
| 84 | |
| 85 | # Check that the URL is a data URL with base64 encoded video |
| 86 | url = result["video_url"]["url"] |
| 87 | assert url.startswith("data:video/mp4;base64,") |
| 88 | |
| 89 | # Decode and verify the video content |
| 90 | base64_data = url.split(",")[1] |
| 91 | video_bytes = base64.b64decode(base64_data) |
| 92 | assert len(video_bytes) > 0 |
| 93 | |
| 94 | # Verify the video can be decoded |
| 95 | with NamedTemporaryFile(suffix=".mp4", delete=False) as temp_file: |
| 96 | temp_path = temp_file.name |
| 97 | temp_file.write(video_bytes) |
| 98 | |
| 99 | try: |
| 100 | cap = cv2.VideoCapture(temp_path) |
| 101 | assert cap.isOpened() |
| 102 | |
| 103 | frame_count = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) |
| 104 | frame_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) |
| 105 | frame_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) |
| 106 | |
| 107 | assert frame_count == 8 |
| 108 | assert frame_width == 48 |
| 109 | assert frame_height == 64 |
| 110 | |
| 111 | cap.release() |
| 112 | finally: |
| 113 | if os.path.exists(temp_path): |
| 114 | os.unlink(temp_path) |
| 115 | |
| 116 | |
| 117 | @pytest.mark.benchmark |
| 118 | @pytest.mark.parametrize("writer_opened", [False, True], ids=["open", "write"]) |
| 119 | def test_generate_synthetic_video_cleans_up_on_error( |
| 120 | video_dataset: RandomMultiModalDataset, |
| 121 | tmp_path: Path, |
| 122 | monkeypatch: pytest.MonkeyPatch, |
| 123 | writer_opened: bool, |
| 124 | ): |
| 125 | """Release the writer and remove temporary storage when encoding fails.""" |
| 126 | monkeypatch.setattr(tempfile, "tempdir", str(tmp_path)) |
| 127 | writer = Mock(spec=cv2.VideoWriter) |
| 128 | writer.isOpened.return_value = writer_opened |
| 129 | writer.write.side_effect = RuntimeError("Failed to write video frame") |
| 130 | monkeypatch.setattr(cv2, "VideoWriter", lambda *args, **kwargs: writer) |
| 131 | |
| 132 | error = ( |
| 133 | "Failed to write video frame" |
| 134 | if writer_opened |
| 135 | else "Failed to create video writer" |
| 136 | ) |
| 137 | with pytest.raises(RuntimeError, match=error): |
| 138 | video_dataset.generate_synthetic_video(48, 64, 8) |
| 139 | |
| 140 | assert not list(tmp_path.iterdir()) |
| 141 | writer.release.assert_called_once_with() |
| 142 | |
| 143 | |
| 144 | @pytest.mark.benchmark |
| 145 | def test_generate_mm_item_image(video_dataset: RandomMultiModalDataset): |
| 146 | """Test generating multimodal items for image configurations.""" |
| 147 | # Test image item generation |
| 148 | image_config = (64, 48, 1) # height, width, num_frames=1 |
| 149 | result = video_dataset.generate_mm_item(image_config) |
| 150 | |
| 151 | # Check the result structure matches OpenAI API format |
| 152 | assert isinstance(result, dict) |
| 153 | assert result["type"] == "image_url" |
| 154 | assert "image_url" in result |
| 155 | assert "url" in result["image_url"] |
| 156 | |
| 157 | # Check that the URL is a data URL with base64 encoded image |
| 158 | url = result["image_url"]["url"] |
| 159 | assert url.startswith("data:image/jpeg;base64,") |
| 160 | |
| 161 | |
| 162 | @pytest.mark.benchmark |
| 163 | def test_generate_mm_item_invalid_config(video_dataset: RandomMultiModalDataset): |
| 164 | """Test error handling for invalid configurations.""" |
| 165 | with pytest.raises(ValueError, match="Invalid multimodal item configuration"): |
| 166 | video_dataset.generate_mm_item((256, 256, 0)) |
| 167 | |
| 168 | |
| 169 | @pytest.mark.benchmark |
| 170 | def test_sample_with_video_buckets( |
| 171 | video_dataset: RandomMultiModalDataset, hf_tokenizer: PreTrainedTokenizerBase |
| 172 | ): |
| 173 | """Test sampling with video bucket configurations.""" |
| 174 | # Configure bucket with video probability > 0 |
| 175 | bucket_config = { |
| 176 | (64, 64, 1): 0.3, # Images |
| 177 | (64, 64, 8): 0.7, # Videos |
| 178 | } |
| 179 | |
| 180 | limit_mm_per_prompt = {"image": 5, "video": 3} |
| 181 | |
| 182 | samples = video_dataset.sample( |
| 183 | tokenizer=hf_tokenizer, |
| 184 | num_requests=5, |
| 185 | base_items_per_request=2, |
| 186 | num_mm_items_range_ratio=0.0, |
| 187 | limit_mm_per_prompt=limit_mm_per_prompt, |
| 188 | bucket_config=bucket_config, |
| 189 | input_len=20, |
| 190 | output_len=5, |
| 191 | ) |
| 192 | |
| 193 | assert len(samples) == 5 |
| 194 | |
| 195 | # Check that samples contain both images and videos |
| 196 | video_count = 0 |
| 197 | image_count = 0 |
| 198 | |
| 199 | for sample in samples: |
| 200 | assert isinstance(sample, SampleRequest) |
| 201 | assert sample.multi_modal_data is not None |
| 202 | assert isinstance(sample.multi_modal_data, list) |
| 203 | |
| 204 | mm_data = cast(list[dict[str, Any]], sample.multi_modal_data) |
| 205 | assert len(mm_data) == 2 # base_items_per_request |
| 206 | |
| 207 | for item in mm_data: |
| 208 | if item["type"] == "video_url": |
| 209 | video_count += 1 |
| 210 | # Verify video URL format |
| 211 | url = item["video_url"]["url"] |
| 212 | assert url.startswith("data:video/mp4;base64,") |
| 213 | elif item["type"] == "image_url": |
| 214 | image_count += 1 |
| 215 | # Verify image URL format |
| 216 | url = item["image_url"]["url"] |
| 217 | assert url.startswith("data:image/jpeg;base64,") |
| 218 | |
| 219 | # Should have some videos due to 0.7 probability |
| 220 | assert video_count > 0 |
| 221 | assert image_count > 0 |
| 222 | |
| 223 | |
| 224 | @pytest.mark.benchmark |
| 225 | def test_sample_video_only_buckets( |
| 226 | video_dataset: RandomMultiModalDataset, hf_tokenizer: PreTrainedTokenizerBase |
| 227 | ): |
| 228 | """Test sampling with only video buckets.""" |
| 229 | bucket_config = { |
| 230 | (64, 64, 8): 1.0, # Only videos |
| 231 | } |
| 232 | |
| 233 | limit_mm_per_prompt = {"image": 0, "video": 2} |
| 234 | |
| 235 | samples = video_dataset.sample( |
| 236 | tokenizer=hf_tokenizer, |
| 237 | num_requests=3, |
| 238 | base_items_per_request=1, |
| 239 | num_mm_items_range_ratio=0.0, |
| 240 | limit_mm_per_prompt=limit_mm_per_prompt, |
| 241 | bucket_config=bucket_config, |
| 242 | input_len=20, |
| 243 | output_len=5, |
| 244 | ) |
| 245 | |
| 246 | assert len(samples) == 3 |
| 247 | |
| 248 | for sample in samples: |
| 249 | assert isinstance(sample, SampleRequest) |
| 250 | assert sample.multi_modal_data is not None |
| 251 | assert isinstance(sample.multi_modal_data, list) |
| 252 | |
| 253 | mm_data = cast(list[dict[str, Any]], sample.multi_modal_data) |
| 254 | assert len(mm_data) == 1 |
| 255 | |
| 256 | item = mm_data[0] |
| 257 | assert item["type"] == "video_url" |
| 258 | url = item["video_url"]["url"] |
| 259 | assert url.startswith("data:video/mp4;base64,") |
| 260 | |
| 261 | |
| 262 | @pytest.mark.benchmark |
| 263 | def test_sample_respects_video_limits( |
| 264 | video_dataset: RandomMultiModalDataset, hf_tokenizer: PreTrainedTokenizerBase |
| 265 | ): |
| 266 | """Test that sampling respects video limits per prompt.""" |
| 267 | bucket_config = { |
| 268 | (64, 64, 8): 1.0, # Only videos |
| 269 | } |
| 270 | |
| 271 | # Set very low video limit |
| 272 | limit_mm_per_prompt = {"image": 0, "video": 1} |
| 273 | |
| 274 | samples = video_dataset.sample( |
| 275 | tokenizer=hf_tokenizer, |
| 276 | num_requests=3, |
| 277 | base_items_per_request=1, |
| 278 | num_mm_items_range_ratio=0.0, |
| 279 | limit_mm_per_prompt=limit_mm_per_prompt, |
| 280 | bucket_config=bucket_config, |
| 281 | input_len=20, |
| 282 | output_len=5, |
| 283 | ) |
| 284 | |
| 285 | assert len(samples) == 3 |
| 286 | |
| 287 | for sample in samples: |
| 288 | mm_data = cast(list[dict[str, Any]], sample.multi_modal_data) |
| 289 | assert len(mm_data) <= 1 # Should respect video limit |
| 290 | |
| 291 | |
| 292 | @pytest.mark.benchmark |
| 293 | def test_sample_mixed_buckets_with_zero_probability( |
| 294 | video_dataset: RandomMultiModalDataset, hf_tokenizer: PreTrainedTokenizerBase |
| 295 | ): |
| 296 | """Test sampling with mixed buckets including zero probability entries.""" |
| 297 | bucket_config = { |
| 298 | (64, 64, 1): 0.5, # Images |
| 299 | (64, 64, 8): 0.5, # Videos |
| 300 | (128, 128, 16): 0.0, # Zero probability videos (should be ignored) |
| 301 | } |
| 302 | |
| 303 | limit_mm_per_prompt = {"image": 2, "video": 2} |
| 304 | |
| 305 | samples = video_dataset.sample( |
| 306 | tokenizer=hf_tokenizer, |
| 307 | num_requests=4, |
| 308 | base_items_per_request=2, |
| 309 | num_mm_items_range_ratio=0.0, |
| 310 | limit_mm_per_prompt=limit_mm_per_prompt, |
| 311 | bucket_config=bucket_config, |
| 312 | input_len=20, |
| 313 | output_len=5, |
| 314 | ) |
| 315 | |
| 316 | assert len(samples) == 4 |
| 317 | |
| 318 | # Should only see 64x64 videos, not 128x128 videos |
| 319 | for sample in samples: |
| 320 | mm_data = cast(list[dict[str, Any]], sample.multi_modal_data) |
| 321 | for item in mm_data: |
| 322 | if item["type"] == "video_url": |
| 323 | # Decode video to verify dimensions |
| 324 | url = item["video_url"]["url"] |
| 325 | base64_data = url.split(",")[1] |
| 326 | video_bytes = base64.b64decode(base64_data) |
| 327 | |
| 328 | with NamedTemporaryFile(suffix=".mp4", delete=False) as temp_file: # noqa |
| 329 | temp_path = temp_file.name |
| 330 | temp_file.write(video_bytes) |
| 331 | |
| 332 | try: |
| 333 | cap = cv2.VideoCapture(temp_path) |
| 334 | frame_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) |
| 335 | frame_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) |
| 336 | cap.release() |
| 337 | |
| 338 | # Should be 64x64, not 128x128 |
| 339 | assert frame_width == 64 |
| 340 | assert frame_height == 64 |
| 341 | finally: |
| 342 | if os.path.exists(temp_path): |
| 343 | os.unlink(temp_path) |
| 344 | |
| 345 | |
| 346 | @pytest.mark.benchmark |
| 347 | def test_sample_deterministic_with_videos(hf_tokenizer: PreTrainedTokenizerBase): |
| 348 | """Test that sampling with videos is deterministic with same seed.""" |
| 349 | dataset1 = RandomMultiModalDataset(random_seed=123) |
| 350 | dataset2 = RandomMultiModalDataset(random_seed=123) |
| 351 | |
| 352 | bucket_config = { |
| 353 | (64, 64, 1): 0.3, # Images |
| 354 | (64, 64, 8): 0.7, # Videos |
| 355 | } |
| 356 | |
| 357 | limit_mm_per_prompt = {"image": 2, "video": 2} |
| 358 | |
| 359 | samples1 = dataset1.sample( |
| 360 | tokenizer=hf_tokenizer, |
| 361 | num_requests=3, |
| 362 | base_items_per_request=1, |
| 363 | num_mm_items_range_ratio=0.0, |
| 364 | limit_mm_per_prompt=limit_mm_per_prompt, |
| 365 | bucket_config=bucket_config, |
| 366 | input_len=20, |
| 367 | output_len=5, |
| 368 | ) |
| 369 | |
| 370 | samples2 = dataset2.sample( |
| 371 | tokenizer=hf_tokenizer, |
| 372 | num_requests=3, |
| 373 | base_items_per_request=1, |
| 374 | num_mm_items_range_ratio=0.0, |
| 375 | limit_mm_per_prompt=limit_mm_per_prompt, |
| 376 | bucket_config=bucket_config, |
| 377 | input_len=20, |
| 378 | output_len=5, |
| 379 | ) |
| 380 | |
| 381 | assert len(samples1) == len(samples2) |
| 382 | |
| 383 | # Compare multimodal data |
| 384 | for s1, s2 in zip(samples1, samples2): |
| 385 | assert s1.multi_modal_data == s2.multi_modal_data |
| 386 | |
| 387 | |
| 388 | @pytest.mark.benchmark |
| 389 | def test_sample_different_seeds_produce_different_videos( |
| 390 | hf_tokenizer: PreTrainedTokenizerBase, |
| 391 | ): |
| 392 | """Test that different seeds produce different video content.""" |
| 393 | dataset1 = RandomMultiModalDataset(random_seed=123) |
| 394 | dataset2 = RandomMultiModalDataset(random_seed=456) |
| 395 | |
| 396 | bucket_config = { |
| 397 | (64, 64, 8): 1.0, # Only videos |
| 398 | } |
| 399 | |
| 400 | limit_mm_per_prompt = {"image": 0, "video": 1} |
| 401 | |
| 402 | samples1 = dataset1.sample( |
| 403 | tokenizer=hf_tokenizer, |
| 404 | num_requests=2, |
| 405 | base_items_per_request=1, |
| 406 | num_mm_items_range_ratio=0.0, |
| 407 | limit_mm_per_prompt=limit_mm_per_prompt, |
| 408 | bucket_config=bucket_config, |
| 409 | input_len=20, |
| 410 | output_len=5, |
| 411 | ) |
| 412 | |
| 413 | samples2 = dataset2.sample( |
| 414 | tokenizer=hf_tokenizer, |
| 415 | num_requests=2, |
| 416 | base_items_per_request=1, |
| 417 | num_mm_items_range_ratio=0.0, |
| 418 | limit_mm_per_prompt=limit_mm_per_prompt, |
| 419 | bucket_config=bucket_config, |
| 420 | input_len=20, |
| 421 | output_len=5, |
| 422 | ) |
| 423 | |
| 424 | # Video content should be different |
| 425 | for s1, s2 in zip(samples1, samples2): |
| 426 | mm_data1 = cast(list[dict[str, Any]], s1.multi_modal_data) |
| 427 | mm_data2 = cast(list[dict[str, Any]], s2.multi_modal_data) |
| 428 | |
| 429 | assert len(mm_data1) == len(mm_data2) == 1 |
| 430 | |
| 431 | url1 = mm_data1[0]["video_url"]["url"] |
| 432 | url2 = mm_data2[0]["video_url"]["url"] |
| 433 | |
| 434 | assert url1 != url2 # Different video content |