[Bugfix][Bench] Clean up synthetic video files and writers (#59529)
Signed-off-by: Prudhvivuda <prudhvi12042001@gmail.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
Prudhvi Vuda committed
Oct 1, 2026 at 22:51 UTC
ea106db806918f371b8f392cc896dc7a2d6677a1
2 files changed
+50
-20
tests/benchmarks/test_random_multimodal_dataset_video.py
+38
-2
@@ -2,8 +2,11 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
import os
5
+import tempfile
6
+from pathlib import Path
7
from tempfile import NamedTemporaryFile
8
from typing import Any, cast
9
+from unittest.mock import Mock
10
11
import cv2
12
import pybase64 as base64
@@ -61,11 +64,17 @@ def test_map_config_to_modality(video_dataset: RandomMultiModalDataset):
64
65
66
@pytest.mark.benchmark
64
-def test_generate_mm_item_video(video_dataset: RandomMultiModalDataset):
65
- """Test generating multimodal items for video configurations."""
67
+def test_generate_mm_item_video(
68
+ video_dataset: RandomMultiModalDataset,
69
+ tmp_path: Path,
70
+ monkeypatch: pytest.MonkeyPatch,
71
+):
72
+ """Generate a decodable video without leaving temporary files behind."""
73
+ monkeypatch.setattr(tempfile, "tempdir", str(tmp_path))
74
# Test video item generation
75
video_config = (64, 48, 8) # height, width, num_frames
76
result = video_dataset.generate_mm_item(video_config)
77
+ assert not list(tmp_path.iterdir())
78
79
# Check the result structure matches OpenAI API format
80
assert isinstance(result, dict)
@@ -105,6 +114,33 @@ def test_generate_mm_item_video(video_dataset: RandomMultiModalDataset):
114
os.unlink(temp_path)
115
116
117
+@pytest.mark.benchmark
118
+@pytest.mark.parametrize("writer_opened", [False, True], ids=["open", "write"])
119
+def test_generate_synthetic_video_cleans_up_on_error(
120
+ video_dataset: RandomMultiModalDataset,
121
+ tmp_path: Path,
122
+ monkeypatch: pytest.MonkeyPatch,
123
+ writer_opened: bool,
124
+):
125
+ """Release the writer and remove temporary storage when encoding fails."""
126
+ monkeypatch.setattr(tempfile, "tempdir", str(tmp_path))
127
+ writer = Mock(spec=cv2.VideoWriter)
128
+ writer.isOpened.return_value = writer_opened
129
+ writer.write.side_effect = RuntimeError("Failed to write video frame")
130
+ monkeypatch.setattr(cv2, "VideoWriter", lambda *args, **kwargs: writer)
131
+
132
+ error = (
133
+ "Failed to write video frame"
134
+ if writer_opened
135
+ else "Failed to create video writer"
136
+ )
137
+ with pytest.raises(RuntimeError, match=error):
138
+ video_dataset.generate_synthetic_video(48, 64, 8)
139
+
140
+ assert not list(tmp_path.iterdir())
141
+ writer.release.assert_called_once_with()
142
+
143
+
144
@pytest.mark.benchmark
145
def test_generate_mm_item_image(video_dataset: RandomMultiModalDataset):
146
"""Test generating multimodal items for image configurations."""
vllm/benchmarks/datasets/datasets.py
+12
-18
@@ -25,7 +25,7 @@ from dataclasses import dataclass, replace
25
from functools import cache
26
from io import BytesIO
27
from pathlib import Path
28
-from tempfile import NamedTemporaryFile
28
+from tempfile import TemporaryDirectory
29
from typing import Any, cast
30
31
import numpy as np
@@ -975,32 +975,26 @@ class RandomMultiModalDataset(RandomDataset):
975
dtype=np.uint8,
976
)
977
978
- # Create a temporary video file in memory
978
fourcc = cv2.VideoWriter_fourcc(*"mp4v")
979
fps = 30 # frames per second
980
982
- with NamedTemporaryFile(suffix=".mp4", delete=False) as temp_file:
983
- temp_path = temp_file.name
981
+ with TemporaryDirectory() as temp_dir:
982
+ temp_path = Path(temp_dir) / "video.mp4"
983
985
- # Create video writer
984
video_writer = cv2.VideoWriter(
987
- temp_path, fourcc=fourcc, fps=fps, frameSize=(width, height)
985
+ str(temp_path), fourcc=fourcc, fps=fps, frameSize=(width, height)
986
)
987
990
- if not video_writer.isOpened():
991
- raise RuntimeError("Failed to create video writer")
992
-
993
- for frame in random_pixels:
994
- video_writer.write(frame)
995
-
996
- video_writer.release()
997
- temp_file.close()
988
+ try:
989
+ if not video_writer.isOpened():
990
+ raise RuntimeError("Failed to create video writer")
991
999
- # Read the video file content
1000
- with open(temp_path, "rb") as f:
1001
- video_content = f.read()
992
+ for frame in random_pixels:
993
+ video_writer.write(frame)
994
+ finally:
995
+ video_writer.release()
996
1003
- return {"bytes": video_content}
997
+ return {"bytes": temp_path.read_bytes()}
998
999
def map_config_to_modality(self, config: tuple[int, int, int]) -> str:
1000
"""Map the configuration to the modality."""