A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 15:15 UTC
[Frontend] Implement robust video frame recovery for corrupted videos (#29197)
Signed-off-by: cmartinez <cmartinez@roblox.com> Signed-off-by: vSeamar <cmartinez@roblox.com>
vSeamar committed
Jan 6, 2026 at 17:13 UTC
6f351548b258d7ff618174817bfbdc0ee4758fb5
4 files changed
+421
-15
docs/features/multimodal_inputs.md
+25
@@ -689,6 +689,31 @@ Full example: [examples/online_serving/openai_chat_completion_client_for_multimo
689
export VLLM_VIDEO_FETCH_TIMEOUT=<timeout>
690
```
691
692
+#### Video Frame Recovery
693
+
694
+For improved robustness when processing potentially corrupted or truncated video files, vLLM supports optional frame recovery using a dynamic window forward-scan approach. When enabled, if a target frame fails to load during sequential reading, the next successfully grabbed frame (before the next target frame) will be used in its place.
695
+
696
+To enable video frame recovery, pass the `frame_recovery` parameter via `--media-io-kwargs`:
697
+
698
+```bash
699
+# Example: Enable frame recovery
700
+vllm serve Qwen/Qwen3-VL-30B-A3B-Instruct \
701
+ --media-io-kwargs '{"video": {"frame_recovery": true}}'
702
+```
703
+
704
+**Parameters:**
705
+
706
+- `frame_recovery`: Boolean flag to enable forward-scan recovery. When `true`, failed frames are recovered using the next available frame within the dynamic window (up to the next target frame). Default is `false`.
707
+
708
+**How it works:**
709
+
710
+1. The system reads frames sequentially
711
+2. If a target frame fails to grab, it's marked as "failed"
712
+3. The next successfully grabbed frame (before reaching the next target) is used to recover the failed frame
713
+4. This approach handles both mid-video corruption and end-of-video truncation
714
+
715
+Works with common video formats like MP4 when using OpenCV backends.
716
+
717
#### Custom RGBA Background Color
718
719
To use a custom background color for RGBA images, pass the `rgba_background_color` parameter via `--media-io-kwargs`:
tests/multimodal/test_video.py
+209
@@ -299,3 +299,212 @@ def test_video_media_io_backend_env_var_fallback(monkeypatch: pytest.MonkeyPatch
299
frames_missing, metadata_missing = videoio_missing.load_bytes(b"test")
300
np.testing.assert_array_equal(frames_missing, FAKE_OUTPUT_2)
301
assert metadata_missing["video_backend"] == "test_video_backend_override_2"
302
+
303
+
304
+# ============================================================================
305
+# Frame Recovery Tests
306
+# ============================================================================
307
+
308
+
309
+def test_video_recovery_simulated_failures(monkeypatch: pytest.MonkeyPatch):
310
+ """
311
+ Test that frame recovery correctly uses the next valid frame when
312
+ target frames fail to load.
313
+
314
+ Uses corrupted.mp4 and mocks VideoCapture.grab() to fail on specific
315
+ frame indices (in addition to the real corruption at frame 17), then
316
+ verifies recovery produces more frames.
317
+ """
318
+ import cv2
319
+
320
+ with monkeypatch.context() as m:
321
+ m.setenv("VLLM_VIDEO_LOADER_BACKEND", "opencv")
322
+
323
+ # Load corrupted.mp4 (26 frames, frame 17 is genuinely corrupted)
324
+ video_path = ASSETS_DIR / "corrupted.mp4"
325
+ with open(video_path, "rb") as f:
326
+ video_data = f.read()
327
+
328
+ # Simulate additional failures on frames 3 and 10
329
+ # (in addition to the real corruption at frame 17)
330
+ fail_on_frames = {3, 10}
331
+
332
+ # Store original VideoCapture class
333
+ original_video_capture = cv2.VideoCapture
334
+
335
+ class MockVideoCapture:
336
+ """Wrapper that simulates grab() failures on specific frames."""
337
+
338
+ def __init__(self, *args, **kwargs):
339
+ self._cap = original_video_capture(*args, **kwargs)
340
+ self._current_frame = -1
341
+
342
+ def grab(self):
343
+ self._current_frame += 1
344
+ if self._current_frame in fail_on_frames:
345
+ return False # Simulate failure
346
+ return self._cap.grab()
347
+
348
+ def retrieve(self):
349
+ return self._cap.retrieve()
350
+
351
+ def get(self, prop):
352
+ return self._cap.get(prop)
353
+
354
+ def isOpened(self):
355
+ return self._cap.isOpened()
356
+
357
+ def release(self):
358
+ return self._cap.release()
359
+
360
+ # Patch cv2.VideoCapture
361
+ m.setattr(cv2, "VideoCapture", MockVideoCapture)
362
+
363
+ loader = VIDEO_LOADER_REGISTRY.load("opencv")
364
+
365
+ # Use num_frames=8 which samples: [0, 3, 7, 10, 14, 17, 21, 25]
366
+ # Frame 3: mocked failure, recovery window [3, 7) -> use frame 4
367
+ # Frame 10: mocked failure, recovery window [10, 14) -> use frame 11
368
+ # Frame 17: real corruption, recovery window [17, 21) -> use frame 18
369
+
370
+ # Test WITHOUT recovery - should have fewer frames due to failures
371
+ frames_no_recovery, meta_no = loader.load_bytes(
372
+ video_data, num_frames=8, frame_recovery=False
373
+ )
374
+
375
+ # Test WITH recovery - should recover using next valid frames
376
+ frames_with_recovery, meta_yes = loader.load_bytes(
377
+ video_data, num_frames=8, frame_recovery=True
378
+ )
379
+
380
+ # With recovery should have MORE frames than without
381
+ # Without: 5 frames (3, 10, 17 all fail)
382
+ # With: 8 frames (all recovered)
383
+ assert frames_with_recovery.shape[0] > frames_no_recovery.shape[0], (
384
+ f"Recovery should produce more frames. "
385
+ f"Without: {frames_no_recovery.shape[0]}, "
386
+ f"With: {frames_with_recovery.shape[0]}"
387
+ )
388
+
389
+ # Verify metadata consistency
390
+ assert frames_no_recovery.shape[0] == len(meta_no["frames_indices"])
391
+ assert frames_with_recovery.shape[0] == len(meta_yes["frames_indices"])
392
+
393
+ # Verify temporal order is preserved
394
+ assert meta_yes["frames_indices"] == sorted(meta_yes["frames_indices"])
395
+
396
+
397
+def test_video_recovery_with_corrupted_file(monkeypatch: pytest.MonkeyPatch):
398
+ """
399
+ Test frame recovery with an actual corrupted video file using sparse sampling.
400
+
401
+ This test uses corrupted.mp4 which has genuine H.264 codec errors on
402
+ frame 17. With num_frames=8, the target frames are [0, 3, 7, 10, 14, 17, 21, 25].
403
+ Frame 17 is corrupted but frames 18-20 are readable, so recovery can use
404
+ frame 18 to fill in for the failed frame 17.
405
+
406
+ This test verifies:
407
+ 1. Without recovery: frame 17 is skipped (7 frames loaded)
408
+ 2. With recovery: frame 18 fills in for frame 17 (8 frames loaded)
409
+ 3. Recovery produces MORE frames than without recovery
410
+ 4. Metadata is consistent with loaded frames
411
+ """
412
+ with monkeypatch.context() as m:
413
+ m.setenv("VLLM_VIDEO_LOADER_BACKEND", "opencv")
414
+
415
+ corrupted_video_path = ASSETS_DIR / "corrupted.mp4"
416
+
417
+ with open(corrupted_video_path, "rb") as f:
418
+ video_data = f.read()
419
+
420
+ loader = VIDEO_LOADER_REGISTRY.load("opencv")
421
+
422
+ # Use num_frames=8 which makes frame 17 a target with recovery window [17, 21)
423
+ # Target frames: [0, 3, 7, 10, 14, 17, 21, 25]
424
+ # Frame 17 is corrupted, but frames 18-20 are readable for recovery
425
+
426
+ # Test without recovery - frame 17 will be skipped
427
+ frames_no_recovery, meta_no_recovery = loader.load_bytes(
428
+ video_data, num_frames=8, frame_recovery=False
429
+ )
430
+
431
+ # Test with recovery - frame 18 should fill in for frame 17
432
+ frames_with_recovery, meta_with_recovery = loader.load_bytes(
433
+ video_data, num_frames=8, frame_recovery=True
434
+ )
435
+
436
+ # Verify metadata consistency for both modes
437
+ assert frames_no_recovery.shape[0] == len(meta_no_recovery["frames_indices"]), (
438
+ "Frame count must match indices without recovery"
439
+ )
440
+ assert frames_with_recovery.shape[0] == len(
441
+ meta_with_recovery["frames_indices"]
442
+ ), "Frame count must match indices with recovery"
443
+
444
+ # KEY ASSERTION: Recovery should produce MORE frames than without recovery
445
+ # Without recovery: 7 frames (frame 17 skipped)
446
+ # With recovery: 8 frames (frame 18 used for frame 17)
447
+ assert frames_with_recovery.shape[0] > frames_no_recovery.shape[0], (
448
+ f"Recovery should produce more frames with sparse sampling. "
449
+ f"Got {frames_with_recovery.shape[0]} with recovery vs "
450
+ f"{frames_no_recovery.shape[0]} without"
451
+ )
452
+
453
+ # Verify we got all 8 requested frames with recovery
454
+ assert frames_with_recovery.shape[0] == 8, (
455
+ f"With recovery, should load all 8 requested frames. "
456
+ f"Got {frames_with_recovery.shape[0]}"
457
+ )
458
+
459
+ # Verify the video metadata is correct
460
+ expected_total_frames = 26
461
+ assert meta_with_recovery["total_num_frames"] == expected_total_frames, (
462
+ f"Expected {expected_total_frames} total frames in metadata"
463
+ )
464
+
465
+
466
+def test_video_recovery_dynamic_backend(monkeypatch: pytest.MonkeyPatch):
467
+ """
468
+ Test that frame_recovery works with the dynamic video backend.
469
+
470
+ The dynamic backend samples frames based on fps/duration rather than
471
+ loading all frames. This test verifies recovery works in that context.
472
+ """
473
+ with monkeypatch.context() as m:
474
+ m.setenv("VLLM_VIDEO_LOADER_BACKEND", "opencv_dynamic")
475
+
476
+ corrupted_video_path = ASSETS_DIR / "corrupted.mp4"
477
+
478
+ with open(corrupted_video_path, "rb") as f:
479
+ video_data = f.read()
480
+
481
+ loader = VIDEO_LOADER_REGISTRY.load("opencv_dynamic")
482
+
483
+ # Test without recovery
484
+ frames_no_recovery, meta_no = loader.load_bytes(
485
+ video_data, fps=2, max_duration=10, frame_recovery=False
486
+ )
487
+
488
+ # Test with frame_recovery enabled
489
+ frames_with_recovery, meta_with = loader.load_bytes(
490
+ video_data, fps=2, max_duration=10, frame_recovery=True
491
+ )
492
+
493
+ # Verify basic properties
494
+ assert frames_no_recovery.shape[0] > 0, (
495
+ "Should load some frames without recovery"
496
+ )
497
+ assert frames_with_recovery.shape[0] > 0, (
498
+ "Should load some frames with recovery"
499
+ )
500
+ assert "do_sample_frames" in meta_with
501
+ assert meta_with["do_sample_frames"] is False # Dynamic backend always False
502
+ assert frames_with_recovery.shape[0] == len(meta_with["frames_indices"])
503
+
504
+ # Key assertion: recovery should help when corrupted frames are sampled
505
+ # We expect recovery to produce >= frames than without recovery
506
+ assert frames_with_recovery.shape[0] >= frames_no_recovery.shape[0], (
507
+ f"Recovery should produce at least as many frames. "
508
+ f"Got {frames_with_recovery.shape[0]} with recovery vs "
509
+ f"{frames_no_recovery.shape[0]} without"
510
+ )
vllm/benchmarks/datasets.py
+1
-1
@@ -867,7 +867,7 @@ class RandomMultiModalDataset(RandomDataset):
867
fourcc = cv2.VideoWriter_fourcc(*"mp4v")
868
fps = 30 # frames per second
869
870
- with NamedTemporaryFile(suffix=".mp4", delete_on_close=False) as temp_file:
870
+ with NamedTemporaryFile(suffix=".mp4", delete=False) as temp_file:
871
temp_path = temp_file.name
872
873
# Create video writer
vllm/multimodal/video.py
+186
-14
@@ -6,12 +6,15 @@ from abc import abstractmethod
6
from functools import partial
7
from io import BytesIO
8
from pathlib import Path
9
-from typing import Any
9
+from typing import TYPE_CHECKING, Any
10
11
import numpy as np
12
import numpy.typing as npt
13
from PIL import Image
14
15
+if TYPE_CHECKING:
16
+ import cv2
17
+
18
from vllm import envs
19
from vllm.logger import init_logger
20
from vllm.utils.registry import ExtensionManager
@@ -63,6 +66,127 @@ class VideoLoader:
66
) -> tuple[npt.NDArray, dict[str, Any]]:
67
raise NotImplementedError
68
69
+ @staticmethod
70
+ def _can_use_for_recovery(
71
+ idx: int,
72
+ failed_frames: list[int],
73
+ next_target_map: dict[int, int],
74
+ total_frames: int,
75
+ ) -> bool:
76
+ """Check if current frame can recover the oldest failed frame."""
77
+ if not failed_frames:
78
+ return False
79
+ oldest_failed = failed_frames[0]
80
+ limit = next_target_map.get(oldest_failed, total_frames)
81
+ return idx < limit
82
+
83
+ @staticmethod
84
+ def _read_frames_with_recovery(
85
+ cap: "cv2.VideoCapture",
86
+ frame_indices: list[int],
87
+ total_frames: int,
88
+ ) -> tuple[npt.NDArray, list[int], dict[int, int]]:
89
+ """
90
+ Read frames with dynamic window forward-scan recovery.
91
+
92
+ When a target frame fails to load, the next successfully grabbed
93
+ frame (before the next target frame) will be used to recover it.
94
+
95
+ Args:
96
+ cap: OpenCV VideoCapture object
97
+ frame_indices: Sorted list of target frame indices to load
98
+ total_frames: Total number of frames in the video
99
+
100
+ Returns:
101
+ Tuple of (frames_array, valid_frame_indices, recovered_map)
102
+ - frames_array: Array of loaded frames
103
+ - valid_frame_indices: List of frame indices that were loaded
104
+ - recovered_map: Dict mapping recovered_idx -> source_idx
105
+ """
106
+ import cv2
107
+
108
+ width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
109
+ height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
110
+
111
+ assert width > 0 and height > 0, (
112
+ f"Invalid video frame size: width={width}, height={height}"
113
+ )
114
+
115
+ frame_idx_set = set(frame_indices)
116
+ max_frame_idx = frame_indices[-1] if frame_indices else 0
117
+
118
+ # Build map: target_idx -> next_target_idx (for recovery window)
119
+ next_target_map: dict[int, int] = {}
120
+ for k in range(len(frame_indices) - 1):
121
+ next_target_map[frame_indices[k]] = frame_indices[k + 1]
122
+ next_target_map[frame_indices[-1]] = total_frames
123
+
124
+ frames_list: list[npt.NDArray] = []
125
+ valid_frame_indices: list[int] = []
126
+ failed_frames_idx: list[int] = []
127
+ recovered_map: dict[int, int] = {}
128
+
129
+ i = 0
130
+ for idx in range(max_frame_idx + 1):
131
+ is_target_frame = idx in frame_idx_set
132
+
133
+ # Attempt to grab the current frame
134
+ ok = cap.grab()
135
+
136
+ if not ok:
137
+ if is_target_frame:
138
+ logger.warning(
139
+ "Failed to grab frame %d during video loading.",
140
+ idx,
141
+ )
142
+ failed_frames_idx.append(idx)
143
+ continue
144
+
145
+ # Check if we should retrieve: target frame OR can recover a failed one
146
+ can_recover = VideoLoader._can_use_for_recovery(
147
+ idx, failed_frames_idx, next_target_map, total_frames
148
+ )
149
+
150
+ if is_target_frame or can_recover:
151
+ ret, frame = cap.retrieve()
152
+
153
+ if ret and frame is not None and frame.size > 0:
154
+ rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
155
+ frames_list.append(rgb_frame)
156
+ valid_frame_indices.append(idx)
157
+ i += 1
158
+
159
+ if can_recover:
160
+ recovered_idx = failed_frames_idx.pop(0)
161
+ recovered_map[recovered_idx] = idx
162
+ logger.info(
163
+ "Recovered frame %d using frame %d (delay: %d)",
164
+ recovered_idx,
165
+ idx,
166
+ idx - recovered_idx,
167
+ )
168
+ elif is_target_frame:
169
+ logger.warning(
170
+ "Failed to retrieve frame %d during video loading.",
171
+ idx,
172
+ )
173
+ failed_frames_idx.append(idx)
174
+
175
+ # Log any remaining failed frames
176
+ for failed_idx in failed_frames_idx:
177
+ logger.warning(
178
+ "Frame %d could not be recovered (end of video).",
179
+ failed_idx,
180
+ )
181
+
182
+ # Stack frames
183
+ if frames_list:
184
+ frames = np.stack(frames_list)
185
+ else:
186
+ frames = np.empty((0, height, width, 3), dtype=np.uint8)
187
+
188
+ return frames, valid_frame_indices, recovered_map
189
+
190
@staticmethod
191
def _read_frames(
192
cap,
@@ -142,8 +266,23 @@ class OpenCVVideoBackend(VideoLoader):
266
data: bytes,
267
num_frames: int = -1,
268
fps: int = -1,
269
+ max_duration: int = 300,
270
+ frame_recovery: bool = False,
271
**kwargs,
272
) -> tuple[npt.NDArray, dict[str, Any]]:
273
+ """
274
+ Load video frames from bytes.
275
+
276
+ Args:
277
+ data: Raw video bytes
278
+ num_frames: Target number of frames to sample (-1 for all)
279
+ fps: Target FPS for sampling (-1 for original)
280
+ max_duration: Maximum duration (unused in base backend)
281
+ frame_recovery: Enable forward-scan recovery for failed frames
282
+
283
+ Returns:
284
+ Tuple of (frames_array, metadata_dict)
285
+ """
286
import cv2
287
288
backend = cls().get_cv2_video_api()
@@ -172,11 +311,22 @@ class OpenCVVideoBackend(VideoLoader):
311
)
312
frame_idx = uniform_sampled_frames.tolist()
313
175
- # Convert to set for O(1) lookup performance
176
- frame_idx_set = set(frame_idx)
177
- frames, valid_num_frames, valid_frame_indices = cls._read_frames(
178
- cap, frame_idx_set, num_frames_to_sample, max(frame_idx)
179
- )
314
+ if frame_recovery:
315
+ frames, valid_frame_indices, recovered_map = cls._read_frames_with_recovery(
316
+ cap, frame_idx, total_frames_num
317
+ )
318
+ valid_num_frames = len(valid_frame_indices)
319
+
320
+ if recovered_map:
321
+ logger.info(
322
+ "Frame recovery: %d frames recovered using forward scan.",
323
+ len(recovered_map),
324
+ )
325
+ else:
326
+ frame_idx_set = set(frame_idx)
327
+ frames, valid_num_frames, valid_frame_indices = cls._read_frames(
328
+ cap, frame_idx_set, num_frames_to_sample, max(frame_idx)
329
+ )
330
331
# Use transformers transformers.video_utils.VideoMetadata format
332
# NOTE(Isotr0py): For models like Qwen3-VL/GLM4.5V, this metadata
@@ -204,8 +354,22 @@ class OpenCVDynamicVideoBackend(OpenCVVideoBackend):
354
num_frames: int = -1,
355
fps: int = 2,
356
max_duration: int = 300,
357
+ frame_recovery: bool = False,
358
**kwargs,
359
) -> tuple[npt.NDArray, dict[str, Any]]:
360
+ """
361
+ Load video frames with dynamic sampling based on duration.
362
+
363
+ Args:
364
+ data: Raw video bytes
365
+ num_frames: Not used in dynamic backend
366
+ fps: Target FPS for sampling (default: 2)
367
+ max_duration: Maximum video duration to process (default: 300s)
368
+ frame_recovery: Enable forward-scan recovery for failed frames
369
+
370
+ Returns:
371
+ Tuple of (frames_array, metadata_dict)
372
+ """
373
import cv2
374
375
backend = cls().get_cv2_video_api()
@@ -245,14 +409,22 @@ class OpenCVDynamicVideoBackend(OpenCVVideoBackend):
409
}
410
)
411
248
- # Convert to set for O(1) lookup performance
249
- frame_indices_set = set(frame_indices_list)
250
- frames, valid_num_frames, valid_frame_indices = cls._read_frames(
251
- cap,
252
- frame_indices_set,
253
- len(frame_indices_list),
254
- total_frames_num - 1,
255
- )
412
+ if frame_recovery:
413
+ frames, valid_frame_indices, recovered_map = cls._read_frames_with_recovery(
414
+ cap, frame_indices_list, total_frames_num
415
+ )
416
+ valid_num_frames = len(valid_frame_indices)
417
+
418
+ if recovered_map:
419
+ logger.info(
420
+ "Frame recovery: %d frames recovered using forward scan.",
421
+ len(recovered_map),
422
+ )
423
+ else:
424
+ frame_indices_set = set(frame_indices_list)
425
+ frames, valid_num_frames, valid_frame_indices = cls._read_frames(
426
+ cap, frame_indices_set, len(frame_indices_list), total_frames_num - 1
427
+ )
428
429
# Use transformers transformers.video_utils.VideoMetadata format
430
metadata = {