A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 16:15 UTC
[Fix] Update lazing loading of video loader backend (#30444)
Signed-off-by: Jeremy Teboul <jeremyteboul@fb.com> Co-authored-by: Jeremy Teboul <jeremyteboul@fb.com>
jeremyteboul committed
Dec 11, 2025 at 02:14 UTC
4515eb1a0b71fbdca68c95eb261b046bbd830d15
2 files changed
+131
-2
tests/multimodal/test_video.py
+123
-1
@@ -147,7 +147,7 @@ def test_video_backend_handles_broken_frames(monkeypatch: pytest.MonkeyPatch):
147
"""
148
Regression test for handling videos with broken frames.
149
This test uses a pre-corrupted video file (assets/corrupted.mp4) that
150
- contains broken/unreadable frames to verify the video loader handles
150
+ contains broken frames to verify the video loader handles
151
them gracefully without crashing and returns accurate metadata.
152
"""
153
with monkeypatch.context() as m:
@@ -177,3 +177,125 @@ def test_video_backend_handles_broken_frames(monkeypatch: pytest.MonkeyPatch):
177
f"Expected fewer than {metadata['total_num_frames']} frames, "
178
f"but loaded {frames.shape[0]} frames"
179
)
180
+
181
+
182
+@VIDEO_LOADER_REGISTRY.register("test_video_backend_override_1")
183
+class TestVideoBackendOverride1(VideoLoader):
184
+ """Test loader that returns FAKE_OUTPUT_1 to verify backend selection."""
185
+
186
+ @classmethod
187
+ def load_bytes(
188
+ cls, data: bytes, num_frames: int = -1, **kwargs
189
+ ) -> tuple[npt.NDArray, dict]:
190
+ return FAKE_OUTPUT_1, {"video_backend": "test_video_backend_override_1"}
191
+
192
+
193
+@VIDEO_LOADER_REGISTRY.register("test_video_backend_override_2")
194
+class TestVideoBackendOverride2(VideoLoader):
195
+ """Test loader that returns FAKE_OUTPUT_2 to verify backend selection."""
196
+
197
+ @classmethod
198
+ def load_bytes(
199
+ cls, data: bytes, num_frames: int = -1, **kwargs
200
+ ) -> tuple[npt.NDArray, dict]:
201
+ return FAKE_OUTPUT_2, {"video_backend": "test_video_backend_override_2"}
202
+
203
+
204
+def test_video_media_io_backend_kwarg_override(monkeypatch: pytest.MonkeyPatch):
205
+ """
206
+ Test that video_backend kwarg can override the VLLM_VIDEO_LOADER_BACKEND
207
+ environment variable.
208
+
209
+ This allows users to dynamically select a different video backend
210
+ via --media-io-kwargs without changing the global env var, which is
211
+ useful when plugins set a default backend but a specific request
212
+ needs a different one.
213
+ """
214
+ with monkeypatch.context() as m:
215
+ # Set the env var to one backend
216
+ m.setenv("VLLM_VIDEO_LOADER_BACKEND", "test_video_backend_override_1")
217
+
218
+ imageio = ImageMediaIO()
219
+
220
+ # Without video_backend kwarg, should use env var backend
221
+ videoio_default = VideoMediaIO(imageio, num_frames=10)
222
+ frames_default, metadata_default = videoio_default.load_bytes(b"test")
223
+ np.testing.assert_array_equal(frames_default, FAKE_OUTPUT_1)
224
+ assert metadata_default["video_backend"] == "test_video_backend_override_1"
225
+
226
+ # With video_backend kwarg, should override env var
227
+ videoio_override = VideoMediaIO(
228
+ imageio, num_frames=10, video_backend="test_video_backend_override_2"
229
+ )
230
+ frames_override, metadata_override = videoio_override.load_bytes(b"test")
231
+ np.testing.assert_array_equal(frames_override, FAKE_OUTPUT_2)
232
+ assert metadata_override["video_backend"] == "test_video_backend_override_2"
233
+
234
+
235
+def test_video_media_io_backend_kwarg_not_passed_to_loader(
236
+ monkeypatch: pytest.MonkeyPatch,
237
+):
238
+ """
239
+ Test that video_backend kwarg is consumed by VideoMediaIO and NOT passed
240
+ through to the underlying video loader's load_bytes method.
241
+
242
+ This ensures the kwarg is properly popped from kwargs before forwarding.
243
+ """
244
+
245
+ @VIDEO_LOADER_REGISTRY.register("test_reject_video_backend_kwarg")
246
+ class RejectVideoBackendKwargLoader(VideoLoader):
247
+ """Test loader that fails if video_backend is passed through."""
248
+
249
+ @classmethod
250
+ def load_bytes(
251
+ cls, data: bytes, num_frames: int = -1, **kwargs
252
+ ) -> tuple[npt.NDArray, dict]:
253
+ # This should never receive video_backend in kwargs
254
+ if "video_backend" in kwargs:
255
+ raise AssertionError(
256
+ "video_backend should be consumed by VideoMediaIO, "
257
+ "not passed to loader"
258
+ )
259
+ return FAKE_OUTPUT_1, {"received_kwargs": list(kwargs.keys())}
260
+
261
+ with monkeypatch.context() as m:
262
+ m.setenv("VLLM_VIDEO_LOADER_BACKEND", "test_reject_video_backend_kwarg")
263
+
264
+ imageio = ImageMediaIO()
265
+
266
+ # Even when video_backend is provided, it should NOT be passed to loader
267
+ videoio = VideoMediaIO(
268
+ imageio,
269
+ num_frames=10,
270
+ video_backend="test_reject_video_backend_kwarg",
271
+ other_kwarg="should_pass_through",
272
+ )
273
+
274
+ # This should NOT raise AssertionError
275
+ frames, metadata = videoio.load_bytes(b"test")
276
+ np.testing.assert_array_equal(frames, FAKE_OUTPUT_1)
277
+ # Verify other kwargs are still passed through
278
+ assert "other_kwarg" in metadata["received_kwargs"]
279
+
280
+
281
+def test_video_media_io_backend_env_var_fallback(monkeypatch: pytest.MonkeyPatch):
282
+ """
283
+ Test that when video_backend kwarg is None or not provided,
284
+ VideoMediaIO falls back to VLLM_VIDEO_LOADER_BACKEND env var.
285
+ """
286
+ with monkeypatch.context() as m:
287
+ m.setenv("VLLM_VIDEO_LOADER_BACKEND", "test_video_backend_override_2")
288
+
289
+ imageio = ImageMediaIO()
290
+
291
+ # Explicit None should fall back to env var
292
+ videoio_none = VideoMediaIO(imageio, num_frames=10, video_backend=None)
293
+ frames_none, metadata_none = videoio_none.load_bytes(b"test")
294
+ np.testing.assert_array_equal(frames_none, FAKE_OUTPUT_2)
295
+ assert metadata_none["video_backend"] == "test_video_backend_override_2"
296
+
297
+ # Not providing video_backend should also fall back to env var
298
+ videoio_missing = VideoMediaIO(imageio, num_frames=10)
299
+ frames_missing, metadata_missing = videoio_missing.load_bytes(b"test")
300
+ np.testing.assert_array_equal(frames_missing, FAKE_OUTPUT_2)
301
+ assert metadata_missing["video_backend"] == "test_video_backend_override_2"
vllm/multimodal/video.py
+8
-1
@@ -283,8 +283,15 @@ class VideoMediaIO(MediaIO[tuple[npt.NDArray, dict[str, Any]]]):
283
# They can be passed to the underlying
284
# media loaders (e.g. custom implementations)
285
# for flexible control.
286
+
287
+ # Allow per-request override of video backend via kwargs.
288
+ # This enables users to specify a different backend than the
289
+ # global VLLM_VIDEO_LOADER_BACKEND env var, e.g.:
290
+ # --media-io-kwargs '{"video": {"video_backend": "torchcodec"}}'
291
+ video_loader_backend = (
292
+ kwargs.pop("video_backend", None) or envs.VLLM_VIDEO_LOADER_BACKEND
293
+ )
294
self.kwargs = kwargs
287
- video_loader_backend = envs.VLLM_VIDEO_LOADER_BACKEND
295
self.video_loader = VIDEO_LOADER_REGISTRY.load(video_loader_backend)
296
297
def load_bytes(self, data: bytes) -> tuple[npt.NDArray, dict[str, Any]]: