[Bugfix][Multimodal] Invalidate retained PyNvVideoCodec decoder after failure (#51139)
Signed-off-by: Dat Mai <dmai@nvidia.com>
dmai-afk committed
Aug 12, 2026 at 18:02 UTC
4c51ceb11bbc2177a3639dfae2592cd34b83a7ef
3 files changed
+134
-1
tests/multimodal/assets/unsupported_8k_h264.mp4
Binary files /dev/null and b/tests/multimodal/assets/unsupported_8k_h264.mp4 differ
tests/multimodal/test_video.py
+123
@@ -328,6 +328,129 @@ def test_pynvvideocodec_decoder_slots_are_configured_once(
328
PyNvVideoCodecVideoBackend._configure_decoder_slots(3)
329
330
331
+def test_pynvvideocodec_failed_rebuild_invalidates_decoder_slot():
332
+ events: list[tuple[str, str]] = []
333
+
334
+ class FakeStream:
335
+ cuda_stream = "cuda-stream"
336
+
337
+ class FakeDecoder:
338
+ poisoned = False
339
+
340
+ def reconfigure_decoder(self, file_path: str):
341
+ self.poisoned = True
342
+ events.append(("reconfigure", file_path))
343
+ raise RuntimeError("reconfigure failed")
344
+
345
+ old_decoder = FakeDecoder()
346
+ slot = PyNvVideoCodecDecoderSlot(FakeStream())
347
+ slot.decoder = old_decoder
348
+ slot.source_path = "valid.mp4"
349
+
350
+ class FakeNvc:
351
+ class OutputColorType:
352
+ RGB = "rgb"
353
+
354
+ @staticmethod
355
+ def SimpleDecoder(file_path: str, **kwargs):
356
+ events.append(("construct", file_path))
357
+ assert slot.decoder is None
358
+ assert slot.source_path is None
359
+ raise RuntimeError("construct failed")
360
+
361
+ old_slots = PyNvVideoCodecVideoBackend._decoder_slots
362
+ old_active_slots = PyNvVideoCodecVideoBackend._active_decoder_slots
363
+ old_cond = PyNvVideoCodecVideoBackend._decoder_slot_cond
364
+ old_max_slots = PyNvVideoCodecVideoBackend._max_decoder_slots
365
+ try:
366
+ PyNvVideoCodecVideoBackend._decoder_slots = [slot]
367
+ PyNvVideoCodecVideoBackend._active_decoder_slots = 1
368
+ PyNvVideoCodecVideoBackend._decoder_slot_cond = threading.Condition()
369
+ PyNvVideoCodecVideoBackend._max_decoder_slots = 1
370
+
371
+ with (
372
+ pytest.raises(RuntimeError, match="construct failed"),
373
+ PyNvVideoCodecVideoBackend._borrow_decoder_slot() as borrowed,
374
+ ):
375
+ assert borrowed is slot
376
+ borrowed.get_decoder(
377
+ "unsupported-8k.mp4",
378
+ FakeNvc,
379
+ device_index=0,
380
+ )
381
+
382
+ assert events == [
383
+ ("reconfigure", "unsupported-8k.mp4"),
384
+ ("construct", "unsupported-8k.mp4"),
385
+ ]
386
+ assert old_decoder.poisoned
387
+ assert slot.decoder is None
388
+ assert slot.source_path is None
389
+ assert PyNvVideoCodecVideoBackend._decoder_slots == [slot]
390
+ finally:
391
+ PyNvVideoCodecVideoBackend._decoder_slots = old_slots
392
+ PyNvVideoCodecVideoBackend._active_decoder_slots = old_active_slots
393
+ PyNvVideoCodecVideoBackend._decoder_slot_cond = old_cond
394
+ PyNvVideoCodecVideoBackend._max_decoder_slots = old_max_slots
395
+
396
+
397
+@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA")
398
+def test_pynvvideocodec_h200_recovers_after_unsupported_8k():
399
+ import PyNvVideoCodec as nvc
400
+ import torch
401
+
402
+ if "H200" not in torch.cuda.get_device_name(0):
403
+ pytest.skip("Requires H200 NVDEC resolution limits")
404
+
405
+ valid_video = create_long_gop_video(num_frames=2, width=64, height=64)
406
+ unsupported_video = (ASSETS_DIR / "unsupported_8k_h264.mp4").read_bytes()
407
+
408
+ old_slots = PyNvVideoCodecVideoBackend._decoder_slots
409
+ old_active_slots = PyNvVideoCodecVideoBackend._active_decoder_slots
410
+ old_cond = PyNvVideoCodecVideoBackend._decoder_slot_cond
411
+ old_max_slots = PyNvVideoCodecVideoBackend._max_decoder_slots
412
+ try:
413
+ PyNvVideoCodecVideoBackend._decoder_slots = []
414
+ PyNvVideoCodecVideoBackend._active_decoder_slots = 0
415
+ PyNvVideoCodecVideoBackend._decoder_slot_cond = threading.Condition()
416
+ PyNvVideoCodecVideoBackend._max_decoder_slots = None
417
+
418
+ loader = VIDEO_LOADER_REGISTRY.load(PYNVVIDEOCODEC_VIDEO_BACKEND)
419
+ frames_before, _ = loader.load_bytes(
420
+ valid_video,
421
+ num_frames=1,
422
+ hw_decoders=1,
423
+ )
424
+
425
+ with pytest.raises(Exception) as exc_info:
426
+ loader.load_bytes(
427
+ unsupported_video,
428
+ num_frames=1,
429
+ hw_decoders=1,
430
+ )
431
+
432
+ root_cause = exc_info.value
433
+ while root_cause.__cause__ is not None:
434
+ root_cause = root_cause.__cause__
435
+ assert isinstance(root_cause, nvc.PyNvVCExceptionUnsupported)
436
+ assert "MBCount not supported" in str(root_cause)
437
+
438
+ frames_after, _ = loader.load_bytes(
439
+ valid_video,
440
+ num_frames=1,
441
+ hw_decoders=1,
442
+ )
443
+
444
+ assert frames_after.shape == frames_before.shape
445
+ finally:
446
+ for slot in PyNvVideoCodecVideoBackend._decoder_slots:
447
+ slot.invalidate()
448
+ PyNvVideoCodecVideoBackend._decoder_slots = old_slots
449
+ PyNvVideoCodecVideoBackend._active_decoder_slots = old_active_slots
450
+ PyNvVideoCodecVideoBackend._decoder_slot_cond = old_cond
451
+ PyNvVideoCodecVideoBackend._max_decoder_slots = old_max_slots
452
+
453
+
454
@pytest.mark.parametrize("hw_decoders", [0, -1, 1.5, True, "2"])
455
def test_pynvvideocodec_rejects_invalid_hw_decoders(hw_decoders: object):
456
with pytest.raises(ValueError, match="hw_decoders must be a positive integer"):
vllm/multimodal/video.py
+11
-1
@@ -261,8 +261,13 @@ class PyNvVideoCodecDecoderSlot:
261
self.decoder = None
262
self.source_path: str | None = None
263
264
+ def invalidate(self) -> None:
265
+ self.decoder = None
266
+ self.source_path = None
267
+
268
def _construct(self, file_path: str, nvc, device_index: int) -> None:
265
- self.decoder = nvc.SimpleDecoder(
269
+ self.invalidate()
270
+ decoder = nvc.SimpleDecoder(
271
file_path,
272
output_color_type=nvc.OutputColorType.RGB,
273
use_device_memory=True,
@@ -271,6 +276,7 @@ class PyNvVideoCodecDecoderSlot:
276
cuda_stream=self.stream.cuda_stream,
277
decoder_cache_size=PYNVVIDEOCODEC_DECODER_CACHE_SIZE,
278
)
279
+ self.decoder = decoder
280
self.source_path = file_path
281
282
def get_decoder(self, file_path: str, nvc, device_index: int):
@@ -747,9 +753,13 @@ class PyNvVideoCodecVideoBackendMixin:
753
cls._decoder_slot_cond.notify()
754
raise
755
756
+ borrow_succeeded = False
757
try:
758
yield slot
759
+ borrow_succeeded = True
760
finally:
761
+ if not borrow_succeeded:
762
+ slot.invalidate()
763
with cls._decoder_slot_cond:
764
cls._decoder_slots.append(slot)
765
cls._decoder_slot_cond.notify()