A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 1, 18:45 UTC
[videoloader] implement glm46v video loader (#44417)
Signed-off-by: JaredforReal <w13431838023@gmail.com>
Jared Wen committed
Jun 7, 2026 at 14:27 UTC
6ac69203e8eb8ec29a3cb67240ac7e7327d9e0b0
2 files changed
+259
tests/multimodal/test_video.py
+133
@@ -11,8 +11,11 @@ from vllm.assets.base import get_vllm_public_assets
11
from vllm.multimodal.video import (
12
VIDEO_LOADER_REGISTRY,
13
DynamicVideoBackend,
14
+ GLM46VVideoBackend,
15
Molmo2VideoBackend,
16
VideoLoader,
17
+ VideoSourceMetadata,
18
+ VideoTargetMetadata,
19
get_video_loader_backend_for_processor,
20
)
21
from vllm.transformers_utils.processor import get_video_processor_cls_name_from_config
@@ -76,6 +79,11 @@ def test_video_loader_type_doesnt_exist():
79
DynamicVideoBackend,
80
id="glm4v",
81
),
82
+ pytest.param(
83
+ "zai-org/GLM-4.6V-Flash",
84
+ GLM46VVideoBackend,
85
+ id="glm46v",
86
+ ),
87
],
88
)
89
def test_video_processor_from_model_repo(
@@ -527,6 +535,20 @@ def test_pyav_backend_returns_target_frames_not_keyframes():
535
60,
536
id="pyav_dynamic-exceeds_max_duration",
537
),
538
+ # glm46v dynamic FPS (1800 frames @ 30fps = 60s)
539
+ # 60s falls in (30, 300] → target_fps=1.0, extract_t = 60*1.0*2 = 120
540
+ pytest.param(
541
+ "glm46v",
542
+ {"backend": "opencv"},
543
+ 120,
544
+ id="glm46v-60s",
545
+ ),
546
+ pytest.param(
547
+ "glm46v",
548
+ {"backend": "pyav"},
549
+ 120,
550
+ id="glm46v-pyav-60s",
551
+ ),
552
],
553
)
554
def test_video_loader_frames_sampling(
@@ -548,3 +570,114 @@ def test_video_loader_frames_sampling(
570
assert frames.ndim == 4
571
assert frames.shape[3] == 3 # RGB
572
assert frames.shape[0] == expected_num_frames
573
+
574
+
575
+# ============================================================================
576
+# GLM-4.6V Dynamic FPS Threshold Tests
577
+# ============================================================================
578
+
579
+
580
+@pytest.mark.parametrize(
581
+ "duration, original_fps, total_frames, temporal_patch_size, expected_extract_t",
582
+ [
583
+ # Short video ≤30s → target_fps=3.0
584
+ # extract_t = 10 * 3.0 * 2 = 60
585
+ pytest.param(10, 30, 300, 2, 60, id="short-10s"),
586
+ # Exactly at boundary → target_fps=3.0
587
+ # extract_t = 30 * 3.0 * 2 = 180
588
+ pytest.param(30, 30, 900, 2, 180, id="boundary-30s"),
589
+ # Medium video → target_fps=1.0
590
+ # extract_t = 60 * 1.0 * 2 = 120
591
+ pytest.param(60, 30, 1800, 2, 120, id="medium-60s"),
592
+ # Medium boundary → target_fps=1.0
593
+ # extract_t = 300 * 1.0 * 2 = 600
594
+ pytest.param(300, 30, 9000, 2, 600, id="boundary-300s"),
595
+ # Long video → target_fps=0.5
596
+ # extract_t = 600 * 0.5 * 2 = 600
597
+ pytest.param(600, 30, 18000, 2, 600, id="long-600s"),
598
+ # Very long video, capped by _MAX_FRAME_COUNT_DYNAMIC=640
599
+ # extract_t = min(2400 * 0.5 * 2, 640) = min(2400, 640) = 640
600
+ pytest.param(2400, 30, 72000, 2, 640, id="long-capped-640"),
601
+ # Duration exceeds _MAX_DURATION=2400
602
+ # effective_duration = min(5000, 2400) = 2400, target_fps=0.5
603
+ # extract_t = min(2400 * 0.5 * 2, 640) = 640
604
+ pytest.param(5000, 30, 150000, 2, 640, id="exceeds-max-duration"),
605
+ # temporal_patch_size=4
606
+ # extract_t = 60 * 1.0 * 4 = 240
607
+ pytest.param(60, 30, 1800, 4, 240, id="medium-patch-size-4"),
608
+ # temporal_patch_size=1
609
+ # extract_t = 60 * 1.0 * 1 = 60
610
+ pytest.param(60, 30, 1800, 1, 60, id="medium-patch-size-1"),
611
+ ],
612
+)
613
+def test_glm46v_dynamic_fps_thresholds(
614
+ duration: int,
615
+ original_fps: int,
616
+ total_frames: int,
617
+ temporal_patch_size: int,
618
+ expected_extract_t: int,
619
+):
620
+ """Test GLM-4.6V dynamic FPS threshold selection and frame count."""
621
+ source = VideoSourceMetadata(
622
+ total_frames_num=total_frames,
623
+ original_fps=original_fps,
624
+ duration=duration,
625
+ )
626
+ target = VideoTargetMetadata(num_frames=-1, fps=-1, max_duration=-1)
627
+
628
+ indices = GLM46VVideoBackend.compute_frames_index_to_sample(
629
+ source, target, temporal_patch_size=temporal_patch_size
630
+ )
631
+
632
+ # Frame count should match expected (may be +1 from even padding)
633
+ assert len(indices) in (expected_extract_t, expected_extract_t + 1), (
634
+ f"Expected ~{expected_extract_t} frames, got {len(indices)}"
635
+ )
636
+
637
+ # Frame count must be even
638
+ assert len(indices) % 2 == 0, f"Frame count must be even, got {len(indices)}"
639
+
640
+ # All indices must be valid
641
+ assert all(0 <= idx < total_frames for idx in indices), (
642
+ f"Indices out of range [0, {total_frames})"
643
+ )
644
+
645
+ # Indices must be sorted and deduplicated
646
+ assert indices == sorted(set(indices)), "Indices must be sorted and deduplicated"
647
+
648
+
649
+def test_glm46v_even_frame_count_enforcement():
650
+ """Test that GLM-4.6V always returns an even number of frames."""
651
+ target = VideoTargetMetadata(num_frames=-1, fps=-1, max_duration=-1)
652
+ # 5-second video at 30fps → 150 frames
653
+ # extract_t = 5 * 3.0 * 2 = 30 (even, no padding needed)
654
+ source_even = VideoSourceMetadata(total_frames_num=150, original_fps=30, duration=5)
655
+ indices_even = GLM46VVideoBackend.compute_frames_index_to_sample(
656
+ source_even, target
657
+ )
658
+ assert len(indices_even) % 2 == 0
659
+
660
+ # 3-second video at 30fps → 90 frames
661
+ # extract_t = 3 * 3.0 * 2 = 18 (even, no padding needed)
662
+ source_even2 = VideoSourceMetadata(total_frames_num=90, original_fps=30, duration=3)
663
+ indices_even2 = GLM46VVideoBackend.compute_frames_index_to_sample(
664
+ source_even2, target
665
+ )
666
+ assert len(indices_even2) % 2 == 0
667
+
668
+
669
+def test_glm46v_duration_estimation_from_fps():
670
+ """Test GLM-4.6V handles missing duration by estimating from fps."""
671
+ target = VideoTargetMetadata(num_frames=-1, fps=-1, max_duration=-1)
672
+ # duration=0 → estimated from total_frames / fps
673
+ # (89 / 30) + 1 ≈ 4s → target_fps=3.0, extract_t = 4 * 3.0 * 2 = 24
674
+ source_no_duration = VideoSourceMetadata(
675
+ total_frames_num=90, original_fps=30, duration=0
676
+ )
677
+ indices = GLM46VVideoBackend.compute_frames_index_to_sample(
678
+ source_no_duration, target
679
+ )
680
+
681
+ assert len(indices) > 0
682
+ assert len(indices) % 2 == 0
683
+ assert all(0 <= idx < 90 for idx in indices)
vllm/multimodal/video.py
+126
@@ -696,6 +696,132 @@ class DynamicVideoBackend(VideoBackend):
696
)
697
698
699
+@VIDEO_LOADER_REGISTRY.register(
700
+ "glm46v",
701
+ video_processor="Glm46VVideoProcessor",
702
+)
703
+class GLM46VVideoBackend(VideoBackend):
704
+ """GLM-4.6V dynamic FPS video backend.
705
+
706
+ Faithfully replicates the frame sampling logic from transformers'
707
+ ``Glm46VVideoProcessor.sample_frames``:
708
+
709
+ - Dynamic FPS thresholds based on effective video duration:
710
+ ``{≤30s: 3fps, ≤300s: 1fps, >300s: 0.5fps}``
711
+ - ``temporal_patch_size`` multiplier (default 2) applied to extract count
712
+ - Duration capped at 2400s, frame count capped at 640
713
+ - Even frame count enforced (append last frame if odd)
714
+ """
715
+
716
+ # Match transformers defaults
717
+ _DYNAMIC_FPS_THRESHOLDS: ClassVar[dict[int, float]] = {
718
+ 30: 3.0,
719
+ 300: 1.0,
720
+ 2400: 0.5,
721
+ }
722
+ _MAX_FRAME_COUNT_DYNAMIC: ClassVar[int] = 640
723
+ _MAX_DURATION: ClassVar[int] = 2400
724
+
725
+ @classmethod
726
+ def compute_frames_index_to_sample(
727
+ cls,
728
+ source: VideoSourceMetadata,
729
+ target: VideoTargetMetadata,
730
+ **kwargs,
731
+ ) -> list[int]:
732
+ # Refer to:
733
+ # https://github.com/huggingface/transformers/blob/v5.9.0/src/transformers/models/glm46v/video_processing_glm46v.py#L97-L102
734
+ total_frames_num = source.total_frames_num
735
+ original_fps = source.original_fps
736
+ duration = source.duration
737
+ temporal_patch_size = kwargs.get("temporal_patch_size", 2)
738
+
739
+ max_frame_idx = total_frames_num - 1
740
+
741
+ # Estimate duration from frame count and fps when not reported
742
+ if not duration and original_fps > 0:
743
+ duration = round(max_frame_idx / original_fps) + 1
744
+
745
+ effective_duration = min(duration, cls._MAX_DURATION)
746
+
747
+ # Select target_fps from dynamic thresholds
748
+ if effective_duration <= 30:
749
+ target_fps = cls._DYNAMIC_FPS_THRESHOLDS[30]
750
+ elif effective_duration <= 300:
751
+ target_fps = cls._DYNAMIC_FPS_THRESHOLDS[300]
752
+ else:
753
+ target_fps = cls._DYNAMIC_FPS_THRESHOLDS[2400]
754
+
755
+ extract_t = int(effective_duration * target_fps * temporal_patch_size)
756
+ extract_t = min(extract_t, cls._MAX_FRAME_COUNT_DYNAMIC)
757
+
758
+ duration_per_frame = 1 / original_fps if original_fps > 0 else 0
759
+ timestamps = [i * duration_per_frame for i in range(total_frames_num)]
760
+ max_second = int(duration) if duration else 0
761
+
762
+ if total_frames_num < extract_t:
763
+ frame_indices = np.linspace(
764
+ 0, total_frames_num - 1, extract_t, dtype=int
765
+ ).tolist()
766
+ else:
767
+ frame_indices = []
768
+ current_second = 0.0
769
+ inv_fps = 1 / (temporal_patch_size * target_fps)
770
+ for frame_index in range(total_frames_num):
771
+ if timestamps[frame_index] >= current_second:
772
+ current_second += inv_fps
773
+ frame_indices.append(frame_index)
774
+ if current_second >= max_second:
775
+ break
776
+
777
+ if len(frame_indices) < extract_t:
778
+ if len(frame_indices) == 0:
779
+ start, end = 0, max(total_frames_num - 1, 0)
780
+ else:
781
+ start, end = frame_indices[0], frame_indices[-1]
782
+ frame_indices = np.linspace(start, end, extract_t, dtype=int).tolist()
783
+ elif len(frame_indices) > extract_t:
784
+ frame_indices = np.linspace(
785
+ 0, total_frames_num - 1, extract_t, dtype=int
786
+ ).tolist()
787
+
788
+ # Deduplicate
789
+ seen: set[int] = set()
790
+ uniq: list[int] = []
791
+ for idx in frame_indices:
792
+ if idx not in seen:
793
+ seen.add(idx)
794
+ uniq.append(idx)
795
+
796
+ # Ensure even frame count
797
+ if len(uniq) & 1:
798
+ uniq.append(uniq[-1])
799
+
800
+ return uniq
801
+
802
+ @classmethod
803
+ def load_bytes(
804
+ cls,
805
+ data: bytes,
806
+ num_frames: int = -1,
807
+ fps: int = -1,
808
+ max_duration: int = 300,
809
+ frame_recovery: bool = False,
810
+ *,
811
+ backend: Literal["opencv", "pyav"] = "opencv",
812
+ **kwargs,
813
+ ) -> tuple[npt.NDArray, dict[str, Any]]:
814
+ return super().load_bytes(
815
+ data,
816
+ num_frames=num_frames,
817
+ fps=fps,
818
+ max_duration=max_duration,
819
+ frame_recovery=frame_recovery,
820
+ backend=backend,
821
+ **kwargs,
822
+ )
823
+
824
+
825
@VIDEO_LOADER_REGISTRY.register(
826
"glmga",
827
video_processor="GlmgaVideoProcessor",