A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 20:00 UTC
[Multimodal] Support custom video metadata for pre-extracted frame sequences (#40133)
Signed-off-by: storyicon <storyicon@foxmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
storyicon committed
Apr 22, 2026 at 15:50 UTC
6aa057c9d7de0b2535cd9d18369b60f9a506a55f
2 files changed
+102
-4
docs/features/multimodal_inputs.md
+64
@@ -780,6 +780,70 @@ vllm serve Qwen/Qwen3-VL-30B-A3B-Instruct \
780
781
Works with common video formats like MP4 when using OpenCV backends.
782
783
+#### Pre-extracted Frame Sequences with `media_io_kwargs`
784
+
785
+When you extract video frames on the client side and send them as `video/jpeg` (base64-concatenated JPEG frames), you can preserve the original video metadata by using `media_io_kwargs` in your request. This enables more accurate video understanding by preserving temporal information that would otherwise be lost during client-side frame extraction.
786
+
787
+**Supported Parameters:**
788
+
789
+| Parameter | Type | Description |
790
+| --------- | ---- | ----------- |
791
+| `fps` | float | Frame rate of the original video |
792
+| `frames_indices` | list[int] | Indices of the actually sampled frames |
793
+| `total_num_frames` | int | Total frame count of the original video |
794
+| `duration` | float | Duration of the original video in seconds |
795
+| `do_sample_frames` | bool | Whether to perform frame sampling |
796
+
797
+??? code
798
+
799
+ ```python
800
+ from openai import OpenAI
801
+
802
+ client = OpenAI(base_url="http://localhost:8000/v1", api_key="EMPTY")
803
+
804
+ # Client-side frame extraction
805
+ frames = extract_frames(video_path, num_frames=32)
806
+ frames_b64 = ",".join([encode_image(f) for f in frames])
807
+ video_url = f"data:video/jpeg;base64,{frames_b64}"
808
+
809
+ # Pass video metadata via media_io_kwargs
810
+ response = client.chat.completions.create(
811
+ model="your-multimodal-model",
812
+ messages=[{
813
+ "role": "user",
814
+ "content": [
815
+ {"type": "video_url", "video_url": {"url": video_url}},
816
+ {"type": "text", "text": "Describe what happens in this video."}
817
+ ]
818
+ }],
819
+ extra_body={
820
+ "media_io_kwargs": {
821
+ "video": {
822
+ "fps": 30.0,
823
+ "frames_indices": [0, 10, 20, 30, 40, 50, 60, 70, 80, 90,
824
+ 100, 110, 120, 130, 140, 150, 160, 170,
825
+ 180, 190, 200, 210, 220, 230, 240, 250,
826
+ 260, 270, 280, 290, 300, 310],
827
+ "total_num_frames": 900,
828
+ "duration": 30.0,
829
+ }
830
+ }
831
+ },
832
+ )
833
+
834
+ print(response.choices[0].message.content)
835
+ ```
836
+
837
+**Why use `media_io_kwargs`?**
838
+
839
+When extracting frames client-side, the server loses important context about the original video:
840
+
841
+- **Temporal information**: Which frames were sampled and their positions in the original timeline
842
+- **Video duration**: How long the original video was
843
+- **Frame rate**: The original playback speed
844
+
845
+By passing this metadata, the model can better understand the temporal distribution of the sampled frames and whether important moments might have been skipped.
846
+
847
#### Custom RGBA Background Color
848
849
To use a custom background color for RGBA images, pass the `rgba_background_color` parameter via `--media-io-kwargs`:
vllm/multimodal/media/video.py
+38
-4
@@ -92,14 +92,48 @@ class VideoMediaIO(MediaIO[tuple[npt.NDArray, dict[str, Any]]]):
92
)
93
total = int(frames.shape[0])
94
fps = float(self.kwargs.get("fps", 1))
95
- duration = total / fps if fps > 0 else 0.0
95
+
96
+ # validate and extract frames_indices
97
+ frames_indices = self.kwargs.get("frames_indices")
98
+ if frames_indices is not None:
99
+ if not (
100
+ isinstance(frames_indices, list)
101
+ and all(isinstance(i, int) for i in frames_indices)
102
+ ):
103
+ raise ValueError("frames_indices must be a list of integers")
104
+ if len(frames_indices) != total:
105
+ raise ValueError(
106
+ f"frames_indices length ({len(frames_indices)}) must "
107
+ f"match number of frames sent ({total})"
108
+ )
109
+ else:
110
+ frames_indices = list(range(total))
111
+
112
+ # validate and extract total_num_frames
113
+ total_num_frames = self.kwargs.get("total_num_frames", total)
114
+ if not isinstance(total_num_frames, int) or total_num_frames < 1:
115
+ raise ValueError("total_num_frames must be a positive integer")
116
+ if total_num_frames < total:
117
+ raise ValueError(
118
+ f"total_num_frames ({total_num_frames}) must be >= "
119
+ f"number of frames sent ({total})"
120
+ )
121
+
122
+ # validate and extract duration
123
+ duration = self.kwargs.get("duration")
124
+ if duration is not None:
125
+ if not isinstance(duration, (int, float)) or duration < 0:
126
+ raise ValueError("duration must be a non-negative number")
127
+ else:
128
+ duration = total_num_frames / fps if fps > 0 else 0.0
129
+
130
metadata = {
97
- "total_num_frames": total,
131
+ "total_num_frames": total_num_frames,
132
"fps": fps,
133
"duration": duration,
134
"video_backend": "jpeg_sequence",
101
- "frames_indices": list(range(total)),
102
- "do_sample_frames": False,
135
+ "frames_indices": frames_indices,
136
+ "do_sample_frames": self.kwargs.get("do_sample_frames", False),
137
}
138
return frames, metadata
139