| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Unit tests for the LLaVA-OneVision-2 codec-video marker mechanism. |
| 4 | |
| 5 | The codec video backend cannot be exercised end-to-end in CI because it |
| 6 | relies on the model's ``trust_remote_code`` package and real video bytes. |
| 7 | These tests cover the pure-Python marker plumbing that wraps a video *path* |
| 8 | for vLLM's ``MultiModalDataParser``: |
| 9 | |
| 10 | * :func:`prepare_codec_video_input` must encode a per-path hash into its |
| 11 | dummy ndarray so distinct codec videos do not collide on ``mm_hash`` |
| 12 | (otherwise ``EncoderCacheManager`` would skip the encoder for every video |
| 13 | after the first). |
| 14 | * :func:`_extract_codec_video_paths` must round-trip the marker back to the |
| 15 | original path(s) after the parser strips the metadata dict, and must return |
| 16 | ``None`` for non-codec inputs. |
| 17 | """ |
| 18 | |
| 19 | import types |
| 20 | |
| 21 | import numpy as np |
| 22 | import pytest |
| 23 | |
| 24 | from vllm.model_executor.models.llava_onevision2 import ( |
| 25 | _CODEC_VIDEO_MARKER, |
| 26 | LlavaOnevision2VideoBackend, |
| 27 | _extract_codec_video_paths, |
| 28 | _frame_video_to_pil_and_timestamps, |
| 29 | _validate_video_source, |
| 30 | prepare_codec_video_input, |
| 31 | ) |
| 32 | from vllm.multimodal.video import VideoSourceMetadata, VideoTargetMetadata |
| 33 | |
| 34 | |
| 35 | def _model_config(local: str = "", domains=None): |
| 36 | # Minimal stand-in for vLLM's ModelConfig: the validator only reads |
| 37 | # ``allowed_local_media_path`` and ``allowed_media_domains``. |
| 38 | return types.SimpleNamespace( |
| 39 | allowed_local_media_path=local, |
| 40 | allowed_media_domains=domains, |
| 41 | ) |
| 42 | |
| 43 | |
| 44 | def test_prepare_codec_video_input_shape_and_marker(): |
| 45 | dummy, meta = prepare_codec_video_input("/data/foo.mp4") |
| 46 | |
| 47 | # 4-D ndarray satisfies MultiModalDataParser's video shape check. |
| 48 | assert isinstance(dummy, np.ndarray) |
| 49 | assert dummy.shape == (1, 1, 16, 3) |
| 50 | assert dummy.dtype == np.uint8 |
| 51 | |
| 52 | # Metadata carries the exact path under the codec marker key. |
| 53 | assert meta == {_CODEC_VIDEO_MARKER: "/data/foo.mp4"} |
| 54 | |
| 55 | |
| 56 | def test_prepare_codec_video_input_is_deterministic(): |
| 57 | # Same path must yield identical dummy bytes (stable mm_hash). |
| 58 | a, _ = prepare_codec_video_input("/data/foo.mp4") |
| 59 | b, _ = prepare_codec_video_input("/data/foo.mp4") |
| 60 | assert a.tobytes() == b.tobytes() |
| 61 | |
| 62 | |
| 63 | def test_prepare_codec_video_input_distinct_paths_distinct_bytes(): |
| 64 | # Distinct paths must yield distinct dummy bytes so the parser-visible |
| 65 | # ndarray (the only part reaching MultiModalHasher) varies per video. |
| 66 | paths = ["/data/a.mp4", "/data/b.mp4", "/data/c.mp4", "/data/a_.mp4"] |
| 67 | payloads = {prepare_codec_video_input(p)[0].tobytes() for p in paths} |
| 68 | assert len(payloads) == len(paths) |
| 69 | |
| 70 | |
| 71 | def test_extract_codec_video_paths_parser_list_shape(): |
| 72 | # Parser yields list-of-(ndarray, metadata-dict) for tuple inputs. |
| 73 | items = [prepare_codec_video_input(p) for p in ("/x/1.mp4", "/x/2.mp4")] |
| 74 | assert _extract_codec_video_paths(items) == ["/x/1.mp4", "/x/2.mp4"] |
| 75 | |
| 76 | |
| 77 | def test_extract_codec_video_paths_single_raw_tuple(): |
| 78 | # Single raw (ndarray, dict) tuple (pre-parser path) is also accepted. |
| 79 | item = prepare_codec_video_input("/x/solo.mp4") |
| 80 | assert _extract_codec_video_paths(item) == ["/x/solo.mp4"] |
| 81 | |
| 82 | |
| 83 | def test_extract_codec_video_paths_non_codec_returns_none(): |
| 84 | # Plain decoded-frame inputs (ndarray / list of ndarray) are not codec |
| 85 | # markers and must fall through to the frame backend. |
| 86 | plain = np.zeros((4, 8, 8, 3), dtype=np.uint8) |
| 87 | assert _extract_codec_video_paths(plain) is None |
| 88 | assert _extract_codec_video_paths([plain, plain]) is None |
| 89 | assert _extract_codec_video_paths([]) is None |
| 90 | # Tuple without the marker key is ignored. |
| 91 | assert _extract_codec_video_paths((plain, {"fps": 2.0})) is None |
| 92 | |
| 93 | |
| 94 | def test_extract_codec_video_paths_mixed_batch_returns_none(): |
| 95 | # If any item in the batch lacks the marker, the whole batch is treated |
| 96 | # as non-codec (the backend does not mix codec and frame videos). |
| 97 | codec = prepare_codec_video_input("/x/1.mp4") |
| 98 | plain = np.zeros((4, 8, 8, 3), dtype=np.uint8) |
| 99 | assert _extract_codec_video_paths([codec, plain]) is None |
| 100 | |
| 101 | |
| 102 | # --------------------------------------------------------------------------- |
| 103 | # Media access controls (_validate_video_source) |
| 104 | # |
| 105 | # The codec backend keeps the raw path string alive past vLLM's |
| 106 | # MultiModalDataParser and hands it to the trust-remote-code codec module, |
| 107 | # which opens it directly (cv2/ffmpeg) outside vLLM's MediaConnector. So the |
| 108 | # codec backend is restricted to *local files* confined to |
| 109 | # --allowed-local-media-path; remote http(s)/data URLs are rejected (they must |
| 110 | # use the frame backend, which rides the connector). The validator also returns |
| 111 | # the *resolved* path so the codec module opens exactly what was validated |
| 112 | # (no symlink-retarget / validate-vs-open gap). |
| 113 | # --------------------------------------------------------------------------- |
| 114 | |
| 115 | |
| 116 | def test_validate_http_url_rejected(): |
| 117 | # Codec backend is local-only: remote URLs bypass the connector's domain |
| 118 | # and redirect controls, so they are rejected regardless of allowlist. |
| 119 | with pytest.raises(ValueError): |
| 120 | _validate_video_source("http://example.com/v.mp4", _model_config()) |
| 121 | |
| 122 | |
| 123 | def test_validate_https_url_rejected_even_when_host_allowlisted(): |
| 124 | with pytest.raises(ValueError): |
| 125 | _validate_video_source( |
| 126 | "https://good.com/v.mp4", |
| 127 | _model_config(domains=["good.com"]), |
| 128 | ) |
| 129 | |
| 130 | |
| 131 | def test_validate_data_url_rejected(): |
| 132 | # data: URLs are a remote/inline source for the frame backend, not codec. |
| 133 | with pytest.raises(ValueError): |
| 134 | _validate_video_source("data:video/mp4;base64,AAAA", _model_config()) |
| 135 | |
| 136 | |
| 137 | def test_validate_local_file_blocked_without_allowed_path(): |
| 138 | # Local file access is opt-in: without --allowed-local-media-path the |
| 139 | # bare path is rejected. |
| 140 | with pytest.raises(ValueError): |
| 141 | _validate_video_source("/data/v.mp4", _model_config()) |
| 142 | |
| 143 | |
| 144 | def test_validate_local_file_allowed_inside_allowed_dir(tmp_path): |
| 145 | f = tmp_path / "v.mp4" |
| 146 | f.touch() |
| 147 | assert _validate_video_source(str(f), _model_config(local=str(tmp_path))) == str(f) |
| 148 | |
| 149 | |
| 150 | def test_validate_local_file_traversal_blocked(): |
| 151 | # Path traversal escaping the allowed root is rejected after resolution. |
| 152 | with pytest.raises(ValueError): |
| 153 | _validate_video_source( |
| 154 | "/tmp/ov2/../../etc/passwd", |
| 155 | _model_config(local="/tmp/ov2"), |
| 156 | ) |
| 157 | |
| 158 | |
| 159 | def test_validate_file_scheme_allowed_inside_allowed_dir(tmp_path): |
| 160 | f = tmp_path / "v.mp4" |
| 161 | f.touch() |
| 162 | assert _validate_video_source( |
| 163 | f.as_uri(), _model_config(local=str(tmp_path)) |
| 164 | ) == str(f) |
| 165 | |
| 166 | |
| 167 | def test_validate_file_scheme_percent_encoded_traversal_blocked(): |
| 168 | # ``%2e%2e`` decodes to ``..``; the confinement check URL-decodes first |
| 169 | # (url2pathname) so the path cannot stay literally under the allowed root. |
| 170 | with pytest.raises(ValueError): |
| 171 | _validate_video_source( |
| 172 | "file:///tmp/ov2/%2e%2e/%2e%2e/etc/passwd", |
| 173 | _model_config(local="/tmp/ov2"), |
| 174 | ) |
| 175 | |
| 176 | |
| 177 | def test_validate_unsupported_scheme_blocked(): |
| 178 | with pytest.raises(ValueError): |
| 179 | _validate_video_source("ftp://example.com/v.mp4", _model_config()) |
| 180 | |
| 181 | |
| 182 | def test_validate_returns_resolved_path_through_symlink(tmp_path): |
| 183 | # The validator resolves symlinks and returns the *real* path, so the codec |
| 184 | # module opens exactly what was validated (closes the validate-vs-open gap). |
| 185 | real = tmp_path / "real.mp4" |
| 186 | real.touch() |
| 187 | link = tmp_path / "link.mp4" |
| 188 | link.symlink_to(real) |
| 189 | assert _validate_video_source(str(link), _model_config(local=str(tmp_path))) == str( |
| 190 | real |
| 191 | ) |
| 192 | |
| 193 | |
| 194 | def test_validate_relative_bare_path_blocked(): |
| 195 | # Bare (scheme=="") paths come only from the codec backend and must be |
| 196 | # absolute: resolving a relative path against an ambiguous CWD before the |
| 197 | # confinement check is brittle/unsafe, so it is rejected outright. |
| 198 | with pytest.raises(ValueError): |
| 199 | _validate_video_source("ov2/v.mp4", _model_config(local="/tmp/ov2")) |
| 200 | |
| 201 | |
| 202 | # --------------------------------------------------------------------------- |
| 203 | # Frame backend marker -> PIL + timestamps (_frame_video_to_pil_and_timestamps) |
| 204 | # |
| 205 | # Non-codec videos reach _apply_hf_processor_main as a ``(frames_ndarray, metadata)`` |
| 206 | # tuple -- produced by the registered ``LlavaOnevision2VideoBackend`` for real |
| 207 | # ``video_url`` inputs, or by the dummy-inputs builder during profiling |
| 208 | # (``video_needs_metadata=True``). The helper materialises PIL frames and |
| 209 | # per-frame timestamps (``frame_index / fps``), padding the frame count up to |
| 210 | # the even temporal-merge boundary. |
| 211 | # --------------------------------------------------------------------------- |
| 212 | |
| 213 | |
| 214 | def test_frame_video_to_pil_and_timestamps_basic(): |
| 215 | frames = np.zeros((4, 8, 8, 3), dtype=np.uint8) |
| 216 | metadata = {"fps": 2.0, "frames_indices": [0, 4, 8, 12]} |
| 217 | pil_frames, timestamps = _frame_video_to_pil_and_timestamps((frames, metadata)) |
| 218 | |
| 219 | assert len(pil_frames) == 4 |
| 220 | assert all(f.size == (8, 8) for f in pil_frames) |
| 221 | # timestamps = frame_index / fps |
| 222 | assert timestamps == [0.0, 2.0, 4.0, 6.0] |
| 223 | |
| 224 | |
| 225 | def test_frame_video_to_pil_and_timestamps_even_pads_odd_frame_count(): |
| 226 | # Odd frame count -> last frame repeated to satisfy temporal merge=2. |
| 227 | frames = np.zeros((3, 8, 8, 3), dtype=np.uint8) |
| 228 | metadata = {"fps": 1.0, "frames_indices": [0, 1, 2]} |
| 229 | pil_frames, timestamps = _frame_video_to_pil_and_timestamps((frames, metadata)) |
| 230 | |
| 231 | assert len(pil_frames) == 4 |
| 232 | assert len(timestamps) == 4 |
| 233 | # The padded frame reuses the final index/timestamp. |
| 234 | assert timestamps == [0.0, 1.0, 2.0, 2.0] |
| 235 | |
| 236 | |
| 237 | def test_frame_video_to_pil_and_timestamps_defaults_when_metadata_sparse(): |
| 238 | # Missing frames_indices -> sequential range; missing/zero fps -> default. |
| 239 | frames = np.zeros((2, 8, 8, 3), dtype=np.uint8) |
| 240 | pil_frames, timestamps = _frame_video_to_pil_and_timestamps((frames, {})) |
| 241 | |
| 242 | assert len(pil_frames) == 2 |
| 243 | # Default fps is 1.0, indices fall back to range(T). |
| 244 | assert timestamps == [0.0, 1.0] |
| 245 | |
| 246 | |
| 247 | def test_frame_video_to_pil_and_timestamps_rejects_non_tuple(): |
| 248 | # Bare arrays (no metadata) must be rejected: the frame backend requires |
| 249 | # the (frames, metadata) tuple produced by the registered loader. |
| 250 | plain = np.zeros((4, 8, 8, 3), dtype=np.uint8) |
| 251 | with pytest.raises(ValueError): |
| 252 | _frame_video_to_pil_and_timestamps(plain) |
| 253 | |
| 254 | |
| 255 | # --------------------------------------------------------------------------- |
| 256 | # LlavaOnevision2VideoBackend.compute_frames_index_to_sample honors the caller |
| 257 | # supplied VideoTargetMetadata (passed via --media-io-kwargs) so benchmarks can |
| 258 | # override the conservative defaults (fps=1.0, max_frames=32). Unset target |
| 259 | # fields (sentinel <= 0) fall back to those OV2 hf-chat reference constants. |
| 260 | # --------------------------------------------------------------------------- |
| 261 | |
| 262 | |
| 263 | def _src(total_frames: int, fps: float) -> VideoSourceMetadata: |
| 264 | return VideoSourceMetadata( |
| 265 | total_frames_num=total_frames, |
| 266 | original_fps=fps, |
| 267 | duration=total_frames / fps if fps > 0 else 0.0, |
| 268 | ) |
| 269 | |
| 270 | |
| 271 | def test_backend_defaults_cap_at_32_frames(): |
| 272 | # 300 frames @ 1fps source, target unset -> capped at default max_frames=32. |
| 273 | src = _src(300, 1.0) |
| 274 | target = VideoTargetMetadata(num_frames=-1, fps=-1, max_duration=300.0) |
| 275 | idx = LlavaOnevision2VideoBackend.compute_frames_index_to_sample(src, target) |
| 276 | |
| 277 | assert len(idx) == 32 |
| 278 | assert idx[0] == 0 |
| 279 | assert idx[-1] == 299 |
| 280 | assert len(idx) % 2 == 0 |
| 281 | |
| 282 | |
| 283 | def test_backend_target_num_frames_overrides_default_cap(): |
| 284 | # VSI-Bench parity: target.num_frames=128 must lift the 32-frame cap. |
| 285 | src = _src(300, 1.0) |
| 286 | target = VideoTargetMetadata(num_frames=128, fps=-1, max_duration=300.0) |
| 287 | idx = LlavaOnevision2VideoBackend.compute_frames_index_to_sample(src, target) |
| 288 | |
| 289 | assert len(idx) == 128 |
| 290 | assert idx[0] == 0 |
| 291 | assert idx[-1] == 299 |
| 292 | |
| 293 | |
| 294 | def test_backend_target_fps_controls_sampling_when_below_cap(): |
| 295 | # 60 frames @ 30fps (2s) with target fps=1 -> ~2 frames (even-padded). |
| 296 | src = _src(60, 30.0) |
| 297 | target = VideoTargetMetadata(num_frames=128, fps=1.0, max_duration=300.0) |
| 298 | idx = LlavaOnevision2VideoBackend.compute_frames_index_to_sample(src, target) |
| 299 | |
| 300 | # fps-derived nframes (2) is below the 128 cap, so fps wins. |
| 301 | assert len(idx) <= 8 |
| 302 | assert len(idx) % 2 == 0 |