@tduncan / Nvidia-Optimization-Stack-VLLM / commits / b0b8a286dd

[Model] Add LLaVA-OneVision-2 (LlavaOnevision2ForConditionalGeneration) (#44785)

Signed-off-by: chengzheng345 <209475443+chengzheng345@users.noreply.github.com> Co-authored-by: chengzheng345 <209475443+chengzheng345@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

chengzheng345 committed Jul 2, 2026 at 16:41 UTC b0b8a286ddf6a2914d89ab2ff39507d8d4a71b65
6 files changed +2593
docs/models/supported_models.md
+1
@@ -581,6 +581,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
581 | `LlavaForConditionalGeneration` | LLaVA-1.5, Pixtral (HF Transformers) | T + I<sup>E+</sup> | `llava-hf/llava-1.5-7b-hf`, `mistral-community/pixtral-12b`, etc. | ✅︎ | ✅︎ |
582 | `LlavaNextForConditionalGeneration` | LLaVA-NeXT, Granite Vision | T + I<sup>E+</sup> | `llava-hf/llava-v1.6-mistral-7b-hf`, `llava-hf/llava-v1.6-vicuna-7b-hf`, `ibm-granite/granite-vision-3.3-2b`, etc. | | ✅︎ |
583 | `LlavaNextVideoForConditionalGeneration` | LLaVA-NeXT-Video | T + V | `llava-hf/LLaVA-NeXT-Video-7B-hf`, etc. | | ✅︎ |
584 +| `LlavaOnevision2ForConditionalGeneration` | LLaVA-OneVision-2 | T + I<sup>+</sup> + V<sup>+</sup> | `lmms-lab-encoder/LLaVA-OneVision-2-8B-Instruct` | | |
585 | `LlavaOnevisionForConditionalGeneration` | LLaVA-Onevision | T + I<sup>+</sup> + V<sup>+</sup> | `llava-hf/llava-onevision-qwen2-7b-ov-hf`, `llava-hf/llava-onevision-qwen2-0.5b-ov-hf`, etc. | | ✅︎ |
586 | `MiDashengLMModel` | MiDashengLM | T + A<sup>+</sup> | `mispeech/midashenglm-7b` | | ✅︎ |
587 | `MiMoV2OmniForCausalLM` | MiMo-V2.5-Omni | T + I<sup>E+</sup> + V<sup>E+</sup> + A<sup>+</sup> | `XiaomiMiMo/MiMo-V2.5-Omni` | | ✅︎ |
tests/models/multimodal/processing/test_common.py
+9
@@ -452,6 +452,15 @@ def test_processing_correctness(
452 "audio placeholders from processed audio lengths. Its vLLM "
453 "processor paths are covered by test_moss_audio.py."
454 )
455 + if model_id == "lmms-lab-encoder/LLaVA-OneVision-2-8B-Instruct":
456 + pytest.skip(
457 + "LLaVA-OneVision-2 video processing routes frames through custom "
458 + "video backends (qwen_vl_utils / codec) that require real encoded "
459 + "video bytes and metadata. The synthetic numpy-array videos used by "
460 + "this test yield empty video features, so the generic correctness "
461 + "check cannot exercise the video path. Image processing is covered "
462 + "by registration/inference tests."
463 + )
464
465 _test_processing_correctness(
466 model_id,
tests/models/multimodal/processing/test_llava_onevision2.py new
+302
@@ -0,0 +1,302 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""Unit tests for the LLaVA-OneVision-2 codec-video marker mechanism.
4 +
5 +The codec video backend cannot be exercised end-to-end in CI because it
6 +relies on the model's ``trust_remote_code`` package and real video bytes.
7 +These tests cover the pure-Python marker plumbing that wraps a video *path*
8 +for vLLM's ``MultiModalDataParser``:
9 +
10 +* :func:`prepare_codec_video_input` must encode a per-path hash into its
11 + dummy ndarray so distinct codec videos do not collide on ``mm_hash``
12 + (otherwise ``EncoderCacheManager`` would skip the encoder for every video
13 + after the first).
14 +* :func:`_extract_codec_video_paths` must round-trip the marker back to the
15 + original path(s) after the parser strips the metadata dict, and must return
16 + ``None`` for non-codec inputs.
17 +"""
18 +
19 +import types
20 +
21 +import numpy as np
22 +import pytest
23 +
24 +from vllm.model_executor.models.llava_onevision2 import (
25 + _CODEC_VIDEO_MARKER,
26 + LlavaOnevision2VideoBackend,
27 + _extract_codec_video_paths,
28 + _frame_video_to_pil_and_timestamps,
29 + _validate_video_source,
30 + prepare_codec_video_input,
31 +)
32 +from vllm.multimodal.video import VideoSourceMetadata, VideoTargetMetadata
33 +
34 +
35 +def _model_config(local: str = "", domains=None):
36 + # Minimal stand-in for vLLM's ModelConfig: the validator only reads
37 + # ``allowed_local_media_path`` and ``allowed_media_domains``.
38 + return types.SimpleNamespace(
39 + allowed_local_media_path=local,
40 + allowed_media_domains=domains,
41 + )
42 +
43 +
44 +def test_prepare_codec_video_input_shape_and_marker():
45 + dummy, meta = prepare_codec_video_input("/data/foo.mp4")
46 +
47 + # 4-D ndarray satisfies MultiModalDataParser's video shape check.
48 + assert isinstance(dummy, np.ndarray)
49 + assert dummy.shape == (1, 1, 16, 3)
50 + assert dummy.dtype == np.uint8
51 +
52 + # Metadata carries the exact path under the codec marker key.
53 + assert meta == {_CODEC_VIDEO_MARKER: "/data/foo.mp4"}
54 +
55 +
56 +def test_prepare_codec_video_input_is_deterministic():
57 + # Same path must yield identical dummy bytes (stable mm_hash).
58 + a, _ = prepare_codec_video_input("/data/foo.mp4")
59 + b, _ = prepare_codec_video_input("/data/foo.mp4")
60 + assert a.tobytes() == b.tobytes()
61 +
62 +
63 +def test_prepare_codec_video_input_distinct_paths_distinct_bytes():
64 + # Distinct paths must yield distinct dummy bytes so the parser-visible
65 + # ndarray (the only part reaching MultiModalHasher) varies per video.
66 + paths = ["/data/a.mp4", "/data/b.mp4", "/data/c.mp4", "/data/a_.mp4"]
67 + payloads = {prepare_codec_video_input(p)[0].tobytes() for p in paths}
68 + assert len(payloads) == len(paths)
69 +
70 +
71 +def test_extract_codec_video_paths_parser_list_shape():
72 + # Parser yields list-of-(ndarray, metadata-dict) for tuple inputs.
73 + items = [prepare_codec_video_input(p) for p in ("/x/1.mp4", "/x/2.mp4")]
74 + assert _extract_codec_video_paths(items) == ["/x/1.mp4", "/x/2.mp4"]
75 +
76 +
77 +def test_extract_codec_video_paths_single_raw_tuple():
78 + # Single raw (ndarray, dict) tuple (pre-parser path) is also accepted.
79 + item = prepare_codec_video_input("/x/solo.mp4")
80 + assert _extract_codec_video_paths(item) == ["/x/solo.mp4"]
81 +
82 +
83 +def test_extract_codec_video_paths_non_codec_returns_none():
84 + # Plain decoded-frame inputs (ndarray / list of ndarray) are not codec
85 + # markers and must fall through to the frame backend.
86 + plain = np.zeros((4, 8, 8, 3), dtype=np.uint8)
87 + assert _extract_codec_video_paths(plain) is None
88 + assert _extract_codec_video_paths([plain, plain]) is None
89 + assert _extract_codec_video_paths([]) is None
90 + # Tuple without the marker key is ignored.
91 + assert _extract_codec_video_paths((plain, {"fps": 2.0})) is None
92 +
93 +
94 +def test_extract_codec_video_paths_mixed_batch_returns_none():
95 + # If any item in the batch lacks the marker, the whole batch is treated
96 + # as non-codec (the backend does not mix codec and frame videos).
97 + codec = prepare_codec_video_input("/x/1.mp4")
98 + plain = np.zeros((4, 8, 8, 3), dtype=np.uint8)
99 + assert _extract_codec_video_paths([codec, plain]) is None
100 +
101 +
102 +# ---------------------------------------------------------------------------
103 +# Media access controls (_validate_video_source)
104 +#
105 +# The codec backend keeps the raw path string alive past vLLM's
106 +# MultiModalDataParser and hands it to the trust-remote-code codec module,
107 +# which opens it directly (cv2/ffmpeg) outside vLLM's MediaConnector. So the
108 +# codec backend is restricted to *local files* confined to
109 +# --allowed-local-media-path; remote http(s)/data URLs are rejected (they must
110 +# use the frame backend, which rides the connector). The validator also returns
111 +# the *resolved* path so the codec module opens exactly what was validated
112 +# (no symlink-retarget / validate-vs-open gap).
113 +# ---------------------------------------------------------------------------
114 +
115 +
116 +def test_validate_http_url_rejected():
117 + # Codec backend is local-only: remote URLs bypass the connector's domain
118 + # and redirect controls, so they are rejected regardless of allowlist.
119 + with pytest.raises(ValueError):
120 + _validate_video_source("http://example.com/v.mp4", _model_config())
121 +
122 +
123 +def test_validate_https_url_rejected_even_when_host_allowlisted():
124 + with pytest.raises(ValueError):
125 + _validate_video_source(
126 + "https://good.com/v.mp4",
127 + _model_config(domains=["good.com"]),
128 + )
129 +
130 +
131 +def test_validate_data_url_rejected():
132 + # data: URLs are a remote/inline source for the frame backend, not codec.
133 + with pytest.raises(ValueError):
134 + _validate_video_source("data:video/mp4;base64,AAAA", _model_config())
135 +
136 +
137 +def test_validate_local_file_blocked_without_allowed_path():
138 + # Local file access is opt-in: without --allowed-local-media-path the
139 + # bare path is rejected.
140 + with pytest.raises(ValueError):
141 + _validate_video_source("/data/v.mp4", _model_config())
142 +
143 +
144 +def test_validate_local_file_allowed_inside_allowed_dir(tmp_path):
145 + f = tmp_path / "v.mp4"
146 + f.touch()
147 + assert _validate_video_source(str(f), _model_config(local=str(tmp_path))) == str(f)
148 +
149 +
150 +def test_validate_local_file_traversal_blocked():
151 + # Path traversal escaping the allowed root is rejected after resolution.
152 + with pytest.raises(ValueError):
153 + _validate_video_source(
154 + "/tmp/ov2/../../etc/passwd",
155 + _model_config(local="/tmp/ov2"),
156 + )
157 +
158 +
159 +def test_validate_file_scheme_allowed_inside_allowed_dir(tmp_path):
160 + f = tmp_path / "v.mp4"
161 + f.touch()
162 + assert _validate_video_source(
163 + f.as_uri(), _model_config(local=str(tmp_path))
164 + ) == str(f)
165 +
166 +
167 +def test_validate_file_scheme_percent_encoded_traversal_blocked():
168 + # ``%2e%2e`` decodes to ``..``; the confinement check URL-decodes first
169 + # (url2pathname) so the path cannot stay literally under the allowed root.
170 + with pytest.raises(ValueError):
171 + _validate_video_source(
172 + "file:///tmp/ov2/%2e%2e/%2e%2e/etc/passwd",
173 + _model_config(local="/tmp/ov2"),
174 + )
175 +
176 +
177 +def test_validate_unsupported_scheme_blocked():
178 + with pytest.raises(ValueError):
179 + _validate_video_source("ftp://example.com/v.mp4", _model_config())
180 +
181 +
182 +def test_validate_returns_resolved_path_through_symlink(tmp_path):
183 + # The validator resolves symlinks and returns the *real* path, so the codec
184 + # module opens exactly what was validated (closes the validate-vs-open gap).
185 + real = tmp_path / "real.mp4"
186 + real.touch()
187 + link = tmp_path / "link.mp4"
188 + link.symlink_to(real)
189 + assert _validate_video_source(str(link), _model_config(local=str(tmp_path))) == str(
190 + real
191 + )
192 +
193 +
194 +def test_validate_relative_bare_path_blocked():
195 + # Bare (scheme=="") paths come only from the codec backend and must be
196 + # absolute: resolving a relative path against an ambiguous CWD before the
197 + # confinement check is brittle/unsafe, so it is rejected outright.
198 + with pytest.raises(ValueError):
199 + _validate_video_source("ov2/v.mp4", _model_config(local="/tmp/ov2"))
200 +
201 +
202 +# ---------------------------------------------------------------------------
203 +# Frame backend marker -> PIL + timestamps (_frame_video_to_pil_and_timestamps)
204 +#
205 +# Non-codec videos reach _call_hf_processor as a ``(frames_ndarray, metadata)``
206 +# tuple -- produced by the registered ``LlavaOnevision2VideoBackend`` for real
207 +# ``video_url`` inputs, or by the dummy-inputs builder during profiling
208 +# (``video_needs_metadata=True``). The helper materialises PIL frames and
209 +# per-frame timestamps (``frame_index / fps``), padding the frame count up to
210 +# the even temporal-merge boundary.
211 +# ---------------------------------------------------------------------------
212 +
213 +
214 +def test_frame_video_to_pil_and_timestamps_basic():
215 + frames = np.zeros((4, 8, 8, 3), dtype=np.uint8)
216 + metadata = {"fps": 2.0, "frames_indices": [0, 4, 8, 12]}
217 + pil_frames, timestamps = _frame_video_to_pil_and_timestamps((frames, metadata))
218 +
219 + assert len(pil_frames) == 4
220 + assert all(f.size == (8, 8) for f in pil_frames)
221 + # timestamps = frame_index / fps
222 + assert timestamps == [0.0, 2.0, 4.0, 6.0]
223 +
224 +
225 +def test_frame_video_to_pil_and_timestamps_even_pads_odd_frame_count():
226 + # Odd frame count -> last frame repeated to satisfy temporal merge=2.
227 + frames = np.zeros((3, 8, 8, 3), dtype=np.uint8)
228 + metadata = {"fps": 1.0, "frames_indices": [0, 1, 2]}
229 + pil_frames, timestamps = _frame_video_to_pil_and_timestamps((frames, metadata))
230 +
231 + assert len(pil_frames) == 4
232 + assert len(timestamps) == 4
233 + # The padded frame reuses the final index/timestamp.
234 + assert timestamps == [0.0, 1.0, 2.0, 2.0]
235 +
236 +
237 +def test_frame_video_to_pil_and_timestamps_defaults_when_metadata_sparse():
238 + # Missing frames_indices -> sequential range; missing/zero fps -> default.
239 + frames = np.zeros((2, 8, 8, 3), dtype=np.uint8)
240 + pil_frames, timestamps = _frame_video_to_pil_and_timestamps((frames, {}))
241 +
242 + assert len(pil_frames) == 2
243 + # Default fps is 1.0, indices fall back to range(T).
244 + assert timestamps == [0.0, 1.0]
245 +
246 +
247 +def test_frame_video_to_pil_and_timestamps_rejects_non_tuple():
248 + # Bare arrays (no metadata) must be rejected: the frame backend requires
249 + # the (frames, metadata) tuple produced by the registered loader.
250 + plain = np.zeros((4, 8, 8, 3), dtype=np.uint8)
251 + with pytest.raises(ValueError):
252 + _frame_video_to_pil_and_timestamps(plain)
253 +
254 +
255 +# ---------------------------------------------------------------------------
256 +# LlavaOnevision2VideoBackend.compute_frames_index_to_sample honors the caller
257 +# supplied VideoTargetMetadata (passed via --media-io-kwargs) so benchmarks can
258 +# override the conservative defaults (fps=1.0, max_frames=32). Unset target
259 +# fields (sentinel <= 0) fall back to those OV2 hf-chat reference constants.
260 +# ---------------------------------------------------------------------------
261 +
262 +
263 +def _src(total_frames: int, fps: float) -> VideoSourceMetadata:
264 + return VideoSourceMetadata(
265 + total_frames_num=total_frames,
266 + original_fps=fps,
267 + duration=total_frames / fps if fps > 0 else 0.0,
268 + )
269 +
270 +
271 +def test_backend_defaults_cap_at_32_frames():
272 + # 300 frames @ 1fps source, target unset -> capped at default max_frames=32.
273 + src = _src(300, 1.0)
274 + target = VideoTargetMetadata(num_frames=-1, fps=-1, max_duration=300.0)
275 + idx = LlavaOnevision2VideoBackend.compute_frames_index_to_sample(src, target)
276 +
277 + assert len(idx) == 32
278 + assert idx[0] == 0
279 + assert idx[-1] == 299
280 + assert len(idx) % 2 == 0
281 +
282 +
283 +def test_backend_target_num_frames_overrides_default_cap():
284 + # VSI-Bench parity: target.num_frames=128 must lift the 32-frame cap.
285 + src = _src(300, 1.0)
286 + target = VideoTargetMetadata(num_frames=128, fps=-1, max_duration=300.0)
287 + idx = LlavaOnevision2VideoBackend.compute_frames_index_to_sample(src, target)
288 +
289 + assert len(idx) == 128
290 + assert idx[0] == 0
291 + assert idx[-1] == 299
292 +
293 +
294 +def test_backend_target_fps_controls_sampling_when_below_cap():
295 + # 60 frames @ 30fps (2s) with target fps=1 -> ~2 frames (even-padded).
296 + src = _src(60, 30.0)
297 + target = VideoTargetMetadata(num_frames=128, fps=1.0, max_duration=300.0)
298 + idx = LlavaOnevision2VideoBackend.compute_frames_index_to_sample(src, target)
299 +
300 + # fps-derived nframes (2) is below the 128 cap, so fps wins.
301 + assert len(idx) <= 8
302 + assert len(idx) % 2 == 0
tests/models/registry.py
+11
@@ -1048,6 +1048,17 @@ _MULTIMODAL_EXAMPLE_MODELS = {
1048 "LlavaNextVideoForConditionalGeneration": _HfExamplesInfo(
1049 "llava-hf/LLaVA-NeXT-Video-7B-hf"
1050 ),
1051 + "LlavaOnevision2ForConditionalGeneration": _HfExamplesInfo(
1052 + "lmms-lab-encoder/LLaVA-OneVision-2-8B-Instruct",
1053 + trust_remote_code=True,
1054 + # Keep the init/schema harness eager (these tests never run forward).
1055 + # NOTE: a separate H200-only hang exists in the core FA3 attention
1056 + # construction (after `FlashAttentionImpl.__init__` logs the FA3
1057 + # version, while building the Qwen3 backbone). enforce_eager does NOT
1058 + # gate that path, so this flag is hygiene, not the fix -- tracked with
1059 + # the maintainer.
1060 + enforce_eager=True,
1061 + ),
1062 "LlavaOnevisionForConditionalGeneration": _HfExamplesInfo(
1063 "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
1064 ),
vllm/model_executor/models/llava_onevision2.py new
+2266
@@ -0,0 +1,2266 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""
4 +Inference-only LLaVA-OneVision-2 (OV2) model for vLLM.
5 +
6 +Architecture notes:
7 +
8 + * LLM backbone is plain Qwen3-8B with 1-D position_ids (no M-RoPE).
9 + * Vision tower removes the CLS token (no class_embedding/class_pos_emb).
10 + * Vision RoPE is 3-D (T:H:W) with a 4:6:6 head_dim split and uses
11 + ``patch_positions`` instead of grid_thw to compute per-token freqs.
12 + * ``rotate_half`` is *interleaved* (``(::2, 1::2)``) rather than
13 + split-half.
14 + * Vision attention uses windowed ``cu_seqlens`` (``frame_windows_size``
15 + in T-dim); two backends implemented (SDPA + flash_attn varlen).
16 + * ``patch_positions: [total_patches, 3]`` is plumbed as a first-class
17 + MM kwarg alongside ``pixel_values`` / ``image_grid_thw``.
18 + * Video frame-backend and codec-backend both alias to the image path
19 + inside the HF processor, so the model implements a single visual
20 + code path.
21 +"""
22 +
23 +from __future__ import annotations
24 +
25 +import hashlib
26 +import importlib
27 +import json
28 +import os
29 +from collections.abc import Callable, Iterable, Mapping, Sequence
30 +from functools import lru_cache
31 +from typing import (
32 + Annotated,
33 + Any,
34 + Literal,
35 +)
36 +
37 +import numpy as np
38 +import regex as re
39 +import torch
40 +import torch.nn as nn
41 +import torch.nn.functional as F
42 +from huggingface_hub import hf_hub_download
43 +from PIL import Image
44 +from transformers import AutoProcessor, AutoTokenizer, BatchFeature
45 +from transformers.dynamic_module_utils import get_class_from_dynamic_module
46 +from transformers.models.qwen2_vl import Qwen2VLImageProcessor
47 +from transformers.models.qwen2_vl.image_processing_qwen2_vl import smart_resize
48 +
49 +from vllm.compilation.decorators import (
50 + should_torch_compile_mm_encoder,
51 + support_torch_compile,
52 +)
53 +from vllm.config import VllmConfig
54 +from vllm.config.multimodal import BaseDummyOptions
55 +from vllm.distributed import parallel_state
56 +from vllm.distributed import utils as dist_utils
57 +from vllm.inputs import ModalityData, MultiModalDataDict
58 +from vllm.logger import init_logger
59 +from vllm.model_executor.layers.attention import MMEncoderAttention
60 +from vllm.model_executor.layers.linear import (
61 + ColumnParallelLinear,
62 + QKVParallelLinear,
63 + RowParallelLinear,
64 +)
65 +from vllm.model_executor.layers.quantization import QuantizationConfig
66 +from vllm.model_executor.models.interfaces import (
67 + MultiModalEmbeddings,
68 + SupportsMultiModal,
69 + SupportsPP,
70 +)
71 +from vllm.model_executor.models.module_mapping import MultiModelKeys
72 +from vllm.model_executor.models.utils import (
73 + AutoWeightsLoader,
74 + WeightsMapper,
75 + init_vllm_registered_model,
76 + maybe_prefix,
77 +)
78 +from vllm.model_executor.models.utils import (
79 + _merge_multimodal_embeddings as merge_multimodal_embeddings,
80 +)
81 +from vllm.model_executor.models.vision import get_vit_attn_backend
82 +from vllm.multimodal import MULTIMODAL_REGISTRY
83 +from vllm.multimodal.inputs import (
84 + ImageItem,
85 + MultiModalFieldConfig,
86 + MultiModalKwargsItems,
87 +)
88 +from vllm.multimodal.parse import (
89 + DictEmbeddingItems,
90 + ImageSize,
91 + ModalityDataItems,
92 + MultiModalDataItems,
93 + MultiModalDataParser,
94 +)
95 +from vllm.multimodal.processing import (
96 + BaseMultiModalProcessor,
97 + BaseProcessingInfo,
98 + PromptReplacement,
99 + PromptUpdate,
100 + PromptUpdateDetails,
101 +)
102 +from vllm.multimodal.processing.dummy_inputs import BaseDummyInputsBuilder
103 +from vllm.multimodal.video import (
104 + VIDEO_LOADER_REGISTRY,
105 + VideoBackend,
106 + VideoSourceMetadata,
107 + VideoTargetMetadata,
108 +)
109 +from vllm.sequence import IntermediateTensors
110 +from vllm.transformers_utils.processor import _merge_mm_kwargs
111 +from vllm.transformers_utils.utils import convert_model_repo_to_path
112 +from vllm.utils.tensor_schema import TensorSchema, TensorShape
113 +
114 +logger = init_logger(__name__)
115 +
116 +
117 +@lru_cache
118 +def _load_ov2_processor(
119 + model: str,
120 + revision: str | None,
121 + trust_remote_code: bool,
122 + **kwargs: Any,
123 +):
124 + # OV2's trust_remote_code processor is a bare class (not a ProcessorMixin),
125 + # so the shared type-checked get_hf_processor rejects it. We also can't use
126 + # AutoProcessor.from_pretrained: OV2's remote from_pretrained drops
127 + # trust_remote_code before building its nested tokenizer, which makes that
128 + # nested load fall back to an interactive stdin prompt that hangs in
129 + # non-interactive CI. Instead, assemble the processor here with
130 + # trust_remote_code threaded through every component explicitly.
131 + path = convert_model_repo_to_path(model)
132 + revision = revision or "main"
133 +
134 + processor_cls = get_class_from_dynamic_module(
135 + "processing_llava_onevision2.LlavaOnevision2Processor",
136 + path,
137 + revision=revision,
138 + trust_remote_code=trust_remote_code,
139 + )
140 + video_processor_cls = get_class_from_dynamic_module(
141 + "video_processing_llava_onevision2.LlavaOnevision2VideoProcessor",
142 + path,
143 + revision=revision,
144 + trust_remote_code=trust_remote_code,
145 + )
146 +
147 + # Slow Qwen2VLImageProcessor mirrors the remote processor (the Fast variant
148 + # has normalization rounding differences that change pixel_values).
149 + image_processor = Qwen2VLImageProcessor.from_pretrained(
150 + path, revision=revision, **kwargs
151 + )
152 + tokenizer = AutoTokenizer.from_pretrained(
153 + path, revision=revision, trust_remote_code=trust_remote_code, **kwargs
154 + )
155 + video_processor = video_processor_cls(
156 + image_processor=image_processor,
157 + min_pixels=getattr(image_processor, "min_pixels", 256 * 28 * 28),
158 + max_pixels=getattr(image_processor, "max_pixels", 1605632),
159 + patch_size=getattr(image_processor, "patch_size", 14),
160 + spatial_merge_size=getattr(image_processor, "merge_size", 2),
161 + )
162 +
163 + # Codec defaults live under the "codec" key of preprocessor_config.json,
164 + # which Qwen2VLImageProcessor does not preserve; read them directly so the
165 + # codec video backend keeps its configured defaults.
166 + codec_config: dict = {}
167 + try:
168 + config_file = os.path.join(path, "preprocessor_config.json")
169 + if not os.path.isfile(config_file):
170 + config_file = hf_hub_download(
171 + path, "preprocessor_config.json", revision=revision
172 + )
173 + with open(config_file, encoding="utf-8") as f:
174 + codec_config = json.load(f).get("codec", {}) or {}
175 + except Exception:
176 + logger.debug("OV2: no codec defaults found in preprocessor_config.json")
177 +
178 + return processor_cls(
179 + image_processor=image_processor,
180 + tokenizer=tokenizer,
181 + video_processor=video_processor,
182 + codec_config=codec_config,
183 + )
184 +
185 +
186 +# Upper bound on frames used when profiling the worst-case video item, mirroring
187 +# Qwen2-VL. The real frame count is decided by the HF VideoProcessor at apply
188 +# time; this only sizes the memory-profiling estimate.
189 +_MAX_FRAMES_PER_VIDEO = 14
190 +
191 +
192 +def _pack_timestamps(per_video: list[list[float]]) -> torch.Tensor:
193 + if not per_video:
194 + return torch.empty((0, 0), dtype=torch.float32)
195 + t_max = max((len(ts) for ts in per_video), default=0)
196 + padded = torch.zeros((len(per_video), t_max), dtype=torch.float32)
197 + for i, ts in enumerate(per_video):
198 + padded[i, : len(ts)] = torch.tensor(ts, dtype=torch.float32)
199 + return padded
200 +
201 +
202 +def _validate_video_source(path: str, model_config) -> str:
203 + """Confine a codec video path to ``--allowed-local-media-path``.
204 +
205 + The codec backend keeps the raw path string alive past vLLM's
206 + ``MultiModalDataParser`` and hands it to the trust-remote-code codec
207 + module, which opens it directly via ``cv2.VideoCapture`` / ffmpeg. That
208 + bypasses both ``MediaConnector``'s access controls and its redirect
209 + handling (``VLLM_MEDIA_URL_ALLOW_REDIRECTS``), so we restrict the codec
210 + backend to **local files only**: remote ``http(s)`` / ``data`` URLs are
211 + rejected here and must instead go through the frame backend (a registered
212 + ``VIDEO_LOADER_REGISTRY`` loader), which rides vLLM's connector and its
213 + domain/redirect gates.
214 +
215 + Returns the *resolved* absolute path so the codec module opens exactly the
216 + file that was validated, closing the validate-vs-open (symlink-retarget)
217 + window. Mirrors the confinement in ``MediaConnector._load_file_url``.
218 + """
219 + from pathlib import Path
220 + from urllib.request import url2pathname
221 +
222 + from urllib3.util import parse_url
223 +
224 + allowed_local = getattr(model_config, "allowed_local_media_path", "") or ""
225 +
226 + parsed = parse_url(str(path))
227 + scheme = (parsed.scheme or "").lower()
228 +
229 + if scheme in ("http", "https", "data"):
230 + raise ValueError(
231 + f"The codec video backend does not support remote {scheme!r} URLs: "
232 + f"its trust-remote-code decoder fetches them outside vLLM's domain "
233 + f"and redirect controls. Use a local file path, or the frame "
234 + f"backend for remote videos."
235 + )
236 + if scheme not in ("", "file"):
237 + raise ValueError(
238 + f"Unsupported codec video URL scheme {scheme!r}; only local file "
239 + f"paths or file:// URLs are supported."
240 + )
241 +
242 + # Local file access is opt-in: require --allowed-local-media-path and
243 + # confine the resolved path to that directory (connector.py:253-271).
244 + if not allowed_local:
245 + raise ValueError(
246 + "Local video file access is disabled. Set "
247 + "--allowed-local-media-path to enable reading local videos."
248 + )
249 + if scheme == "file":
250 + # Decode percent-encoding (mirrors MediaConnector._load_file_url),
251 + # including the netloc so file://host/path is handled identically.
252 + local = Path(url2pathname((parsed.netloc or "") + (parsed.path or "")))
253 + else:
254 + local = Path(str(path))
255 + # Require an absolute path: resolving a relative path against an ambiguous
256 + # CWD before the confinement check is brittle/unsafe.
257 + if not local.is_absolute():
258 + raise ValueError(
259 + f"Local video path {str(path)!r} must be absolute; "
260 + f"relative paths are not supported."
261 + )
262 + allowed_root = Path(allowed_local).resolve()
263 + resolved = local.resolve()
264 + if resolved != allowed_root and allowed_root not in resolved.parents:
265 + raise ValueError(
266 + f"Video path {str(path)!r} is outside the allowed local media "
267 + f"directory {allowed_local!r}."
268 + )
269 + return str(resolved)
270 +
271 +
272 +def _validate_video_sources(paths, model_config) -> list[str]:
273 + # Return the resolved paths so the codec module opens exactly what was
274 + # validated (no validate-vs-open / symlink-retarget differential).
275 + return [_validate_video_source(path, model_config) for path in paths]
276 +
277 +
278 +# Design note: the two video backends take deliberately different paths.
279 +#
280 +# * frame backend: a normal vLLM VIDEO_LOADER_REGISTRY loader
281 +# (``LlavaOnevision2VideoBackend`` below). Decoding to RGB
282 +# ``(frames, metadata)`` is exactly what loaders are for, so frame sampling
283 +# participates in the standard decode-stage pipeline.
284 +#
285 +# * codec backend: NOT a loader. OV2's codec path needs the video path string
286 +# to survive into ``_call_hf_processor``, where the HF processor builds the
287 +# codec canvas + smart_resize + patchify
288 +# (pixel_values/image_grid_thw/patch_positions). That transform is
289 +# path-level and inseparable; it cannot be reconstructed from pre-decoded
290 +# RGB frames, so it must run at the processor stage rather than the decode
291 +# stage. The small marker/parser machinery below keeps the path alive for
292 +# codec; ``_validate_video_source`` then confines it to a local file (codec
293 +# does not support remote URLs -- see its docstring).
294 +_CODEC_VIDEO_MARKER = "ov2_codec_video"
295 +
296 +
297 +def prepare_codec_video_input(video_path: str) -> tuple:
298 + """Wrap a video path for vLLM's MultiModalDataParser + OV2 codec backend.
299 +
300 + Returns ``(dummy_ndarray, metadata)`` where the ndarray satisfies the
301 + parser's 4-D shape check and the metadata carries the actual path to
302 + our ``_call_hf_processor``. Use as::
303 +
304 + multi_modal_data = {"video": prepare_codec_video_input("foo.mp4")}
305 +
306 + The dummy ndarray bytes encode a hash of ``video_path`` so distinct codec
307 + videos get distinct mm_hashes: the parser drops the metadata dict before
308 + hashing (only the ndarray reaches MultiModalHasher), so without this
309 + variance every video after the first would collide and skip the encoder.
310 + """
311 + path_str = str(video_path)
312 + digest = hashlib.blake2b(path_str.encode("utf-8"), digest_size=16).digest()
313 + dummy = np.frombuffer(digest, dtype=np.uint8).reshape(1, 1, 16, 1)
314 + dummy = np.broadcast_to(dummy, (1, 1, 16, 3)).copy()
315 + return (dummy, {_CODEC_VIDEO_MARKER: str(video_path)})
316 +
317 +
318 +def _extract_codec_video_paths(videos: Any) -> list[str] | None:
319 + # vLLM's parser yields list-of-(ndarray, metadata-dict) for tuple inputs.
320 + # We accept either that shape or a single raw tuple (pre-parser cases).
321 + def _path_from(item):
322 + if (
323 + isinstance(item, tuple)
324 + and len(item) == 2
325 + and isinstance(item[1], dict)
326 + and _CODEC_VIDEO_MARKER in item[1]
327 + ):
328 + return item[1][_CODEC_VIDEO_MARKER]
329 + return None
330 +
331 + if isinstance(videos, list):
332 + paths: list[str] = []
333 + for item in videos:
334 + p = _path_from(item)
335 + if p is None:
336 + return None
337 + paths.append(p)
338 + return paths if paths else None
339 + p = _path_from(videos)
340 + return [p] if p is not None else None
341 +
342 +
343 +_CODEC_FPS_CACHE: dict[str, float] = {}
344 +
345 +
346 +def _codec_fps_for(video_path: str, hf_processor) -> float:
347 + if video_path in _CODEC_FPS_CACHE:
348 + return _CODEC_FPS_CACHE[video_path]
349 + # The codec module is shipped inside the HF transformers_modules package
350 + # for OV2, so an absolute import does not resolve. Locate it relative to
351 + # the processor module (which lives in the same package).
352 + proc_module_name = type(hf_processor).__module__
353 + pkg = proc_module_name.rsplit(".", 1)[0] if "." in proc_module_name else ""
354 + codec_mod = importlib.import_module(
355 + f"{pkg}.codec_video_processing_llava_onevision2"
356 + if pkg
357 + else "codec_video_processing_llava_onevision2"
358 + )
359 + CodecConfig = codec_mod.CodecConfig
360 + process_codec_video = codec_mod.process_codec_video
361 + cfg_defaults = dict(getattr(hf_processor, "_codec_config_defaults", {}))
362 + cfg = CodecConfig(**cfg_defaults)
363 + payload = process_codec_video(video_path, cfg)
364 + fps = float(payload["fps"])
365 + _CODEC_FPS_CACHE[video_path] = fps
366 + return fps
367 +
368 +
369 +def _codec_timestamp_runs(
370 + patch_positions: torch.Tensor,
371 + fps: float,
372 + spatial_merge_size: int,
373 +) -> list[tuple[float, int]]:
374 + # Mirrors HF's _timestamp_runs (codec_video_processing_llava_onevision2.py)
375 + # exactly: same column, same merge factor, same negative-t skip, same
376 + # zero-token-count skip. Keeping the logic local avoids importing the
377 + # private helper from transformers_modules.
378 + t_values = patch_positions[:, 0]
379 + unique_t, counts = torch.unique_consecutive(t_values, return_counts=True)
380 + merge_factor = int(spatial_merge_size) ** 2
381 + runs: list[tuple[float, int]] = []
382 + for t_val, count in zip(unique_t.tolist(), counts.tolist()):
383 + if int(t_val) < 0:
384 + continue
385 + token_count = int(count) // merge_factor
386 + if token_count <= 0:
387 + continue
388 + runs.append((float(t_val) / float(fps), token_count))
389 + return runs
390 +
391 +
392 +def _create_field_factory(
393 + spatial_merge_size: int,
394 +) -> Callable[[Mapping[str, torch.Tensor]], Mapping[str, MultiModalFieldConfig]]:
395 + """Build the per-batch field-config callback.
396 +
397 + OV2-specific: also exposes ``patch_positions`` as a flat-from-sizes
398 + field, sized by the total per-image patch count (T*H*W). The merger and
399 + the 3-D RoPE both consume it.
400 + """
401 +
402 + def _field_config(hf_inputs: Mapping[str, torch.Tensor]):
403 + image_grid_thw = hf_inputs.get("image_grid_thw", torch.empty((0, 3)))
404 + image_pixel_grid_sizes = image_grid_thw.prod(-1)
405 + image_embed_grid_sizes = (
406 + image_pixel_grid_sizes // spatial_merge_size // spatial_merge_size
407 + )
408 +
409 + video_grid_thw = hf_inputs.get("video_grid_thw", torch.empty((0, 3)))
410 + # OV2 emits one grid_thw row per frame, so vLLM's per-video sharding
411 + # requires explicit frame counts. video_patch_sizes sums H*W over the
412 + # frames that belong to each video; video_grid_thw uses the frame
413 + # count directly (one row per frame).
414 + video_num_frames = hf_inputs.get(
415 + "video_num_frames", torch.empty((0,), dtype=torch.long)
416 + )
417 + if video_num_frames.numel() > 0:
418 + per_row_patches = video_grid_thw.prod(-1)
419 + offsets = torch.cumsum(
420 + torch.cat([torch.zeros(1, dtype=torch.long), video_num_frames[:-1]]), 0
421 + ).tolist()
422 + video_patch_sizes = torch.tensor(
423 + [
424 + int(per_row_patches[int(s) : int(s) + int(n)].sum())
425 + for s, n in zip(offsets, video_num_frames.tolist())
426 + ],
427 + dtype=torch.long,
428 + )
429 + else:
430 + video_patch_sizes = torch.empty((0,), dtype=torch.long)
431 +
432 + return dict(
433 + pixel_values=MultiModalFieldConfig.flat_from_sizes(
434 + "image", image_pixel_grid_sizes
435 + ),
436 + image_embeds=MultiModalFieldConfig.flat_from_sizes(
437 + "image", image_embed_grid_sizes
438 + ),
439 + image_grid_thw=MultiModalFieldConfig.batched("image"),
440 + # OV2 first-class MM kwarg: per-patch (t,h,w)
441 + # positions required by the 3-D vision RoPE.
442 + patch_positions=MultiModalFieldConfig.flat_from_sizes(
443 + "image", image_pixel_grid_sizes
444 + ),
445 + pixel_values_videos=MultiModalFieldConfig.flat_from_sizes(
446 + "video", video_patch_sizes
447 + ),
448 + video_grid_thw=MultiModalFieldConfig.flat_from_sizes(
449 + "video", video_num_frames
450 + ),
451 + patch_positions_videos=MultiModalFieldConfig.flat_from_sizes(
452 + "video", video_patch_sizes
453 + ),
454 + video_num_frames=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
455 + frame_timestamps=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
456 + # Per-video flag: 0 = frame-sampling backend, 1 = codec backend.
457 + # Drives codec-aware ``\n`` insertion in ``get_video_replacement``.
458 + video_is_codec=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
459 + # Codec backend: per-video source-frame fps. Needed at
460 + # replacement time to convert patch_positions t-indices into
461 + # the timestamp tags HF writes (``<sec seconds>``).
462 + codec_fps=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
463 + )
464 +
465 + return _field_config
466 +
467 +
468 +# ---------------------------------------------------------------------------
469 +# Frame backend helpers
470 +# ---------------------------------------------------------------------------
471 +# The default video pathway materialises each video as a series of PIL frames
472 +# (decoded + sampled by the registered ``LlavaOnevision2VideoBackend``) and
473 +# feeds them to the HF processor through the *image* branch (per-frame timestamp
474 +# marker + ``<|image_pad|>``). This mirrors the validated lmms-eval
475 +# ``vllm_hf_chat`` adapter, and empirically scores higher on Video-MME than
476 +# OV2's native VideoProcessor frame extractor.
477 +_DEFAULT_TIMESTAMP_DECIMALS = 1
478 +_DEFAULT_FPS = 1.0
479 +_DEFAULT_MAX_FRAMES = 32
480 +# OV2 vision tower has spatial_merge_size=2 -> temporal frame count must be
481 +# even. The hf-chat reference pads by repeating the last frame; same here.
482 +_TEMPORAL_MERGE_SIZE = 2
483 +
484 +# Token sequence emitted by the OV2 vLLM dummy inputs builder for each video
485 +# item. The frame backend expands each marker into per-frame image markers
486 +# (timestamp + image_pad block).
487 +_VIDEO_MARKER = "<|vision_start|><|video_pad|><|vision_end|>"
488 +_IMAGE_MARKER = "<|vision_start|><|image_pad|><|vision_end|>"
489 +
490 +
491 +def _frame_video_to_pil_and_timestamps(
492 + item: Any,
493 +) -> tuple[list[Image.Image], list[float]]:
494 + """Convert a ``(frames_ndarray, metadata)`` video item into
495 + ``(pil_frames, timestamps_seconds)``.
496 +
497 + Both real ``video_url`` inputs (decoded + sampled by the registered
498 + ``LlavaOnevision2VideoBackend``) and dummy profiling videos arrive here as a
499 + ``(frames, metadata)`` tuple because the data parser runs with
500 + ``video_needs_metadata=True``. ``frames`` is a ``(T, H, W, C)`` uint8 array;
501 + ``metadata`` carries ``frames_indices`` and the source ``fps``.
502 +
503 + Timestamps follow the qwen_vl_utils policy: ``frame_index / original_fps``.
504 + The frame count is padded up to ``_TEMPORAL_MERGE_SIZE`` (repeating the last
505 + frame) because OV2's vision tower merges frames temporally in pairs.
506 + """
507 + if not (isinstance(item, (tuple, list)) and len(item) == 2):
508 + raise ValueError(
509 + "LlavaOnevision2 frame backend expects each video as a "
510 + f"(frames_ndarray, metadata) tuple; got {type(item).__name__}. "
511 + "Pass videos via `video_url` so the registered backend can decode "
512 + "and sample them."
513 + )
514 + frames, metadata = item
515 + if isinstance(frames, torch.Tensor):
516 + frames_np = frames.cpu().numpy()
517 + else:
518 + frames_np = np.asarray(frames)
519 +
520 + pil_frames = [Image.fromarray(f.astype(np.uint8)) for f in frames_np]
521 +
522 + indices = metadata.get("frames_indices") if isinstance(metadata, Mapping) else None
523 + if indices is None:
524 + indices = list(range(len(pil_frames)))
525 + elif not isinstance(indices, list):
526 + indices = list(indices)
527 + # Keep indices aligned with the actual frame count.
528 + if len(indices) != len(pil_frames):
529 + if len(indices) > len(pil_frames):
530 + indices = indices[: len(pil_frames)]
531 + else:
532 + indices = list(indices) + [indices[-1] if indices else 0] * (
533 + len(pil_frames) - len(indices)
534 + )
535 +
536 + fps = _DEFAULT_FPS
537 + if isinstance(metadata, Mapping) and metadata.get("fps"):
538 + fps = float(metadata["fps"])
539 + if fps <= 0:
540 + fps = _DEFAULT_FPS
541 +
542 + # OV2 vision tower: temporal merge=2 -> frame count must be even.
543 + if len(pil_frames) % _TEMPORAL_MERGE_SIZE != 0:
544 + pad = _TEMPORAL_MERGE_SIZE - len(pil_frames) % _TEMPORAL_MERGE_SIZE
545 + pil_frames = pil_frames + [pil_frames[-1]] * pad
546 + indices = indices + [indices[-1]] * pad
547 +
548 + timestamps = [idx / fps for idx in indices]
549 + return pil_frames, timestamps
550 +
551 +
552 +def _expand_video_markers_in_prompt(
553 + prompt: str,
554 + per_video_timestamps: list[list[float]],
555 + *,
556 + timestamp_decimals: int,
557 +) -> str:
558 + """Replace each ``<|vision_start|><|video_pad|><|vision_end|>`` with a
559 + sequence of ``<{t:.Nf} seconds><|vision_start|><|image_pad|><|vision_end|>``
560 + blocks -- one per frame -- matching ``vllm_hf_chat._build_prompt``.
561 +
562 + Replacement is positional: the *i*-th marker consumes
563 + ``per_video_timestamps[i]``.
564 + """
565 + parts: list[str] = []
566 + cursor = 0
567 + idx = 0
568 + pattern = re.escape(_VIDEO_MARKER)
569 + for m in re.finditer(pattern, prompt):
570 + parts.append(prompt[cursor : m.start()])
571 + if idx >= len(per_video_timestamps):
572 + raise ValueError(
573 + f"Prompt has more video markers than supplied timestamp "
574 + f"groups ({len(per_video_timestamps)})"
575 + )
576 + timestamps = per_video_timestamps[idx]
577 + expanded = "".join(
578 + f"<{t:.{timestamp_decimals}f} seconds>{_IMAGE_MARKER}" for t in timestamps
579 + )
580 + parts.append(expanded)
581 + cursor = m.end()
582 + idx += 1
583 + parts.append(prompt[cursor:])
584 + if idx != len(per_video_timestamps):
585 + raise ValueError(
586 + f"Prompt has {idx} video markers but {len(per_video_timestamps)} "
587 + f"timestamp groups were supplied"
588 + )
589 + return "".join(parts)
590 +
591 +
592 +# ---------------------------------------------------------------------------
593 +# Registered video loader backend (frame sampling)
594 +# ---------------------------------------------------------------------------
595 +# OV2 frame-sampling policy, expressed as a vLLM ``VideoBackend`` so videos
596 +# entering through the standard ``video_url`` -> MediaConnector -> VideoMediaIO
597 +# path are decoded + sampled inside vLLM (no qwen_vl_utils dependency, and the
598 +# connector's SSRF / local-file gates apply automatically).
599 +#
600 +# Parity note: ``compute_frames_index_to_sample`` replicates
601 +# ``qwen_vl_utils.smart_nframes`` (the policy the OV2 hf-chat recipe validated)
602 +# -- frame count and index selection are byte-identical to qwen. The one
603 +# residual difference is the source frame *count*: qwen decodes via decord
604 +# whereas vLLM uses OpenCV/PyAV, whose ``CAP_PROP_FRAME_COUNT`` (a duration x
605 +# fps estimate) can differ by +/-1 frame on some containers, shifting sampled
606 +# indices by one frame on those files. Downstream metrics stay within noise.
607 +_OV2_FRAME_FACTOR = 2
608 +_OV2_FPS_MIN_FRAMES = 4
609 +
610 +
611 +def _round_by_factor(n: float, factor: int) -> int:
612 + return round(n / factor) * factor
613 +
614 +
615 +def _ceil_by_factor(n: float, factor: int) -> int:
616 + import math as _math
617 +
618 + return _math.ceil(n / factor) * factor
619 +
620 +
621 +def _floor_by_factor(n: float, factor: int) -> int:
622 + import math as _math
623 +
624 + return _math.floor(n / factor) * factor
625 +
626 +
627 +def _ov2_smart_nframes(
628 + total_frames: int,
629 + video_fps: float,
630 + *,
631 + fps: float,
632 + min_frames: int,
633 + max_frames: int,
634 +) -> int:
635 + """Replicate ``qwen_vl_utils.smart_nframes`` (fps branch).
636 +
637 + Returns an even frame count in ``[min_frames, min(max_frames, total)]``.
638 + """
639 + min_frames = _ceil_by_factor(min_frames, _OV2_FRAME_FACTOR)
640 + max_frames = _floor_by_factor(max_frames, _OV2_FRAME_FACTOR)
641 + nframes = total_frames / video_fps * fps if video_fps > 0 else total_frames
642 + nframes = min(min(max(nframes, min_frames), max_frames), total_frames)
643 + nframes = _floor_by_factor(nframes, _OV2_FRAME_FACTOR)
644 + return max(int(nframes), _OV2_FRAME_FACTOR)
645 +
646 +
647 +@VIDEO_LOADER_REGISTRY.register(
648 + "llava_onevision2",
649 + video_processor="LlavaOnevision2VideoProcessor",
650 +)
651 +class LlavaOnevision2VideoBackend(VideoBackend):
652 + """Frame-sampling backend for LLaVA-OneVision-2.
653 +
654 + Selected automatically for OV2 via the ``video_processor`` binding
655 + (``video_processor_type == "LlavaOnevision2VideoProcessor"`` in the model's
656 + ``video_preprocessor_config.json``). Decoding uses the inherited OpenCV /
657 + PyAV codecs; only the sampling index policy is overridden to match qwen.
658 + """
659 +
660 + _sampling_suffix = "_llava_onevision2"
661 +
662 + # OV2 hf-chat reference sampling constants (mirror the validated adapter).
663 + _FPS = _DEFAULT_FPS
664 + _MAX_FRAMES = _DEFAULT_MAX_FRAMES
665 + _MIN_FRAMES = _OV2_FPS_MIN_FRAMES
666 +
667 + @classmethod
668 + def compute_frames_index_to_sample(
669 + cls,
670 + source: VideoSourceMetadata,
671 + target: VideoTargetMetadata,
672 + **kwargs,
673 + ) -> list[int]:
674 + total = int(source.total_frames_num)
675 + if total <= 0:
676 + return []
677 + video_fps = float(source.original_fps)
678 + # Honor caller-provided sampling targets (via ``--media-io-kwargs`` →
679 + # ``VideoTargetMetadata``) so benchmarks can override the conservative
680 + # defaults (e.g. VSI-Bench needs max_frames=128). Fall back to the OV2
681 + # hf-chat reference constants when the target leaves a field unset
682 + # (sentinel ``<= 0``).
683 + target_fps = float(target.fps) if target.fps > 0 else cls._FPS
684 + target_max_frames = (
685 + int(target.num_frames) if target.num_frames > 0 else cls._MAX_FRAMES
686 + )
687 + n = _ov2_smart_nframes(
688 + total,
689 + video_fps,
690 + fps=target_fps,
691 + min_frames=cls._MIN_FRAMES,
692 + max_frames=target_max_frames,
693 + )
694 + # qwen uses linspace().round() (NOT the floor cast used by the base
695 + # uniform backend), so replicate the rounding exactly.
696 + idx = np.linspace(0, total - 1, n).round().astype(int).tolist()
697 + # smart_nframes floors to FRAME_FACTOR so ``n`` is even; guard anyway
698 + # since OV2's vision tower (temporal merge = 2) requires even counts.
699 + if len(idx) % _OV2_FRAME_FACTOR != 0:
700 + idx.append(idx[-1])
701 + return idx
702 +
703 +
704 +class LlavaOnevision2ImagePixelInputs(TensorSchema):
705 + type: Literal["pixel_values"]
706 +
707 + pixel_values: Annotated[torch.Tensor, TensorShape("np", "cps")]
708 + image_grid_thw: Annotated[torch.Tensor, TensorShape("ni", 3)]
709 + patch_positions: Annotated[torch.Tensor, TensorShape("np", 3)]
710 +
711 +
712 +class LlavaOnevision2ImageEmbeddingInputs(TensorSchema):
713 + type: Literal["image_embeds"]
714 +
715 + image_embeds: Annotated[torch.Tensor, TensorShape("nf", "hs")]
716 + image_grid_thw: Annotated[torch.Tensor, TensorShape("ni", 3)]
717 +
718 +
719 +class LlavaOnevision2VideoPixelInputs(TensorSchema):
720 + type: Literal["pixel_values_videos"]
721 +
722 + pixel_values_videos: Annotated[torch.Tensor, TensorShape("np", "cps")]
723 + video_grid_thw: Annotated[torch.Tensor, TensorShape("nf", 3)]
724 + patch_positions_videos: Annotated[torch.Tensor, TensorShape("np", 3)]
725 + video_num_frames: Annotated[torch.Tensor, TensorShape("nv")]
726 +
727 +
728 +LlavaOnevision2ImageInputs = (
729 + LlavaOnevision2ImagePixelInputs | LlavaOnevision2ImageEmbeddingInputs
730 +)
731 +
732 +
733 +class LlavaOnevision2VisionRotaryEmbedding(nn.Module):
734 + """3-D rotary frequency constructor with 4:6:6 (T:H:W) split.
735 +
736 + Mirrors ``VisionRotaryEmbedding`` in the HF reference
737 + (``modeling_llava_onevision2.py`` L79-L210). The three ``inv_freq_*``
738 + buffers are non-persistent — they are *not* in the checkpoint and must
739 + be reconstructed at module init time (which we do here).
740 +
741 + Public entry points used by the vision tower:
742 + * ``forward_from_positions(patch_positions)`` — per-patch (t,h,w)
743 + positions → per-token freqs [N, half].
744 + """
745 +
746 + def __init__(self, head_dim: int, theta: float = 10000.0) -> None:
747 + super().__init__()
748 + assert head_dim % 2 == 0, "head_dim must be even"
749 + assert head_dim % 16 == 0, "head_dim must be divisible by 16 (4:6:6)"
750 + half = head_dim // 2
751 + assert half % 16 == 0, "head_dim//2 must be divisible by 16"
752 +
753 + self.head_dim = head_dim
754 + self.half = half
755 + self.base = float(theta)
756 +
757 + unit = half // 16
758 + self.t_size = 4 * unit
759 + self.h_size = 6 * unit
760 + self.w_size = 6 * unit
761 + assert self.t_size + self.h_size + self.w_size == half
762 +
763 + self.register_buffer(
764 + "inv_freq_t",
765 + 1.0
766 + / (
767 + self.base
768 + ** (torch.arange(self.t_size, dtype=torch.float32) / self.t_size)
769 + ),
770 + persistent=False,
771 + )
772 + self.register_buffer(
773 + "inv_freq_h",
774 + 1.0
775 + / (
776 + self.base
777 + ** (torch.arange(self.h_size, dtype=torch.float32) / self.h_size)
778 + ),
779 + persistent=False,
780 + )
781 + self.register_buffer(
782 + "inv_freq_w",
783 + 1.0
784 + / (
785 + self.base
786 + ** (torch.arange(self.w_size, dtype=torch.float32) / self.w_size)
787 + ),
788 + persistent=False,
789 + )
790 +
791 + def forward_from_positions(self, patch_positions: torch.Tensor) -> torch.Tensor:
792 + """[N, 3] (t,h,w) int → [N, half] float frequencies."""
793 + device = patch_positions.device
794 + inv_t = self.inv_freq_t.to(device=device)
795 + inv_h = self.inv_freq_h.to(device=device)
796 + inv_w = self.inv_freq_w.to(device=device)
797 +
798 + t_pos = patch_positions[:, 0].float()
799 + h_pos = patch_positions[:, 1].float()
800 + w_pos = patch_positions[:, 2].float()
801 +
802 + ft = torch.outer(t_pos, inv_t)
803 + fh = torch.outer(h_pos, inv_h)
804 + fw = torch.outer(w_pos, inv_w)
805 + return torch.cat([ft, fh, fw], dim=-1)
806 +
807 +
808 +class LlavaOnevision2VisionEmbeddings(nn.Module):
809 + def __init__(
810 + self, patch_size: int = 14, in_channels: int = 3, embed_dim: int = 1024
811 + ) -> None:
812 + super().__init__()
813 + self.patch_size = patch_size
814 + self.in_channels = in_channels
815 + self.embed_dim = embed_dim
816 + self.patch_embedding = nn.Conv2d(
817 + in_channels,
818 + embed_dim,
819 + kernel_size=(patch_size, patch_size),
820 + stride=(patch_size, patch_size),
821 + bias=False,
822 + )
823 +
824 + def forward(self, x: torch.Tensor) -> torch.Tensor:
825 + x = x.view(-1, self.in_channels, self.patch_size, self.patch_size)
826 + x = self.patch_embedding(x).view(-1, self.embed_dim)
827 + return x
828 +
829 +
830 +class LlavaOnevision2VisionMLP(nn.Module):
831 + def __init__(
832 + self,
833 + in_features: int,
834 + hidden_features: int,
835 + bias: bool = True,
836 + quant_config: QuantizationConfig | None = None,
837 + prefix: str = "",
838 + use_data_parallel: bool = False,
839 + ) -> None:
840 + super().__init__()
841 + if quant_config is not None:
842 + raise RuntimeError("LLaVAOneVision2 does not support quantization")
843 + self.fc1 = ColumnParallelLinear(
844 + in_features,
845 + hidden_features,
846 + bias=bias,
847 + quant_config=quant_config,
848 + prefix=f"{prefix}.fc1",
849 + disable_tp=use_data_parallel,
850 + )
851 + self.fc2 = RowParallelLinear(
852 + hidden_features,
853 + in_features,
854 + bias=bias,
855 + quant_config=quant_config,
856 + prefix=f"{prefix}.fc2",
857 + disable_tp=use_data_parallel,
858 + )
859 + self.act_fn = F.gelu
860 +
861 + def forward(self, x: torch.Tensor) -> torch.Tensor:
862 + x, _ = self.fc1(x)
863 + x = self.act_fn(x)
864 + x, _ = self.fc2(x)
865 + return x
866 +
867 +
868 +class LlavaOnevision2VisionAttn(nn.Module):
869 + """Vision self-attention with windowed cu_seqlens.
870 +
871 + The HF checkpoint ships a *fused* qkv linear (``self_attn.qkv``), so
872 + we load directly into ``QKVParallelLinear`` with no stacked_params
873 + mapping. (Compare OV1.5, whose checkpoint had separate q/k/v.)
874 + """
875 +
876 + def __init__(
877 + self,
878 + embed_dim: int,
879 + num_heads: int,
880 + projection_size: int,
881 + quant_config: QuantizationConfig | None = None,
882 + prefix: str = "",
883 + use_data_parallel: bool = False,
884 + ) -> None:
885 + super().__init__()
886 + if quant_config is not None:
887 + raise RuntimeError("LLaVAOneVision2 does not support quantization")
888 +
889 + self.tp_size = (
890 + 1
891 + if use_data_parallel
892 + else parallel_state.get_tensor_model_parallel_world_size()
893 + )
894 + self.tp_rank = parallel_state.get_tensor_model_parallel_rank()
895 + self.num_heads = num_heads
896 + self.hidden_size_per_attn_head = dist_utils.divide(projection_size, num_heads)
897 + self.num_attn_heads_per_partition = dist_utils.divide(num_heads, self.tp_size)
898 +
899 + self.qkv = QKVParallelLinear(
900 + hidden_size=embed_dim,
901 + head_size=self.hidden_size_per_attn_head,
902 + total_num_heads=num_heads,
903 + total_num_kv_heads=num_heads,
904 + bias=True,
905 + quant_config=quant_config,
906 + prefix=f"{prefix}.qkv",
907 + disable_tp=use_data_parallel,
908 + )
909 +
910 + self.proj = RowParallelLinear(
911 + input_size=projection_size,
912 + output_size=embed_dim,
913 + quant_config=quant_config,
914 + prefix=f"{prefix}.proj",
915 + disable_tp=use_data_parallel,
916 + )
917 +
918 + self.attn = MMEncoderAttention(
919 + num_heads=self.num_attn_heads_per_partition,
920 + head_size=self.hidden_size_per_attn_head,
921 + scale=self.hidden_size_per_attn_head**-0.5,
922 + prefix=f"{prefix}.attn",
923 + )
924 +
925 + @staticmethod
926 + def _rotate_half_interleaved(x: torch.Tensor) -> torch.Tensor:
927 + """OV2-specific interleaved rotate_half.
928 +
929 + Pairs adjacent dims: (x[::2], x[1::2]) -> (-x[1::2], x[::2]).
930 + NOT compatible with the split-half rotate used in OV1.5/LLaMA.
931 + """
932 + x_even = x[..., 0::2]
933 + x_odd = x[..., 1::2]
934 + out = torch.stack((-x_odd, x_even), dim=-1)
935 + return out.flatten(-2)
936 +
937 + def _apply_rotary_pos_embed(
938 + self, t: torch.Tensor, freqs: torch.Tensor
939 + ) -> torch.Tensor:
940 + # freqs is [seq_len, half]; cat([f,f],-1) pair-repeat layout
941 + # matches the interleaved rotate above.
942 + orig_dtype = t.dtype
943 + t = t.float()
944 + emb = torch.cat((freqs, freqs), dim=-1)
945 + cos = emb.cos().unsqueeze(-2).float()
946 + sin = emb.sin().unsqueeze(-2).float()
947 + t = (t * cos) + (self._rotate_half_interleaved(t) * sin)
948 + return t.to(orig_dtype)
949 +
950 + def forward(
951 + self,
952 + x: torch.Tensor,
953 + cu_seqlens: torch.Tensor,
954 + rotary_pos_emb: torch.Tensor,
955 + max_seqlen: torch.Tensor | None = None,
956 + sequence_lengths: torch.Tensor | None = None,
957 + ) -> torch.Tensor:
958 + x, _ = self.qkv(x)
959 + seq_len = x.shape[0]
960 + # QKVParallelLinear packs q/k/v along the last dim as
961 + # [q_heads, k_heads, v_heads]; view splits the three sections, each
962 + # holding this partition's heads (no all-gather: MMEncoderAttention
963 + # runs per-partition and ``proj`` reduces across TP ranks).
964 + qkv = x.view(
965 + seq_len,
966 + 3,
967 + self.num_attn_heads_per_partition,
968 + self.hidden_size_per_attn_head,
969 + )
970 + q, k, v = qkv.unbind(1)
971 +
972 + if rotary_pos_emb is not None:
973 + # OV2 uses interleaved RoPE on the raw freqs, applied outside
974 + # MMEncoderAttention (which is rotary-agnostic).
975 + q = self._apply_rotary_pos_embed(q, rotary_pos_emb)
976 + k = self._apply_rotary_pos_embed(k, rotary_pos_emb)
977 +
978 + # Add a leading batch dim (b=1) for MMEncoderAttention, which expects
979 + # (batch, seq_len, num_heads, head_size) and windows via cu_seqlens.
980 + output = self.attn(
981 + query=q.unsqueeze(0),
982 + key=k.unsqueeze(0),
983 + value=v.unsqueeze(0),
984 + cu_seqlens=cu_seqlens,
985 + max_seqlen=max_seqlen,
986 + sequence_lengths=sequence_lengths,
987 + )
988 + output = output.reshape(seq_len, -1)
989 +
990 + output, _ = self.proj(output)
991 + return output
992 +
993 +
994 +@support_torch_compile(
995 + dynamic_arg_dims={
996 + "x": 0,
997 + "cu_seqlens": 0,
998 + "rotary_pos_emb": 0,
999 + "sequence_lengths": 0,
1000 + },
1001 + enable_if=should_torch_compile_mm_encoder,
1002 + is_encoder=True,
1003 +)
1004 +class LlavaOnevision2VisionTowerBlock(nn.Module):
1005 + def __init__(
1006 + self,
1007 + dim: int,
1008 + num_heads: int,
1009 + mlp_hidden_dim: int,
1010 + norm_eps: float = 1e-6,
1011 + quant_config: QuantizationConfig | None = None,
1012 + prefix: str = "",
1013 + use_data_parallel: bool = False,
1014 + ) -> None:
1015 + super().__init__()
1016 + self.layer_norm1 = nn.LayerNorm(dim, eps=norm_eps)
1017 + self.layer_norm2 = nn.LayerNorm(dim, eps=norm_eps)
1018 + self.self_attn = LlavaOnevision2VisionAttn(
1019 + embed_dim=dim,
1020 + num_heads=num_heads,
1021 + projection_size=dim,
1022 + quant_config=quant_config,
1023 + prefix=f"{prefix}.self_attn",
1024 + use_data_parallel=use_data_parallel,
1025 + )
1026 + self.mlp = LlavaOnevision2VisionMLP(
1027 + dim,
1028 + mlp_hidden_dim,
1029 + bias=True,
1030 + quant_config=quant_config,
1031 + prefix=f"{prefix}.mlp",
1032 + use_data_parallel=use_data_parallel,
1033 + )
1034 +
1035 + def forward(
1036 + self,
1037 + x: torch.Tensor,
1038 + cu_seqlens: torch.Tensor,
1039 + rotary_pos_emb: torch.Tensor,
1040 + max_seqlen: torch.Tensor | None = None,
1041 + sequence_lengths: torch.Tensor | None = None,
1042 + ) -> torch.Tensor:
1043 + x = x + self.self_attn(
1044 + self.layer_norm1(x),
1045 + cu_seqlens=cu_seqlens,
1046 + rotary_pos_emb=rotary_pos_emb,
1047 + max_seqlen=max_seqlen,
1048 + sequence_lengths=sequence_lengths,
1049 + )
1050 + x = x + self.mlp(self.layer_norm2(x))
1051 + return x
1052 +
1053 +
1054 +class LlavaOnevision2PatchMerger(nn.Module):
1055 + def __init__(
1056 + self,
1057 + d_model: int,
1058 + context_dim: int,
1059 + spatial_merge_size: int = 2,
1060 + norm_eps: float = 1e-6,
1061 + quant_config: QuantizationConfig | None = None,
1062 + prefix: str = "",
1063 + use_data_parallel: bool = False,
1064 + ) -> None:
1065 + super().__init__()
1066 + self.hidden_size = context_dim * (spatial_merge_size**2)
1067 + self.ln_q = nn.LayerNorm(context_dim, eps=norm_eps)
1068 + self.mlp = nn.ModuleList(
1069 + [
1070 + ColumnParallelLinear(
1071 + self.hidden_size,
1072 + self.hidden_size,
1073 + bias=True,
1074 + quant_config=quant_config,
1075 + prefix=f"{prefix}.mlp.0",
1076 + disable_tp=use_data_parallel,
1077 + ),
1078 + nn.GELU(),
1079 + RowParallelLinear(
1080 + self.hidden_size,
1081 + d_model,
1082 + bias=True,
1083 + quant_config=quant_config,
1084 + prefix=f"{prefix}.mlp.2",
1085 + disable_tp=use_data_parallel,
1086 + ),
1087 + ]
1088 + )
1089 +
1090 + def forward(
1091 + self, x: torch.Tensor, patch_positions: torch.Tensor | None = None
1092 + ) -> torch.Tensor:
1093 + # patch_positions accepted for API symmetry with the HF impl,
1094 + # but unused: pixel_values already arrive in spatial-merge block
1095 + # order (Qwen2VLImageProcessor handles that across image / video-
1096 + # frames / video-codec backends). See codec_video_processing for
1097 + # the codec backend's ``codec_positions_for_processor`` call.
1098 + del patch_positions
1099 + x = self.ln_q(x)
1100 + x = x.view(-1, self.hidden_size)
1101 + fc1, act, fc2 = self.mlp
1102 + x, _ = fc1(x)
1103 + x = act(x)
1104 + x, _ = fc2(x)
1105 + return x
1106 +
1107 +
1108 +class LlavaOnevision2VisionTower(nn.Module):
1109 + """OV2 vision tower (no CLS token, 3-D RoPE, windowed attention).
1110 +
1111 + Module attribute names mirror HF checkpoint names verbatim so the
1112 + WeightsMapper only needs prefix rewrites (no substring rules, which would
1113 + otherwise collide with the Qwen3 text-path ``self_attn`` modules):
1114 + visual.embeddings.patch_embedding
1115 + visual.layernorm_pre
1116 + visual.encoder.layers.{i}.self_attn.{qkv,proj}
1117 + visual.encoder.layers.{i}.layer_norm{1,2}
1118 + visual.encoder.layers.{i}.mlp.fc{1,2}
1119 + visual.merger.{ln_q, mlp.{0,2}}
1120 + visual.rotary_pos_emb (non-persistent inv_freq buffers)
1121 + """
1122 +
1123 + def __init__(
1124 + self,
1125 + vision_config,
1126 + text_hidden_size: int,
1127 + norm_eps: float = 1e-6,
1128 + quant_config: QuantizationConfig | None = None,
1129 + prefix: str = "",
1130 + use_data_parallel: bool = False,
1131 + ) -> None:
1132 + super().__init__()
1133 + if quant_config is not None:
1134 + raise RuntimeError("LLaVAOneVision2 does not support quantization")
1135 +
1136 + patch_size = vision_config.patch_size
1137 + spatial_merge_size = vision_config.spatial_merge_size
1138 + in_channels = getattr(vision_config, "num_channels", 3)
1139 + hidden_size = vision_config.hidden_size
1140 + embed_dim = hidden_size
1141 + depth = vision_config.num_hidden_layers
1142 + num_heads = vision_config.num_attention_heads
1143 + mlp_hidden_dim = vision_config.intermediate_size
1144 + frame_windows_size = getattr(vision_config, "frame_windows_size", 4)
1145 + rope_theta = getattr(vision_config, "rope_theta", 10000.0)
1146 +
1147 + self.spatial_merge_size = spatial_merge_size
1148 + self.frame_windows_size = int(frame_windows_size)
1149 + self.num_heads = num_heads
1150 + self.embed_dim = embed_dim
1151 + self.head_dim = embed_dim // num_heads
1152 + self.use_data_parallel = use_data_parallel
1153 + self.tp_size = (
1154 + 1
1155 + if use_data_parallel
1156 + else parallel_state.get_tensor_model_parallel_world_size()
1157 + )
1158 +
1159 + self.embeddings = LlavaOnevision2VisionEmbeddings(
1160 + patch_size=patch_size, in_channels=in_channels, embed_dim=embed_dim
1161 + )
1162 + self.layernorm_pre = nn.LayerNorm(embed_dim, eps=norm_eps)
1163 +
1164 + self.rotary_pos_emb = LlavaOnevision2VisionRotaryEmbedding(
1165 + self.head_dim, theta=rope_theta
1166 + )
1167 +
1168 + self.encoder = nn.Module()
1169 + self.encoder.layers = nn.ModuleList(
1170 + [
1171 + LlavaOnevision2VisionTowerBlock(
1172 + dim=embed_dim,
1173 + num_heads=num_heads,
1174 + mlp_hidden_dim=mlp_hidden_dim,
1175 + norm_eps=norm_eps,
1176 + quant_config=quant_config,
1177 + prefix=f"{prefix}.encoder.layers.{i}",
1178 + use_data_parallel=use_data_parallel,
1179 + )
1180 + for i in range(depth)
1181 + ]
1182 + )
1183 +
1184 + self.merger = LlavaOnevision2PatchMerger(
1185 + d_model=text_hidden_size,
1186 + context_dim=embed_dim,
1187 + spatial_merge_size=spatial_merge_size,
1188 + norm_eps=norm_eps,
1189 + quant_config=quant_config,
1190 + prefix=f"{prefix}.merger",
1191 + use_data_parallel=use_data_parallel,
1192 + )
1193 +
1194 + # Vision attention backend; mirrors the one MMEncoderAttention picks
1195 + # internally so the cu_seqlens / max_seqlen metadata computed below
1196 + # matches the kernel actually used.
1197 + self.attn_backend = get_vit_attn_backend(
1198 + head_size=self.head_dim, dtype=torch.get_default_dtype()
1199 + )
1200 +
1201 + @property
1202 + def dtype(self) -> torch.dtype:
1203 + return self.embeddings.patch_embedding.weight.dtype
1204 +
1205 + @property
1206 + def device(self) -> torch.device:
1207 + return self.embeddings.patch_embedding.weight.device
1208 +
1209 + def _build_window_cu_seqlens(self, grid_thw: torch.Tensor) -> np.ndarray:
1210 + """Build cu_seqlens that chunk each sample's T-axis into windows of
1211 + ``frame_windows_size`` frames.
1212 +
1213 + Returns an int32 ``np.ndarray`` of shape [num_windows+1] (the
1214 + canonical prefix-sum format). Backend-specific transforms are applied
1215 + afterwards via ``MMEncoderAttention.maybe_recompute_cu_seqlens``.
1216 + """
1217 + win = self.frame_windows_size
1218 + chunk_lengths: list[int] = []
1219 + for row in grid_thw.tolist():
1220 + t, h, w = int(row[0]), int(row[1]), int(row[2])
1221 + per_frame = h * w
1222 + t_remaining = t
1223 + while t_remaining > 0:
1224 + this_t = min(win, t_remaining)
1225 + chunk_lengths.append(this_t * per_frame)
1226 + t_remaining -= this_t
1227 + cu = np.concatenate(
1228 + [
1229 + np.zeros(1, dtype=np.int32),
1230 + np.array(chunk_lengths, dtype=np.int32).cumsum(dtype=np.int32),
1231 + ]
1232 + )
1233 + return cu
1234 +
1235 + def forward(
1236 + self,
1237 + pixel_values: torch.Tensor,
1238 + grid_thw: torch.Tensor,
1239 + patch_positions: torch.Tensor,
1240 + ) -> torch.Tensor:
1241 + x = pixel_values.to(device=self.device, dtype=self.dtype)
1242 + x = self.embeddings(x)
1243 + x = self.layernorm_pre(x)
1244 +
1245 + rotary_pos_emb = self.rotary_pos_emb.forward_from_positions(
1246 + patch_positions.to(self.device)
1247 + )
1248 +
1249 + # Build window cu_seqlens, then derive backend-specific attention
1250 + # metadata (passthrough for FA/SDPA; transformed for FlashInfer).
1251 + cu_seqlens_np = self._build_window_cu_seqlens(grid_thw)
1252 + sequence_lengths = MMEncoderAttention.maybe_compute_seq_lens(
1253 + self.attn_backend, cu_seqlens_np, self.device
1254 + )
1255 + max_seqlen = torch.tensor(
1256 + MMEncoderAttention.compute_max_seqlen(self.attn_backend, cu_seqlens_np),
1257 + dtype=torch.int32,
1258 + )
1259 + cu_seqlens = MMEncoderAttention.maybe_recompute_cu_seqlens(
1260 + self.attn_backend,
1261 + cu_seqlens_np,
1262 + self.embed_dim,
1263 + self.tp_size,
1264 + self.device,
1265 + )
1266 +
1267 + for blk in self.encoder.layers:
1268 + x = blk(
1269 + x,
1270 + cu_seqlens=cu_seqlens,
1271 + rotary_pos_emb=rotary_pos_emb,
1272 + max_seqlen=max_seqlen,
1273 + sequence_lengths=sequence_lengths,
1274 + )
1275 +
1276 + return self.merger(x, patch_positions=patch_positions)
1277 +
1278 +
1279 +class LlavaOnevision2ProcessingInfo(BaseProcessingInfo):
1280 + def get_hf_config(self):
1281 + return self.ctx.get_hf_config()
1282 +
1283 + def get_data_parser(self) -> MultiModalDataParser:
1284 + # ``video_needs_metadata=True`` makes the parser preserve both the
1285 + # ``(frames, metadata)`` tuples from the frame backend and the
1286 + # ``(dummy, {marker: path})`` tuples from prepare_codec_video_input;
1287 + # both are dispatched by metadata content in ``_call_hf_processor``.
1288 + return LlavaOnevision2MultiModalDataParser(
1289 + self.get_hf_config().vision_config.spatial_merge_size,
1290 + video_needs_metadata=True,
1291 + )
1292 +
1293 + def get_hf_processor(self, **kwargs: object):
1294 + # OV2's trust_remote_code processor is a bare class (not a
1295 + # ProcessorMixin), so the shared get_hf_processor cannot load it; load
1296 + # via AutoProcessor here (as other trust_remote_code models do).
1297 + model_config = self.ctx.model_config
1298 + # ``_merge_mm_kwargs`` restricts caller ``mm_processor_kwargs`` to known
1299 + # processor args and wraps values as hashable for the lru_cache.
1300 + merged = _merge_mm_kwargs(model_config, AutoProcessor, **kwargs)
1301 + merged.setdefault("use_fast", True)
1302 + return _load_ov2_processor(
1303 + model_config.model,
1304 + model_config.revision,
1305 + model_config.trust_remote_code,
1306 + **merged,
1307 + )
1308 +
1309 + def get_image_processor(self, **kwargs: object) -> Qwen2VLImageProcessor:
1310 + return self.get_hf_processor(**kwargs).image_processor
1311 +
1312 + def get_mm_max_tokens_per_item(
1313 + self,
1314 + seq_len: int,
1315 + mm_counts: Mapping[str, int],
1316 + ) -> Mapping[str, int]:
1317 + return {
1318 + "image": self.get_max_image_tokens(),
1319 + "video": self.get_max_video_tokens(seq_len, mm_counts),
1320 + }
1321 +
1322 + def _get_vision_info(
1323 + self,
1324 + *,
1325 + image_width: int,
1326 + image_height: int,
1327 + num_frames: int = 1,
1328 + do_resize: bool = True,
1329 + image_processor: Qwen2VLImageProcessor | None,
1330 + ) -> tuple[ImageSize, int]:
1331 + if image_processor is None:
1332 + image_processor = self.get_image_processor()
1333 + hf_config = self.get_hf_config()
1334 + vision_config = hf_config.vision_config
1335 + patch_size = vision_config.patch_size
1336 + merge_size = vision_config.spatial_merge_size
1337 + temporal_patch_size = getattr(vision_config, "temporal_patch_size", 1)
1338 + if do_resize:
1339 + min_pixels = getattr(image_processor, "min_pixels", None)
1340 + max_pixels = getattr(image_processor, "max_pixels", None)
1341 + if min_pixels is None or max_pixels is None:
1342 + size = image_processor.size
1343 + min_pixels = (
1344 + getattr(size, "shortest_edge", None) or size["shortest_edge"]
1345 + )
1346 + max_pixels = getattr(size, "longest_edge", None) or size["longest_edge"]
1347 + rh, rw = smart_resize(
1348 + height=image_height,
1349 + width=image_width,
1350 + factor=patch_size * merge_size,
1351 + min_pixels=min_pixels,
1352 + max_pixels=max_pixels,
1353 + )
1354 + preprocessed = ImageSize(width=rw, height=rh)
1355 + else:
1356 + preprocessed = ImageSize(width=image_width, height=image_height)
1357 + padded_frames = num_frames + num_frames % temporal_patch_size
1358 + grid_t = max(padded_frames // temporal_patch_size, 1)
1359 + grid_h = preprocessed.height // patch_size
1360 + grid_w = preprocessed.width // patch_size
1361 + num_patches = grid_t * grid_h * grid_w
1362 + return preprocessed, num_patches // (merge_size**2)
1363 +
1364 + def get_num_image_tokens(
1365 + self,
1366 + *,
1367 + image_width: int,
1368 + image_height: int,
1369 + image_processor: Qwen2VLImageProcessor | None,
1370 + ) -> int:
1371 + _, n = self._get_vision_info(
1372 + image_width=image_width,
1373 + image_height=image_height,
1374 + image_processor=image_processor,
1375 + )
1376 + return n
1377 +
1378 + def get_image_size_with_most_features(self) -> ImageSize:
1379 + sz, _ = self._get_vision_info(
1380 + image_width=1800, image_height=1800, image_processor=None
1381 + )
1382 + return sz
1383 +
1384 + def get_max_image_tokens(self) -> int:
1385 + w, h = self.get_image_size_with_most_features()
1386 + return self.get_num_image_tokens(
1387 + image_width=w, image_height=h, image_processor=None
1388 + )
1389 +
1390 + def get_num_video_tokens(
1391 + self,
1392 + *,
1393 + image_width: int,
1394 + image_height: int,
1395 + num_frames: int,
1396 + image_processor: Qwen2VLImageProcessor | None = None,
1397 + ) -> int:
1398 + _, n = self._get_vision_info(
1399 + image_width=image_width,
1400 + image_height=image_height,
1401 + num_frames=num_frames,
1402 + image_processor=image_processor,
1403 + )
1404 + return n
1405 +
1406 + def _get_max_video_frames(self, max_tokens: int, start_num_frames: int = 1) -> int:
1407 + w, h = self.get_image_size_with_most_features()
1408 + num_frames = start_num_frames
1409 + while True:
1410 + next_num_frames = num_frames + 1
1411 + next_max_tokens = self.get_num_video_tokens(
1412 + image_width=w,
1413 + image_height=h,
1414 + num_frames=next_num_frames,
1415 + )
1416 + if next_max_tokens > max_tokens:
1417 + break
1418 + num_frames = next_num_frames
1419 + return num_frames
1420 +
1421 + def get_num_frames_with_most_features(
1422 + self,
1423 + seq_len: int,
1424 + mm_counts: Mapping[str, int],
1425 + max_frames_per_video: int = _MAX_FRAMES_PER_VIDEO,
1426 + ) -> int:
1427 + max_videos = mm_counts.get("video", 0)
1428 + max_total_frames = self._get_max_video_frames(seq_len)
1429 + max_frames_per_video = min(
1430 + max_total_frames // max(max_videos, 1), max_frames_per_video
1431 + )
1432 + return max(max_frames_per_video, 1)
1433 +
1434 + def get_max_video_tokens(
1435 + self,
1436 + seq_len: int,
1437 + mm_counts: Mapping[str, int],
1438 + ) -> int:
1439 + w, h = self.get_image_size_with_most_features()
1440 + return self.get_num_video_tokens(
1441 + image_width=w,
1442 + image_height=h,
1443 + num_frames=self.get_num_frames_with_most_features(seq_len, mm_counts),
1444 + )
1445 +
1446 + def get_supported_mm_limits(self) -> Mapping[str, int | None]:
1447 + return {"image": None, "video": None}
1448 +
1449 +
1450 +class LlavaOnevision2DummyInputsBuilder(
1451 + BaseDummyInputsBuilder[LlavaOnevision2ProcessingInfo]
1452 +):
1453 + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
1454 + n_img = mm_counts.get("image", 0)
1455 + n_vid = mm_counts.get("video", 0)
1456 + return (
1457 + "<|vision_start|><|image_pad|><|vision_end|>" * n_img
1458 + + "<|vision_start|><|video_pad|><|vision_end|>" * n_vid
1459 + )
1460 +
1461 + def get_dummy_mm_data(
1462 + self,
1463 + seq_len: int,
1464 + mm_counts: Mapping[str, int],
1465 + mm_options: Mapping[str, BaseDummyOptions] | None = None,
1466 + ) -> MultiModalDataDict:
1467 + n_img = mm_counts.get("image", 0)
1468 + n_vid = mm_counts.get("video", 0)
1469 + w, h = self.info.get_image_size_with_most_features()
1470 + out: MultiModalDataDict = {}
1471 + if n_img:
1472 + out["image"] = self._get_dummy_images(width=w, height=h, num_images=n_img)
1473 + if n_vid:
1474 + # 4 frames per dummy video keeps profiling cheap; the real frame
1475 + # count is decided by the HF VideoProcessor at apply time.
1476 + out["video"] = self._get_dummy_videos(
1477 + width=w, height=h, num_frames=4, num_videos=n_vid
1478 + )
1479 + return out
1480 +
1481 + def _get_dummy_videos(
1482 + self,
1483 + *,
1484 + width: int,
1485 + height: int,
1486 + num_frames: int,
1487 + num_videos: int,
1488 + overrides=None,
1489 + ):
1490 + # ``video_needs_metadata=True`` (see ProcessingInfo.get_data_parser)
1491 + # makes the parser require a metadata dict on every video item, so the
1492 + # dummy profiling videos must carry one too. ``do_sample_frames=False``
1493 + # plus ``frames_indices=range(T)`` tells the frame path to consume the
1494 + # pre-built frames verbatim (no resampling) -- mirrors GLM-4V.
1495 + # OV2's vision tower (temporal merge = 2) needs an even frame count.
1496 + num_frames = max(num_frames, _OV2_FRAME_FACTOR)
1497 + if num_frames % _OV2_FRAME_FACTOR != 0:
1498 + num_frames += 1
1499 + videos = super()._get_dummy_videos(
1500 + width=width,
1501 + height=height,
1502 + num_frames=num_frames,
1503 + num_videos=num_videos,
1504 + overrides=overrides,
1505 + )
1506 + video_items = []
1507 + for video in videos:
1508 + t = video.shape[0]
1509 + metadata = {
1510 + "fps": 1.0,
1511 + "duration": float(t),
1512 + "total_num_frames": int(t),
1513 + "frames_indices": list(range(t)),
1514 + "video_backend": "llava_onevision2",
1515 + "do_sample_frames": False,
1516 + }
1517 + video_items.append((video, metadata))
1518 + return video_items
1519 +
1520 +
1521 +class LlavaOnevision2MultiModalDataParser(MultiModalDataParser):
1522 + def __init__(self, spatial_merge_size: int, *args, **kwargs):
1523 + self._spatial_merge_size = spatial_merge_size
1524 + super().__init__(*args, **kwargs)
1525 +
1526 + def _parse_image_data(
1527 + self,
1528 + data: dict[str, torch.Tensor] | ModalityData[ImageItem],
1529 + ) -> ModalityDataItems[Any, Any] | None:
1530 + if isinstance(data, dict):
1531 + return DictEmbeddingItems(
1532 + data,
1533 + modality="image",
1534 + required_fields={"image_embeds", "image_grid_thw"},
1535 + fields_factory=_create_field_factory(self._spatial_merge_size),
1536 + )
1537 + return super()._parse_image_data(data)
1538 +
1539 +
1540 +class LlavaOnevision2MultiModalProcessor(
1541 + BaseMultiModalProcessor[LlavaOnevision2ProcessingInfo]
1542 +):
1543 + def _get_data_parser(self) -> MultiModalDataParser:
1544 + # Retained for symmetry; vLLM actually fetches the parser via
1545 + # info.get_data_parser() (see ProcessingInfo override above).
1546 + return LlavaOnevision2MultiModalDataParser(
1547 + self.info.get_hf_config().vision_config.spatial_merge_size
1548 + )
1549 +
1550 + def _call_hf_processor(
1551 + self,
1552 + prompt: str,
1553 + mm_data: Mapping[str, object],
1554 + mm_kwargs: Mapping[str, object],
1555 + tok_kwargs: Mapping[str, object],
1556 + ) -> BatchFeature:
1557 + # The wrapped OV2 processor is a bare custom class without the standard
1558 + # ProcessorMixin ``_merge_kwargs`` machinery, so vLLM's default path
1559 + # fails; overriding this method routes the base class to call us
1560 + # directly.
1561 + hf_processor = self.info.get_hf_processor(**mm_kwargs)
1562 + merged_kwargs = self.info.ctx.get_merged_mm_kwargs(
1563 + dict(**mm_kwargs, **tok_kwargs)
1564 + )
1565 + merged_kwargs.setdefault("return_tensors", "pt")
1566 + call_kwargs = {
1567 + k: v
1568 + for k, v in merged_kwargs.items()
1569 + if k
1570 + in {
1571 + "return_tensors",
1572 + "padding",
1573 + "num_frames",
1574 + "max_frames",
1575 + "target_fps",
1576 + "video_backend",
1577 + "max_pixels",
1578 + "codec_config",
1579 + }
1580 + }
1581 + mm_data = dict(mm_data)
1582 + # Explicit None + length checks: ``mm_data[...]`` may be a list, numpy
1583 + # array, or tensor, and ``and <array>`` would raise on the ambiguous
1584 + # truth value of a multi-element array.
1585 + _videos = mm_data.get("videos")
1586 + videos_present = _videos is not None and len(_videos) > 0
1587 +
1588 + codec_video_paths = (
1589 + _extract_codec_video_paths(mm_data["videos"]) if videos_present else None
1590 + )
1591 + is_codec_marker = codec_video_paths is not None
1592 + # Fallback: caller passed video_backend=codec via mm_processor_kwargs
1593 + # without wrapping paths through prepare_codec_video_input (e.g.
1594 + # lmms-eval's chat/vllm.py ov2_path_video=True). Recover the path
1595 + # strings directly from mm_data["videos"] so the codec rename
1596 + # branch still fires and video-modality fields get populated.
1597 + is_codec_kwarg = (
1598 + not is_codec_marker
1599 + and videos_present
1600 + and call_kwargs.get("video_backend") == "codec"
1601 + )
1602 + if is_codec_kwarg:
1603 + raw = mm_data["videos"]
1604 + if isinstance(raw, str):
1605 + codec_video_paths = [raw]
1606 + elif isinstance(raw, (list, tuple)) and all(
1607 + isinstance(x, str) for x in raw
1608 + ):
1609 + codec_video_paths = list(raw)
1610 + else:
1611 + # Non-string payload (PIL/ndarray/etc.) - codec backend
1612 + # cannot consume pre-decoded frames; fall through to frame
1613 + # path.
1614 + is_codec_kwarg = False
1615 + is_codec = is_codec_marker or is_codec_kwarg
1616 +
1617 + if is_codec:
1618 + # Confine codec paths to --allowed-local-media-path (local-only;
1619 + # SSRF / local-file-read protection) and use the *resolved* paths
1620 + # downstream so the codec module opens exactly the file that was
1621 + # validated (no symlink-retarget / validate-vs-open gap).
1622 + codec_video_paths = _validate_video_sources(
1623 + codec_video_paths, self.info.ctx.model_config
1624 + )
1625 + # Codec backend: HF processor consumes the path string directly and
1626 + # performs decode + canvas-packing internally. The dummy ndarray
1627 + # we attached during prepare_codec_video_input is discarded here.
1628 + mm_data["videos"] = (
1629 + codec_video_paths
1630 + if len(codec_video_paths) > 1
1631 + else codec_video_paths[0]
1632 + )
1633 + # Route through the base ``_call_hf_processor`` so float-tensor
1634 + # dtype postprocessing is applied automatically; inject
1635 + # ``video_backend="codec"`` via mm_kwargs so the wrapped processor
1636 + # dispatches to its codec branch.
1637 + output = super()._call_hf_processor(
1638 + prompt=prompt,
1639 + mm_data=mm_data,
1640 + mm_kwargs={**mm_kwargs, "video_backend": "codec"},
1641 + tok_kwargs=tok_kwargs,
1642 + )
1643 + data = dict(output)
1644 + return BatchFeature(
1645 + self._rename_codec_outputs_to_video(
1646 + data, codec_video_paths, hf_processor
1647 + )
1648 + )
1649 +
1650 + # ---- Frame backend (registered LlavaOnevision2VideoBackend) ------
1651 + # Every non-codec video reaches here as a ``(frames_ndarray, metadata)``
1652 + # tuple (``video_needs_metadata=True``): the connector decoded + sampled
1653 + # it through ``LlavaOnevision2VideoBackend`` for real ``video_url``
1654 + # inputs, or the dummy-inputs builder attached synthetic metadata during
1655 + # profiling. We materialise the frames as PIL images + per-frame
1656 + # timestamp markers and feed them through the HF processor's *image*
1657 + # branch (smart_resize + patchify), then re-tag the image-series outputs
1658 + # as video-series so vLLM's ``<|video_pad|>`` placeholder replacement
1659 + # finds them. Sampling parity with the original qwen_vl_utils policy is
1660 + # provided by the backend's ``compute_frames_index_to_sample``; SSRF /
1661 + # local-file gating is enforced by the connector before decoding.
1662 + if videos_present:
1663 + timestamp_decimals = int(
1664 + mm_kwargs.get("timestamp_decimals", _DEFAULT_TIMESTAMP_DECIMALS)
1665 + )
1666 +
1667 + per_video_frames: list[list[Image.Image]] = []
1668 + per_video_timestamps: list[list[float]] = []
1669 + for item in mm_data["videos"]:
1670 + pil_frames, timestamps = _frame_video_to_pil_and_timestamps(item)
1671 + per_video_frames.append(pil_frames)
1672 + per_video_timestamps.append(timestamps)
1673 +
1674 + # Rewrite the prompt so each video marker becomes a sequence of
1675 + # ``<{t} seconds><|vision_start|><|image_pad|><|vision_end|>``
1676 + # blocks (matches the OV2 hf-chat reference exactly).
1677 + new_prompt = _expand_video_markers_in_prompt(
1678 + prompt,
1679 + per_video_timestamps,
1680 + timestamp_decimals=timestamp_decimals,
1681 + )
1682 +
1683 + # Build the merged ``images`` list the wrapped HF processor will
1684 + # consume. The processor binds the merged list *positionally* to the
1685 + # ``<|image_pad|>`` slots in prompt order, so it must follow the
1686 + # interleaved marker order of the prompt (not a fixed "videos first"
1687 + # order) -- otherwise mixed image+video requests bind frames to the
1688 + # wrong placeholder. ``row_is_video`` labels each grid row so outputs
1689 + # can be split back into per-modality keys below.
1690 + merged_mm_data = dict(mm_data)
1691 + existing_images = merged_mm_data.pop("images", None)
1692 + caller_images: list[Image.Image] = []
1693 + if existing_images:
1694 + if isinstance(existing_images, list):
1695 + caller_images.extend(existing_images)
1696 + else:
1697 + caller_images.append(existing_images)
1698 +
1699 + marker_pattern = re.compile(
1700 + "(?P<image>" + re.escape(_IMAGE_MARKER) + ")"
1701 + "|(?P<video>" + re.escape(_VIDEO_MARKER) + ")"
1702 + )
1703 + flat_frames: list[Image.Image] = []
1704 + row_is_video: list[bool] = []
1705 + vid_idx = 0
1706 + img_idx = 0
1707 + for marker in marker_pattern.finditer(prompt):
1708 + if marker.lastgroup == "video":
1709 + frames = per_video_frames[vid_idx]
1710 + vid_idx += 1
1711 + flat_frames.extend(frames)
1712 + row_is_video.extend([True] * len(frames))
1713 + else:
1714 + flat_frames.append(caller_images[img_idx])
1715 + img_idx += 1
1716 + row_is_video.append(False)
1717 + # Fallback: if the prompt carried no parseable markers (defensive --
1718 + # should not happen for well-formed inputs), preserve the legacy
1719 + # "video frames first, then caller images" ordering.
1720 + if vid_idx == 0 and img_idx == 0:
1721 + for frames in per_video_frames:
1722 + flat_frames.extend(frames)
1723 + row_is_video.extend([True] * len(frames))
1724 + flat_frames.extend(caller_images)
1725 + row_is_video.extend([False] * len(caller_images))
1726 +
1727 + merged_mm_data.pop("videos", None)
1728 + merged_mm_data["images"] = flat_frames
1729 +
1730 + # Route through the base ``_call_hf_processor`` (applies float-tensor
1731 + # dtype postprocessing automatically). The wrapped processor's image
1732 + # branch ignores video/codec-only kwargs and does not forward extra
1733 + # **kwargs to the image processor, so passing the full merged kwarg
1734 + # set here is a no-op beyond return_tensors/padding.
1735 + output = super()._call_hf_processor(
1736 + prompt=new_prompt,
1737 + mm_data=merged_mm_data,
1738 + mm_kwargs=mm_kwargs,
1739 + tok_kwargs=tok_kwargs,
1740 + )
1741 + data = dict(output)
1742 +
1743 + # Split the image-series processor outputs back into video rows and
1744 + # genuine-image rows (one grid row per frame/image, in flat_frames
1745 + # order). Video rows become video-series keys; caller-image rows
1746 + # stay under image-series keys. Without this split a mixed
1747 + # image+video request silently drops the image modality.
1748 + grid = data.get("image_grid_thw")
1749 + has_caller_images = any(not v for v in row_is_video)
1750 + if grid is not None and has_caller_images:
1751 + pixel_values = data.pop("pixel_values")
1752 + patch_positions = data.pop("patch_positions")
1753 +
1754 + per_row_patches = grid.prod(-1).tolist()
1755 + row_offsets = [0]
1756 + for p in per_row_patches:
1757 + row_offsets.append(row_offsets[-1] + int(p))
1758 +
1759 + def _gather_rows(tensor, rows):
1760 + if not rows:
1761 + return tensor[:0]
1762 + return torch.cat(
1763 + [tensor[row_offsets[i] : row_offsets[i + 1]] for i in rows],
1764 + dim=0,
1765 + )
1766 +
1767 + video_rows = [i for i, v in enumerate(row_is_video) if v]
1768 + image_rows = [i for i, v in enumerate(row_is_video) if not v]
1769 +
1770 + data["pixel_values_videos"] = _gather_rows(pixel_values, video_rows)
1771 + data["patch_positions_videos"] = _gather_rows(
1772 + patch_positions, video_rows
1773 + )
1774 + data["video_grid_thw"] = grid[video_rows]
1775 +
1776 + data["pixel_values"] = _gather_rows(pixel_values, image_rows)
1777 + data["patch_positions"] = _gather_rows(patch_positions, image_rows)
1778 + data["image_grid_thw"] = grid[image_rows]
1779 + else:
1780 + # Video-only (no caller images): re-tag every image-series
1781 + # output as video-series.
1782 + if "pixel_values" in data:
1783 + data["pixel_values_videos"] = data.pop("pixel_values")
1784 + if "image_grid_thw" in data:
1785 + data["video_grid_thw"] = data.pop("image_grid_thw")
1786 + if "patch_positions" in data:
1787 + data["patch_positions_videos"] = data.pop("patch_positions")
1788 + data["frame_timestamps"] = _pack_timestamps(per_video_timestamps)
1789 + data["video_num_frames"] = torch.tensor(
1790 + [len(ts) for ts in per_video_timestamps], dtype=torch.long
1791 + )
1792 + data["video_is_codec"] = torch.zeros(
1793 + (len(per_video_timestamps),), dtype=torch.long
1794 + )
1795 +
1796 + return BatchFeature(data)
1797 +
1798 + # ---- Image-only / text-only call --------------------------------
1799 + # No videos present: delegate to the base ``_call_hf_processor``, which
1800 + # runs the wrapped processor over the (possibly empty) image set and
1801 + # applies float-tensor dtype postprocessing automatically.
1802 + return super()._call_hf_processor(
1803 + prompt=prompt,
1804 + mm_data=mm_data,
1805 + mm_kwargs=mm_kwargs,
1806 + tok_kwargs=tok_kwargs,
1807 + )
1808 +
1809 + def _rename_codec_outputs_to_video(
1810 + self,
1811 + data: dict,
1812 + codec_video_paths: list[str],
1813 + hf_processor,
1814 + ) -> dict:
1815 + # Codec branch emits image-series keys; vLLM expects video-series.
1816 + # Timestamps are NOT synthesized here: the codec format packs multiple
1817 + # source-frame timestamps into one canvas, so a per-canvas list is
1818 + # ill-defined. We ship per-video fps and let get_video_replacement
1819 + # reconstruct (timestamp, token_count) runs from patch_positions_videos.
1820 + data["pixel_values_videos"] = data.pop("pixel_values")
1821 + video_grid_thw = data.pop("image_grid_thw")
1822 + data["video_grid_thw"] = video_grid_thw
1823 + patch_positions = data.pop("patch_positions")
1824 + data["patch_positions_videos"] = patch_positions
1825 +
1826 + # Read per-video fps from processor output (order matches
1827 + # codec_video_paths); fall back to _codec_fps_for only if the processor
1828 + # didn't populate codec_fps (older model snapshots).
1829 + processor_fps = data.pop("codec_fps", None)
1830 + per_video_canvas_counts: list[int] = []
1831 + per_video_fps: list[float] = []
1832 + grid_offset = 0
1833 + for idx, video_path in enumerate(codec_video_paths):
1834 + num_canvases = self._count_canvases_for_video(
1835 + patch_positions, video_grid_thw, 0, grid_offset
1836 + )
1837 + per_video_canvas_counts.append(num_canvases)
1838 + if processor_fps is not None and idx < len(processor_fps):
1839 + per_video_fps.append(float(processor_fps[idx]))
1840 + else:
1841 + per_video_fps.append(_codec_fps_for(video_path, hf_processor))
1842 + grid_offset += num_canvases
1843 +
1844 + data["video_num_frames"] = torch.tensor(
1845 + per_video_canvas_counts, dtype=torch.long
1846 + )
1847 + data["video_is_codec"] = torch.ones(
1848 + (len(per_video_canvas_counts),), dtype=torch.long
1849 + )
1850 + # Stored as scaled int64 (fps * 1e9 rounded) because vLLM casts
1851 + # all float MM fields to model dtype (bfloat16), which rounds
1852 + # 29.97003 fps to 30.0 and shifts every timestamp tag by ~0.1s.
1853 + # Integer storage round-trips losslessly through the field-config
1854 + # pipeline; we divide back to float at replacement time.
1855 + data["codec_fps"] = torch.tensor(
1856 + [int(round(f * 1_000_000_000)) for f in per_video_fps], dtype=torch.int64
1857 + )
1858 + # frame_timestamps is required by the field-config schema; emit an
1859 + # empty per-video list (frames-backend populates it, codec ignores it).
1860 + data["frame_timestamps"] = _pack_timestamps([[] for _ in codec_video_paths])
1861 + return data
1862 +
1863 + def _count_canvases_for_video(
1864 + self,
1865 + patch_positions: torch.Tensor,
1866 + video_grid_thw: torch.Tensor,
1867 + canvas_offset: int,
1868 + grid_offset: int,
1869 + ) -> int:
1870 + # Single-video case: every remaining grid row belongs to this video.
1871 + # HF merges per-canvas rows into one [N,H,W] row per video, so the
1872 + # canvas count lives in the t-dim, not shape[0].
1873 + return int(video_grid_thw[grid_offset:, 0].sum().item())
1874 +
1875 + def _get_prompt_updates(
1876 + self,
1877 + mm_items: MultiModalDataItems,
1878 + hf_processor_mm_kwargs: Mapping[str, Any],
1879 + out_mm_kwargs: MultiModalKwargsItems,
1880 + ) -> Sequence[PromptUpdate]:
1881 + image_processor = self.info.get_image_processor(**hf_processor_mm_kwargs)
1882 + tokenizer = self.info.get_tokenizer()
1883 + vocab = tokenizer.get_vocab()
1884 + image_pad_id = vocab["<|image_pad|>"]
1885 + video_pad_id = vocab["<|video_pad|>"]
1886 + vision_start_id = vocab["<|vision_start|>"]
1887 + vision_end_id = vocab["<|vision_end|>"]
1888 + newline_ids = tokenizer.encode("\n", add_special_tokens=False)
1889 + merge_length = image_processor.merge_size**2
1890 + decimals = int(hf_processor_mm_kwargs.get("timestamp_decimals", 1))
1891 +
1892 + def get_image_replacement(item_idx: int):
1893 + out_item = out_mm_kwargs["image"][item_idx]
1894 + grid_thw = out_item["image_grid_thw"].data
1895 + n = int(grid_thw.prod(-1).sum()) // merge_length
1896 + return [image_pad_id] * n
1897 +
1898 + def get_video_replacement(item_idx: int):
1899 + out_item = out_mm_kwargs["video"][item_idx]
1900 + grid_thw = out_item["video_grid_thw"].data
1901 + is_codec_field = out_item.get("video_is_codec")
1902 + is_codec = (
1903 + bool(int(is_codec_field.data.item()))
1904 + if is_codec_field is not None
1905 + else False
1906 + )
1907 + tokens: list[int] = []
1908 +
1909 + if is_codec:
1910 + # Codec packs multiple source-frame timestamps into one canvas,
1911 + # so timestamps are per-run (consecutive patches sharing the
1912 + # same source-frame t), not per-canvas. Mirrors HF's
1913 + # rewrite_text_with_codec_positions: group patch_positions by
1914 + # run, emit ``<sec seconds><|vision_start|><pad*N><|vision_end|>\n``.
1915 + patch_positions = out_item["patch_positions_videos"].data
1916 + fps_t = out_item["codec_fps"].data
1917 + fps = float(int(fps_t.item())) / 1_000_000_000.0
1918 + runs = _codec_timestamp_runs(
1919 + patch_positions, fps, image_processor.merge_size
1920 + )
1921 + for sec, token_count in runs:
1922 + tag = f"<{sec:.{decimals}f} seconds>"
1923 + tag_ids = tokenizer.encode(tag, add_special_tokens=False)
1924 + tokens.extend(tag_ids)
1925 + tokens.append(vision_start_id)
1926 + tokens.extend([image_pad_id] * token_count)
1927 + tokens.append(vision_end_id)
1928 + tokens.extend(newline_ids)
1929 + else:
1930 + timestamps = out_item["frame_timestamps"].data
1931 + T_total = int(grid_thw.shape[0])
1932 + for t in range(T_total):
1933 + sec = float(timestamps[t].item())
1934 + tag = f"<{sec:.{decimals}f} seconds>"
1935 + tag_ids = tokenizer.encode(tag, add_special_tokens=False)
1936 + n_per_frame = int(grid_thw[t].prod()) // merge_length
1937 + tokens.extend(tag_ids)
1938 + tokens.append(vision_start_id)
1939 + tokens.extend([image_pad_id] * n_per_frame)
1940 + tokens.append(vision_end_id)
1941 + # Replacement mixes timestamp/marker tokens with image_pad
1942 + # placeholders; only image_pad positions carry vision embeddings,
1943 + # so a partial-embed mask is emitted via select_token_id.
1944 + return PromptUpdateDetails.select_token_id(tokens, image_pad_id)
1945 +
1946 + return [
1947 + PromptReplacement(
1948 + modality="image",
1949 + target=[image_pad_id],
1950 + replacement=get_image_replacement,
1951 + ),
1952 + PromptReplacement(
1953 + modality="video",
1954 + target=[vision_start_id, video_pad_id, vision_end_id],
1955 + replacement=get_video_replacement,
1956 + ),
1957 + ]
1958 +
1959 + def _get_mm_fields_config(
1960 + self,
1961 + hf_inputs: BatchFeature,
1962 + hf_processor_mm_kwargs: Mapping[str, object],
1963 + ) -> Mapping[str, MultiModalFieldConfig]:
1964 + return _create_field_factory(
1965 + self.info.get_hf_config().vision_config.spatial_merge_size,
1966 + )(hf_inputs)
1967 +
1968 +
1969 +@MULTIMODAL_REGISTRY.register_processor(
1970 + LlavaOnevision2MultiModalProcessor,
1971 + info=LlavaOnevision2ProcessingInfo,
1972 + dummy_inputs=LlavaOnevision2DummyInputsBuilder,
1973 +)
1974 +class LlavaOnevision2ForConditionalGeneration(
1975 + nn.Module, SupportsMultiModal, SupportsPP
1976 +):
1977 + """vLLM-side OV2 top-level model.
1978 +
1979 + Weight name rewriting (HF checkpoint → vLLM module tree):
1980 + Prefix rewrites only (longest match wins). Vision tower attribute names
1981 + mirror HF names verbatim, so no substring rules are needed. Substring
1982 + rules would otherwise collide with the Qwen3 text-path ``self_attn``
1983 + modules and break the language-model loader.
1984 + ``model.language_model.`` → ``language_model.model.``
1985 + ``model.visual.`` → ``visual.``
1986 + ``lm_head.`` → ``language_model.lm_head.``
1987 + ``model.`` (fallback) → ``language_model.model.``
1988 + """
1989 +
1990 + hf_to_vllm_mapper = WeightsMapper(
1991 + orig_to_new_prefix={
1992 + "model.language_model.": "language_model.model.",
1993 + "model.visual.": "visual.",
1994 + "lm_head.": "language_model.lm_head.",
1995 + "model.": "language_model.model.",
1996 + }
1997 + )
1998 +
1999 + @classmethod
2000 + def get_placeholder_str(cls, modality: str, i: int) -> str | None:
2001 + if modality.startswith("image"):
2002 + return "<|vision_start|><|image_pad|><|vision_end|>"
2003 + if modality.startswith("video"):
2004 + return "<|vision_start|><|video_pad|><|vision_end|>"
2005 + raise ValueError("Only image or video modality is supported")
2006 +
2007 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
2008 + super().__init__()
2009 + config = vllm_config.model_config.hf_config
2010 + multimodal_config = vllm_config.model_config.multimodal_config
2011 +
2012 + self.config = config
2013 + self.multimodal_config = multimodal_config
2014 +
2015 + # Build the vision tower under the tower marker so it is shared by the
2016 + # image, video-frame and video-codec backends (all routed through
2017 + # ``self.visual``). When both modalities are disabled via
2018 + # ``--limit-mm-per-prompt`` the marker turns the tower into a skipped
2019 + # placeholder whose weights are dropped automatically by the loader.
2020 + with self._mark_tower_model(vllm_config, {"image", "video"}):
2021 + self.visual = LlavaOnevision2VisionTower(
2022 + config.vision_config,
2023 + text_hidden_size=config.text_config.hidden_size,
2024 + norm_eps=getattr(config.vision_config, "layer_norm_eps", 1e-6),
2025 + quant_config=None,
2026 + prefix=maybe_prefix(prefix, "visual"),
2027 + )
2028 +
2029 + # OV2 LLM is plain Qwen3 -- 1-D positions, no M-RoPE. The wrapper
2030 + # LlavaOnevision2Config keeps text/vision configs nested (unlike OV1.5
2031 + # which promoted text fields), so explicitly hand text_config to the
2032 + # Qwen3 init path or qwen3.py will hit AttributeError on vocab_size.
2033 + self.language_model = init_vllm_registered_model(
2034 + vllm_config=vllm_config,
2035 + hf_config=config.text_config,
2036 + prefix=maybe_prefix(prefix, "language_model"),
2037 + architectures=["Qwen3ForCausalLM"],
2038 + )
2039 +
2040 + self.make_empty_intermediate_tensors = (
2041 + self.language_model.make_empty_intermediate_tensors
2042 + )
2043 +
2044 + def _validate_and_reshape_mm_tensor(
2045 + self, mm_input: object, name: str
2046 + ) -> torch.Tensor:
2047 + if not isinstance(mm_input, (torch.Tensor, list)):
2048 + raise ValueError(f"Incorrect type of {name}: {type(mm_input)}")
2049 + if isinstance(mm_input, torch.Tensor):
2050 + if mm_input.ndim == 2:
2051 + return mm_input
2052 + if mm_input.ndim != 3:
2053 + raise ValueError(
2054 + f"{name} must be 2D or batched-3D, got shape={mm_input.shape}"
2055 + )
2056 + # Flatten the leading batch dim into the patch dim: (b, n, d) ->
2057 + # (b*n, d), avoiding a Python list of row tensors.
2058 + return mm_input.flatten(0, 1)
2059 + return torch.concat(mm_input)
2060 +
2061 + def _parse_and_validate_image_input(
2062 + self, **kwargs: object
2063 + ) -> LlavaOnevision2ImageInputs | None:
2064 + pixel_values = kwargs.pop("pixel_values", None)
2065 + image_embeds = kwargs.pop("image_embeds", None)
2066 + image_grid_thw = kwargs.pop("image_grid_thw", None)
2067 + patch_positions = kwargs.pop("patch_positions", None)
2068 +
2069 + if pixel_values is None and image_embeds is None:
2070 + return None
2071 +
2072 + if pixel_values is not None:
2073 + pixel_values = self._validate_and_reshape_mm_tensor(
2074 + pixel_values, "image pixel values"
2075 + )
2076 + image_grid_thw = self._validate_and_reshape_mm_tensor(
2077 + image_grid_thw, "image grid_thw"
2078 + )
2079 + if patch_positions is None:
2080 + raise ValueError(
2081 + "OV2 requires patch_positions alongside pixel_values; "
2082 + "ensure the HF processor produces it (image, video-"
2083 + "frames, and video-codec backends all do)."
2084 + )
2085 + patch_positions = self._validate_and_reshape_mm_tensor(
2086 + patch_positions, "image patch_positions"
2087 + )
2088 + return LlavaOnevision2ImagePixelInputs(
2089 + type="pixel_values",
2090 + pixel_values=pixel_values,
2091 + image_grid_thw=image_grid_thw,
2092 + patch_positions=patch_positions,
2093 + )
2094 +
2095 + # image_embeds path
2096 + image_embeds = self._validate_and_reshape_mm_tensor(
2097 + image_embeds, "image embeds"
2098 + )
2099 + image_grid_thw = self._validate_and_reshape_mm_tensor(
2100 + image_grid_thw, "image grid_thw"
2101 + )
2102 + return LlavaOnevision2ImageEmbeddingInputs(
2103 + type="image_embeds",
2104 + image_embeds=image_embeds,
2105 + image_grid_thw=image_grid_thw,
2106 + )
2107 +
2108 + def _process_image_input(
2109 + self, image_input: LlavaOnevision2ImageInputs
2110 + ) -> tuple[torch.Tensor, ...]:
2111 + grid_thw = image_input["image_grid_thw"]
2112 + assert grid_thw.ndim == 2
2113 +
2114 + if image_input["type"] == "image_embeds":
2115 + image_embeds = image_input["image_embeds"]
2116 + else:
2117 + image_embeds = self.visual(
2118 + image_input["pixel_values"],
2119 + grid_thw=grid_thw,
2120 + patch_positions=image_input["patch_positions"],
2121 + )
2122 +
2123 + merge_size = self.visual.spatial_merge_size
2124 + sizes = grid_thw.prod(-1) // merge_size // merge_size
2125 + return image_embeds.split(sizes.tolist())
2126 +
2127 + def _parse_and_validate_video_input(
2128 + self, **kwargs: object
2129 + ) -> LlavaOnevision2VideoPixelInputs | None:
2130 + pixel_values_videos = kwargs.pop("pixel_values_videos", None)
2131 + video_grid_thw = kwargs.pop("video_grid_thw", None)
2132 + patch_positions_videos = kwargs.pop("patch_positions_videos", None)
2133 + video_num_frames = kwargs.pop("video_num_frames", None)
2134 + kwargs.pop("frame_timestamps", None)
2135 + kwargs.pop("video_is_codec", None)
2136 +
2137 + if pixel_values_videos is None:
2138 + return None
2139 +
2140 + pixel_values_videos = self._validate_and_reshape_mm_tensor(
2141 + pixel_values_videos, "video pixel values"
2142 + )
2143 + video_grid_thw = self._validate_and_reshape_mm_tensor(
2144 + video_grid_thw, "video grid_thw"
2145 + )
2146 + if patch_positions_videos is None:
2147 + raise ValueError(
2148 + "OV2 requires patch_positions_videos alongside pixel_values_videos."
2149 + )
2150 + patch_positions_videos = self._validate_and_reshape_mm_tensor(
2151 + patch_positions_videos, "video patch_positions"
2152 + )
2153 + if video_num_frames is None:
2154 + raise ValueError(
2155 + "OV2 requires video_num_frames alongside pixel_values_videos."
2156 + )
2157 + if isinstance(video_num_frames, list):
2158 + video_num_frames = torch.cat([v.flatten() for v in video_num_frames])
2159 + else:
2160 + video_num_frames = video_num_frames.flatten()
2161 + return LlavaOnevision2VideoPixelInputs(
2162 + type="pixel_values_videos",
2163 + pixel_values_videos=pixel_values_videos,
2164 + video_grid_thw=video_grid_thw,
2165 + patch_positions_videos=patch_positions_videos,
2166 + video_num_frames=video_num_frames,
2167 + )
2168 +
2169 + def _process_video_input(
2170 + self,
2171 + video_input: LlavaOnevision2VideoPixelInputs,
2172 + ) -> tuple[torch.Tensor, ...]:
2173 + # OV2 encodes each video frame independently through the same
2174 + # vision stack as still images, then splits the resulting token
2175 + # stream into per-video chunks using video_num_frames.
2176 + grid_thw = video_input["video_grid_thw"]
2177 + assert grid_thw.ndim == 2
2178 + video_embeds = self.visual(
2179 + video_input["pixel_values_videos"],
2180 + grid_thw=grid_thw,
2181 + patch_positions=video_input["patch_positions_videos"],
2182 + )
2183 + merge_size = self.visual.spatial_merge_size
2184 + per_frame_tokens = grid_thw.prod(-1) // merge_size // merge_size
2185 + # Aggregate per-frame token counts into per-video token counts.
2186 + num_frames = video_input["video_num_frames"].tolist()
2187 + sizes: list[int] = []
2188 + cursor = 0
2189 + for n in num_frames:
2190 + n = int(n)
2191 + sizes.append(int(per_frame_tokens[cursor : cursor + n].sum()))
2192 + cursor += n
2193 + return video_embeds.split(sizes)
2194 +
2195 + def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
2196 + modalities = {}
2197 + for key in kwargs:
2198 + if key in ("pixel_values", "image_embeds") and "images" not in modalities:
2199 + modalities["images"] = self._parse_and_validate_image_input(**kwargs)
2200 + if key == "pixel_values_videos" and "videos" not in modalities:
2201 + modalities["videos"] = self._parse_and_validate_video_input(**kwargs)
2202 + return modalities
2203 +
2204 + def get_language_model(self) -> torch.nn.Module:
2205 + return self.language_model
2206 +
2207 + def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
2208 + modalities = self._parse_and_validate_multimodal_inputs(**kwargs)
2209 + if not modalities:
2210 + return []
2211 + multimodal_embeddings: tuple[torch.Tensor, ...] = ()
2212 + for modality in modalities:
2213 + if modality == "images":
2214 + multimodal_embeddings += self._process_image_input(modalities["images"])
2215 + elif modality == "videos":
2216 + multimodal_embeddings += self._process_video_input(modalities["videos"])
2217 + return multimodal_embeddings
2218 +
2219 + def get_input_embeddings(
2220 + self,
2221 + input_ids: torch.Tensor,
2222 + multimodal_embeddings: MultiModalEmbeddings | None = None,
2223 + ) -> torch.Tensor:
2224 + inputs_embeds = self.language_model.get_input_embeddings(input_ids)
2225 + if multimodal_embeddings is not None and len(multimodal_embeddings) != 0:
2226 + inputs_embeds = merge_multimodal_embeddings(
2227 + inputs_embeds,
2228 + multimodal_embeddings,
2229 + input_ids == self.config.image_token_id,
2230 + )
2231 + return inputs_embeds
2232 +
2233 + def forward(
2234 + self,
2235 + input_ids: torch.Tensor,
2236 + positions: torch.Tensor,
2237 + intermediate_tensors: IntermediateTensors | None = None,
2238 + inputs_embeds: torch.Tensor | None = None,
2239 + **kwargs: object,
2240 + ) -> torch.Tensor | IntermediateTensors:
2241 + # V1 precomputes multimodal ``inputs_embeds`` via ``embed_multimodal`` /
2242 + # ``get_input_embeddings``, so the model forward only threads them into
2243 + # the language model (image + video share the same embedding merge).
2244 + if intermediate_tensors is not None:
2245 + inputs_embeds = None
2246 +
2247 + return self.language_model.model(
2248 + input_ids=input_ids,
2249 + positions=positions,
2250 + intermediate_tensors=intermediate_tensors,
2251 + inputs_embeds=inputs_embeds,
2252 + )
2253 +
2254 + def compute_logits(self, hidden_states: torch.Tensor):
2255 + return self.language_model.compute_logits(hidden_states)
2256 +
2257 + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
2258 + loader = AutoWeightsLoader(self)
2259 + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
2260 +
2261 + def get_mm_mapping(self) -> MultiModelKeys:
2262 + return MultiModelKeys.from_string_field(
2263 + language_model="language_model",
2264 + connector="visual.merger.",
2265 + tower_model="visual.",
2266 + )
vllm/model_executor/models/registry.py
+4
@@ -467,6 +467,10 @@ _MULTIMODAL_MODELS = {
467 "llava_onevision",
468 "LlavaOnevisionForConditionalGeneration",
469 ),
470 + "LlavaOnevision2ForConditionalGeneration": (
471 + "llava_onevision2",
472 + "LlavaOnevision2ForConditionalGeneration",
473 + ),
474 "MiDashengLMModel": ("midashenglm", "MiDashengLMModel"),
475 "MiMoV2OmniForCausalLM": ("mimo_v2_omni", "MiMoV2OmniForCausalLM"),
476 "MiniMaxM3SparseForConditionalGeneration": (