[Model] Enable LoRA support for tower and connector in LlavaNextVideo (#48594)
Signed-off-by: gangula-karthik <gkarthik923@gmail.com>
karthik committed
Jul 14, 2026 at 20:09 UTC
b6754f536e9cb40a1e117705a44d74218873b437
3 files changed
+149
-3
docs/models/supported_models.md
+1
-1
@@ -555,7 +555,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
555
| `Llama_Nemotron_Nano_VL` | Llama Nemotron Nano VL | T + I<sup>E+</sup> | `nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1` | ✅︎ | ✅︎ |
556
| `LlavaForConditionalGeneration` | LLaVA-1.5, Pixtral (HF Transformers) | T + I<sup>E+</sup> | `llava-hf/llava-1.5-7b-hf`, `mistral-community/pixtral-12b`, etc. | ✅︎ | ✅︎ |
557
| `LlavaNextForConditionalGeneration` | LLaVA-NeXT, Granite Vision | T + I<sup>E+</sup> | `llava-hf/llava-v1.6-mistral-7b-hf`, `llava-hf/llava-v1.6-vicuna-7b-hf`, `ibm-granite/granite-vision-3.3-2b`, etc. | | ✅︎ |
558
-| `LlavaNextVideoForConditionalGeneration` | LLaVA-NeXT-Video | T + V | `llava-hf/LLaVA-NeXT-Video-7B-hf`, etc. | | ✅︎ |
558
+| `LlavaNextVideoForConditionalGeneration` | LLaVA-NeXT-Video | T + V | `llava-hf/LLaVA-NeXT-Video-7B-hf`, etc. | ✅︎ | ✅︎ |
559
| `LlavaOnevision2ForConditionalGeneration` | LLaVA-OneVision-2 | T + I<sup>+</sup> + V<sup>+</sup> | `lmms-lab-encoder/LLaVA-OneVision-2-8B-Instruct` | | |
560
| `LlavaOnevisionForConditionalGeneration` | LLaVA-Onevision | T + I<sup>+</sup> + V<sup>+</sup> | `llava-hf/llava-onevision-qwen2-7b-ov-hf`, `llava-hf/llava-onevision-qwen2-0.5b-ov-hf`, etc. | | ✅︎ |
561
| `MiDashengLMModel` | MiDashengLM | T + A<sup>+</sup> | `mispeech/midashenglm-7b` | | ✅︎ |
tests/models/multimodal/processing/test_llava_next_video.py
new
+88
@@ -0,0 +1,88 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+import math
5
+
6
+import pytest
7
+
8
+from vllm.model_executor.models.llava_next_video import (
9
+ LlavaNextVideoForConditionalGeneration,
10
+)
11
+from vllm.model_executor.models.vision import get_vision_encoder_info
12
+
13
+from ...utils import build_model_context
14
+
15
+
16
+class _StubModel:
17
+ """Carries only the two attributes the token helpers read from
18
+ `self`, so the real methods can be exercised without constructing
19
+ the full `nn.Module` (vision tower, language model, etc.)."""
20
+
21
+ patch_grid_length: int
22
+ pooled_grid_length: int
23
+
24
+
25
+get_num_mm_encoder_tokens = (
26
+ LlavaNextVideoForConditionalGeneration.get_num_mm_encoder_tokens
27
+)
28
+get_num_mm_connector_tokens = (
29
+ LlavaNextVideoForConditionalGeneration.get_num_mm_connector_tokens
30
+)
31
+
32
+
33
+@pytest.mark.parametrize("model_id", ["llava-hf/LLaVA-NeXT-Video-7B-hf"])
34
+def test_num_mm_tokens_match_real_config(model_id):
35
+ """The stored grid lengths must match what `__init__` derives from
36
+ the real HF config, and the two helpers must invert each other's
37
+ frame-level scaling exactly."""
38
+ ctx = build_model_context(model_id, limit_mm_per_prompt={"video": 1})
39
+ config = ctx.model_config.hf_config
40
+
41
+ vision_encoder_info = get_vision_encoder_info(config)
42
+ patch_grid_length = vision_encoder_info.get_patch_grid_length()
43
+ pooled_grid_length = math.ceil(patch_grid_length / config.spatial_pool_stride)
44
+
45
+ stub = _StubModel()
46
+ stub.patch_grid_length = patch_grid_length
47
+ stub.pooled_grid_length = pooled_grid_length
48
+
49
+ for num_frames in (1, 2, 8, 16, 32):
50
+ num_video_tokens = num_frames * pooled_grid_length**2
51
+
52
+ encoder_tokens = get_num_mm_encoder_tokens(stub, num_video_tokens)
53
+ assert encoder_tokens == num_frames * patch_grid_length**2
54
+
55
+ connector_tokens = get_num_mm_connector_tokens(stub, encoder_tokens)
56
+ assert connector_tokens == num_video_tokens
57
+
58
+
59
+@pytest.mark.parametrize(
60
+ ("patch_grid_length", "pooled_grid_length", "num_frames"),
61
+ [
62
+ (24, 12, 1), # llava-hf/LLaVA-NeXT-Video-7B-hf: 336 / 14, stride 2
63
+ (24, 12, 16),
64
+ (27, 14, 5), # non-power-of-2 pooled grid (ceil rounding)
65
+ (16, 8, 6),
66
+ ],
67
+)
68
+def test_num_mm_tokens_roundtrip(patch_grid_length, pooled_grid_length, num_frames):
69
+ stub = _StubModel()
70
+ stub.patch_grid_length = patch_grid_length
71
+ stub.pooled_grid_length = pooled_grid_length
72
+
73
+ num_video_tokens = num_frames * pooled_grid_length**2
74
+
75
+ encoder_tokens = get_num_mm_encoder_tokens(stub, num_video_tokens)
76
+ assert encoder_tokens == num_frames * patch_grid_length**2
77
+
78
+ connector_tokens = get_num_mm_connector_tokens(stub, encoder_tokens)
79
+ assert connector_tokens == num_video_tokens
80
+
81
+
82
+def test_num_mm_tokens_zero():
83
+ stub = _StubModel()
84
+ stub.patch_grid_length = 24
85
+ stub.pooled_grid_length = 12
86
+
87
+ assert get_num_mm_encoder_tokens(stub, 0) == 0
88
+ assert get_num_mm_connector_tokens(stub, 0) == 0
vllm/model_executor/models/llava_next_video.py
+60
-2
@@ -36,8 +36,14 @@ from vllm.sequence import IntermediateTensors
36
from vllm.utils.collection_utils import is_list_of
37
from vllm.utils.tensor_schema import TensorSchema, TensorShape
38
39
-from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
39
+from .interfaces import (
40
+ MultiModalEmbeddings,
41
+ SupportsLoRA,
42
+ SupportsMultiModal,
43
+ SupportsPP,
44
+)
45
from .llava import init_vision_tower_for_llava
46
+from .module_mapping import MultiModelKeys
47
from .siglip import SiglipVisionModel
48
from .utils import (
49
AutoWeightsLoader,
@@ -298,7 +304,14 @@ class LlavaNextMultiModalProjector(nn.Module):
304
info=LlavaNextVideoProcessingInfo,
305
dummy_inputs=LlavaNextVideoDummyInputsBuilder,
306
)
301
-class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
307
+class LlavaNextVideoForConditionalGeneration(
308
+ nn.Module, SupportsLoRA, SupportsMultiModal, SupportsPP
309
+):
310
+ packed_modules_mapping = {
311
+ "qkv_proj": ["q_proj", "k_proj", "v_proj"],
312
+ "gate_up_proj": ["gate_proj", "up_proj"],
313
+ }
314
+
315
hf_to_vllm_mapper = WeightsMapper(
316
orig_to_new_prefix={
317
# mapping for new names in checkpoint saved after transformers v4.52
@@ -327,6 +340,12 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
340
self.config = config
341
self.multimodal_config = multimodal_config
342
343
+ vision_encoder_info = get_vision_encoder_info(config)
344
+ self.patch_grid_length = vision_encoder_info.get_patch_grid_length()
345
+ self.pooled_grid_length = math.ceil(
346
+ self.patch_grid_length / config.spatial_pool_stride
347
+ )
348
+
349
with self._mark_tower_model(vllm_config, "video"):
350
# Initialize the vision tower only up to the required feature layer
351
self.vision_tower = init_vision_tower_for_llava(
@@ -460,3 +479,42 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
479
ignore_unexpected_prefixes=["image_newline"],
480
)
481
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
482
+
483
+ def get_mm_mapping(self) -> MultiModelKeys:
484
+ """
485
+ Get the module prefix in multimodal models
486
+ """
487
+ return MultiModelKeys.from_string_field(
488
+ language_model="language_model",
489
+ connector="multi_modal_projector",
490
+ tower_model="vision_tower",
491
+ )
492
+
493
+ def get_num_mm_encoder_tokens(
494
+ self,
495
+ num_video_tokens: int,
496
+ ) -> int:
497
+ # Invert the spatial pooling done by `vision_resampler`: each frame
498
+ # contributes `pooled_grid_length ** 2` tokens after the language
499
+ # model's placeholder count, but `patch_grid_length ** 2` tokens
500
+ # when it leaves the vision encoder.
501
+ pooled_tokens_per_frame = self.pooled_grid_length**2
502
+ if num_video_tokens <= 0 or pooled_tokens_per_frame <= 0:
503
+ return 0
504
+
505
+ num_frames = num_video_tokens // pooled_tokens_per_frame
506
+ return num_frames * (self.patch_grid_length**2)
507
+
508
+ def get_num_mm_connector_tokens(
509
+ self,
510
+ num_vision_tokens: int,
511
+ ) -> int:
512
+ # The projector is length-preserving; it runs after
513
+ # `vision_resampler` has already pooled each frame down to
514
+ # `pooled_grid_length ** 2` tokens.
515
+ patch_tokens_per_frame = self.patch_grid_length**2
516
+ if num_vision_tokens <= 0 or patch_tokens_per_frame <= 0:
517
+ return 0
518
+
519
+ num_frames = num_vision_tokens // patch_tokens_per_frame
520
+ return num_frames * (self.pooled_grid_length**2)