@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 60915c972c

[Feature] Add VidCom2 video token pruning (#47750)

Signed-off-by: Benedikt Falk <bfalk@nvidia.com>

nvbfalk committed Jul 28, 2026 at 05:20 UTC 60915c972ccf0fa5a1993fef15a8bb667dbaa0f6

17 files changed +421 -48

docs/design/cuda_graphs_multimodal.md

+1 -1

@@ -101,7 +101,7 @@ When `mm_encoder_tp_mode="data"`, the manager distributes images across TP ranks

101 Following <https://github.com/vllm-project/vllm/pull/35963> (ViT full CUDA graph support for image inference), <https://github.com/vllm-project/vllm/pull/38061> extends the encoder CUDA graph framework to support video inference for Qwen3-VL. Previously, the CUDA graph capture/replay path only handled image inputs (`pixel_values` + `image_grid_thw`). Video inputs use different keys (`pixel_values_videos` + `video_grid_thw`) and require larger `cu_seqlens` buffers because each video item contributes multiple frames (`T` attention sequences). This PR generalizes the protocol and manager to handle both modalities through a single shared graph manager.

102

103 !!! note

104 - Video CUDA graphs are automatically disabled when EVS (Efficient Video Sampling) pruning is enabled, since EVS makes the token count data-dependent and incompatible with CUDA graph capture.

104 + Video CUDA graphs are automatically disabled when video token pruning (EVS or VidCom2) is enabled, since pruning makes the token count data-dependent and incompatible with CUDA graph capture.

105

106 Mixed inputs (image+video) per prompt are also supported now.

107

docs/features/multimodal_inputs.md

+25

@@ -350,6 +350,31 @@ Instead of NumPy arrays, you can also pass `'torch.Tensor'` instances, as shown

350

351 Full example: [examples/generate/multimodal/vision_language_offline.py](../../examples/generate/multimodal/vision_language_offline.py)

352

353 +#### Video Token Pruning

354 +

355 +For supported models, vLLM can prune video tokens after the vision encoder to

356 +reduce prefill time and KV cache usage, at some cost in accuracy. Set

357 +`--video-pruning-rate <q>` to prune the fraction `q` of video tokens from each

358 +video, and `--video-pruning-method` to choose the training-free algorithm:

359 +

360 +- **`evs`** (Efficient Video Sampling, default): drops the tokens with the

361 + lowest temporal dissimilarity to the previous frame. The first frame is

362 + always fully retained.

363 +- **`vidcom2`** (Video Compression Commander): scores tokens by similarity to

364 + video-level and frame-level feature centers and gives distinctive frames a

365 + larger share of the budget. At least one token per frame is retained.

366 +

367 +```bash

368 +vllm serve Qwen/Qwen3-VL-8B-Instruct \

369 + --video-pruning-rate 0.75 --video-pruning-method vidcom2

370 +```

371 +

372 +!!! note

373 + `evs` is supported by all models implementing multimodal pruning;

374 + `vidcom2` is currently supported by Qwen3-VL only. Unsupported combinations

375 + are rejected at startup. Enabling video pruning also disables encoder CUDA

376 + graphs, since the retained token count becomes data-dependent.

377 +

378 ### Audio Inputs

379

380 You can pass a tuple `(array, sampling_rate)` to the `'audio'` field of the multi-modal dictionary.

tests/multimodal/test_vidcom2.py new

+144

@@ -0,0 +1,144 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +

4 +import pytest

5 +import torch

6 +

7 +from vllm.multimodal.video_prune.vidcom2 import (

8 + compute_retained_tokens_count,

9 + compute_retention_mask,

10 +)

11 +

12 +

13 +def _fake_video_embeds(

14 + num_frames: int,

15 + rows: int,

16 + cols: int,

17 + hidden: int = 64,

18 + seed: int = 0,

19 +) -> torch.Tensor:

20 + """Deterministic fake ViT output with a distinct mean per frame."""

21 + g = torch.Generator().manual_seed(seed)

22 + frames = []

23 + for f in range(num_frames):

24 + base = torch.randn(hidden, generator=g) * (0.1 + 0.05 * f)

25 + frames.append(

26 + base[None, :].expand(rows * cols, hidden)

27 + + 0.01 * torch.randn(rows * cols, hidden, generator=g)

28 + )

29 + return torch.cat(frames, dim=0)

30 +

31 +

32 +@pytest.mark.parametrize("q", [0.25, 0.5, 0.75, 0.9])

33 +@pytest.mark.parametrize("num_frames", [1, 4, 16])

34 +def test_mask_shape_and_dtype(q: float, num_frames: int) -> None:

35 + merge = 2

36 + rows, cols = 6, 8

37 + embeds = _fake_video_embeds(num_frames, rows, cols)

38 + mask = compute_retention_mask(

39 + embeds,

40 + (num_frames, rows * merge, cols * merge),

41 + spatial_merge_size=merge,

42 + q=q,

43 + )

44 + assert mask.dtype == torch.bool

45 + assert mask.shape == (num_frames * rows * cols,)

46 +

47 +

48 +def test_retained_count_floors_at_one_token_per_frame() -> None:

49 + """The global minimum is one token per frame (not a full first frame)."""

50 + assert (

51 + compute_retained_tokens_count(tokens_per_frame=48, num_frames=4, q=0.999) == 4

52 + )

53 + assert (

54 + compute_retained_tokens_count(tokens_per_frame=48, num_frames=4, q=0.0)

55 + == 48 * 4

56 + )

57 +

58 +

59 +@pytest.mark.parametrize("q", [0.25, 0.5, 0.75, 0.9])

60 +@pytest.mark.parametrize("num_frames", [1, 4, 16])

61 +def test_total_retained_matches_target(q: float, num_frames: int) -> None:

62 + """Mask total must equal the placeholder-sizing helper."""

63 + merge = 2

64 + rows, cols = 6, 8

65 + tpf = rows * cols

66 + embeds = _fake_video_embeds(num_frames, rows, cols)

67 + mask = compute_retention_mask(

68 + embeds,

69 + (num_frames, rows * merge, cols * merge),

70 + spatial_merge_size=merge,

71 + q=q,

72 + )

73 + expected = compute_retained_tokens_count(

74 + tokens_per_frame=tpf, num_frames=num_frames, q=q

75 + )

76 + assert int(mask.sum().item()) == expected

77 +

78 +

79 +def test_per_frame_min_one_when_budget_allows() -> None:

80 + """No frame is fully dropped when the budget allows."""

81 + merge = 2

82 + rows, cols = 6, 8

83 + num_frames = 8

84 + embeds = _fake_video_embeds(num_frames, rows, cols)

85 + mask = compute_retention_mask(

86 + embeds,

87 + (num_frames, rows * merge, cols * merge),

88 + spatial_merge_size=merge,

89 + q=0.25,

90 + )

91 + per_frame = mask.view(num_frames, rows * cols).sum(dim=1)

92 + assert (per_frame >= 1).all(), f"zero-token frame detected: {per_frame.tolist()}"

93 +

94 +

95 +def test_dynamic_per_frame_budget() -> None:

96 + """A distinctive frame gets more retained tokens than bland ones."""

97 + merge = 2

98 + rows, cols = 6, 8

99 + tpf = rows * cols

100 + hidden = 64

101 + torch.manual_seed(0)

102 + bland = 0.01 * torch.randn(tpf, hidden)

103 + frames = [torch.randn(tpf, hidden) * 1.0]

104 + for _ in range(7):

105 + frames.append(bland + 0.001 * torch.randn(tpf, hidden))

106 + embeds = torch.cat(frames, dim=0)

107 + mask = compute_retention_mask(

108 + embeds,

109 + (8, rows * merge, cols * merge),

110 + spatial_merge_size=merge,

111 + q=0.5,

112 + )

113 + per_frame = mask.view(8, tpf).sum(dim=1)

114 + assert per_frame[0].item() > per_frame[1:].float().mean().item()

115 +

116 +

117 +def test_empty_input_safe() -> None:

118 + embeds = torch.zeros(0, 32)

119 + mask = compute_retention_mask(embeds, (0, 0, 0), spatial_merge_size=2, q=0.25)

120 + assert mask.numel() == 0

121 +

122 +

123 +@pytest.mark.parametrize("q", [0.0, 0.25, 0.5, 0.75])

124 +def test_first_frame_not_privileged(q: float) -> None:

125 + """A bland first frame is not force-retained (unlike EVS)."""

126 + merge = 2

127 + rows, cols = 6, 8

128 + tpf = rows * cols

129 + torch.manual_seed(1)

130 + bland = 0.01 * torch.randn(tpf, 64)

131 + frames = [bland]

132 + for f in range(7):

133 + frames.append(torch.randn(tpf, 64) * (1.0 + 0.1 * f))

134 + embeds = torch.cat(frames, dim=0)

135 + mask = compute_retention_mask(

136 + embeds,

137 + (8, rows * merge, cols * merge),

138 + spatial_merge_size=merge,

139 + q=q,

140 + )

141 + per_frame = mask.view(8, tpf).sum(dim=1)

142 + assert per_frame[0].item() <= tpf

143 + if q > 0.0:

144 + assert per_frame[0].item() < int(mask.sum().item())

vllm/config/model.py

+16

@@ -376,6 +376,7 @@ class ModelConfig:

376 interleave_mm_strings: InitVar[bool | None] = None

377 skip_mm_profiling: InitVar[bool | None] = None

378 video_pruning_rate: InitVar[float | None] = None

379 + video_pruning_method: InitVar[str | None] = None

380 mm_tensor_ipc: InitVar[MMTensorIPC] = None

381 mm_ipc_gpu_memory_gb: InitVar[float | None] = None

382

@@ -504,6 +505,7 @@ class ModelConfig:

505 interleave_mm_strings: bool | None,

506 skip_mm_profiling: bool | None,

507 video_pruning_rate: float | None,

508 + video_pruning_method: str | None,

509 mm_tensor_ipc: MMTensorIPC,

510 mm_ipc_gpu_memory_gb: float | None,

511 ) -> None:

@@ -735,6 +737,7 @@ class ModelConfig:

737 interleave_mm_strings=interleave_mm_strings,

738 skip_mm_profiling=skip_mm_profiling,

739 video_pruning_rate=video_pruning_rate,

740 + video_pruning_method=video_pruning_method,

741 mm_tensor_ipc=mm_tensor_ipc,

742 mm_ipc_gpu_memory_gb=mm_ipc_gpu_memory_gb,

743 )

@@ -745,6 +748,19 @@ class ModelConfig:

748

749 self.multimodal_config = MultiModalConfig(**mm_config_kwargs) # type: ignore[arg-type]

750

751 + pruning_spec = self.multimodal_config.get_video_pruning_spec()

752 + supported_pruning = self._model_info.supported_video_pruning_methods

753 + if (

754 + pruning_spec is not None

755 + and supported_pruning

756 + and pruning_spec[0] not in supported_pruning

757 + ):

758 + raise ValueError(

759 + f"Video pruning method '{pruning_spec[0]}' is not "

760 + f"supported by {self._model_info.architecture} "

761 + f"(supported methods: {supported_pruning})."

762 + )

763 +

764 if (

765 self.renderer_num_workers > 1

766 and self.multimodal_config.mm_processor_cache_gb > 0

vllm/config/multimodal.py

+17 -4

@@ -61,6 +61,7 @@ class MultiModalDummyOptionsBuiltins(TypedDict, total=False):

62 MMEncoderTPMode = Literal["weights", "data"]

63 MMCacheType = Literal["shm", "lru"]

64 +VideoPruningMethod = Literal["evs", "vidcom2"]

65 MMTensorIPC = Literal["direct_rpc", "torch_shm"]

66 MMDummyOptions: TypeAlias = dict[str, BaseDummyOptions]

67 """

@@ -189,9 +190,14 @@ class MultiModalConfig:

190 estimating the peak memory usage of the activation of multimodal encoder and

191 embedding cache."""

192 video_pruning_rate: float | None = Field(default=None, ge=0.0, lt=1.0)

192 - """Sets pruning rate for video pruning via Efficient Video Sampling.

193 - Value sits in range [0;1) and determines fraction of media tokens

194 - from each video to be pruned.

193 + """Fraction of video tokens to prune from each video. Value sits in range

194 + [0;1); pruning is enabled when it is greater than 0. The pruning algorithm

195 + is selected by `video_pruning_method`.

196 + """

197 + video_pruning_method: VideoPruningMethod = "evs"

198 + """Video token pruning algorithm applied when `video_pruning_rate` > 0:

199 + - "evs": Efficient Video Sampling.

200 + - "vidcom2": Video Compression Commander.

201 """

202 mm_tensor_ipc: MMTensorIPC = "direct_rpc"

203 """IPC (inter-process communication) method for multimodal tensors.

@@ -360,4 +366,11 @@ class MultiModalConfig:

366 )

367

368 def is_multimodal_pruning_enabled(self):

363 - return self.video_pruning_rate is not None and self.video_pruning_rate > 0

369 + return self.get_video_pruning_spec() is not None

370 +

371 + def get_video_pruning_spec(self) -> tuple[VideoPruningMethod, float] | None:

372 + """Return `(method, rate)` when video pruning is enabled, else None.

373 + `rate` is the fraction of video tokens to prune."""

374 + if self.video_pruning_rate is not None and self.video_pruning_rate > 0:

375 + return (self.video_pruning_method, float(self.video_pruning_rate))

376 + return None

vllm/engine/arg_utils.py

@@ -586,6 +586,7 @@ class EngineArgs:

586 renderer_num_workers: int = 1

587 skip_mm_profiling: bool = MultiModalConfig.skip_mm_profiling

588 video_pruning_rate: float | None = MultiModalConfig.video_pruning_rate

589 + video_pruning_method: str = MultiModalConfig.video_pruning_method

590 mm_tensor_ipc: MMTensorIPC = MultiModalConfig.mm_tensor_ipc

591 mm_ipc_gpu_memory_gb: float = MultiModalConfig.mm_ipc_gpu_memory_gb

592 # LoRA fields

@@ -1333,6 +1334,10 @@ class EngineArgs:

1334 multimodal_group.add_argument(

1335 "--video-pruning-rate", **multimodal_kwargs["video_pruning_rate"]

1336 )

1337 + multimodal_group.add_argument(

1338 + "--video-pruning-method",

1339 + **multimodal_kwargs["video_pruning_method"],

1340 + )

1341 multimodal_group.add_argument(

1342 "--mm-tensor-ipc", **multimodal_kwargs["mm_tensor_ipc"]

1343 )

@@ -1715,6 +1720,7 @@ class EngineArgs:

1720 override_attention_dtype=self.override_attention_dtype,

1721 logits_processors=self.logits_processors,

1722 video_pruning_rate=self.video_pruning_rate,

1723 + video_pruning_method=self.video_pruning_method,

1724 mm_tensor_ipc=self.mm_tensor_ipc,

1725 mm_ipc_gpu_memory_gb=self.mm_ipc_gpu_memory_gb,

1726 io_processor_plugin=self.io_processor_plugin,

vllm/model_executor/models/interfaces.py

@@ -41,6 +41,7 @@ if TYPE_CHECKING:

41 SpeechToTextParams,

42 VllmConfig,

43 )

44 + from vllm.config.multimodal import VideoPruningMethod

45 from vllm.inputs import PromptType, TokensPrompt

46 from vllm.lora.model_manager import LoRAModelManager

47 from vllm.model_executor.layers.fused_moe import MoERunner

@@ -424,6 +425,13 @@ class SupportsMultiModalPruning(Protocol):

425

426 supports_multimodal_pruning: ClassVar[Literal[True]] = True

427

428 + supported_video_pruning_methods: ClassVar[tuple["VideoPruningMethod", ...]] = (

429 + "evs",

430 + )

431 + """Video pruning methods (as reported by

432 + `MultiModalConfig.get_video_pruning_spec`) implemented by this model.

433 + Models supporting methods beyond EVS should override this."""

434 +

435 def recompute_mrope_positions(

436 self,

437 input_ids: list[int] | torch.Tensor,

vllm/model_executor/models/interns1_pro.py

+1 -4

@@ -570,10 +570,7 @@ class InternS1ProForConditionalGeneration(

570 self.config = config

571 self.multimodal_config = multimodal_config

572 self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"

573 - self.video_pruning_rate = multimodal_config.video_pruning_rate

574 - self.is_multimodal_pruning_enabled = (

575 - multimodal_config.is_multimodal_pruning_enabled()

576 - )

573 + self._init_video_pruning(multimodal_config)

574

575 with self._mark_tower_model(vllm_config, {"image", "video"}):

576 self.visual = Qwen3_VisionTransformer(

vllm/model_executor/models/nano_nemotron_vl.py

+4 -4

@@ -42,10 +42,6 @@ from vllm.model_executor.models.utils import (

42 maybe_prefix,

43 )

44 from vllm.multimodal import MULTIMODAL_REGISTRY

45 -from vllm.multimodal.evs import (

46 - compute_retained_tokens_count,

47 - compute_retention_mask,

48 -)

45 from vllm.multimodal.inputs import (

46 AudioItem,

47 BatchedTensorInputs,

@@ -74,6 +70,10 @@ from vllm.multimodal.processing.processor import (

70 PromptReplacement,

71 PromptUpdate,

72 )

73 +from vllm.multimodal.video_prune.evs import (

74 + compute_retained_tokens_count,

75 + compute_retention_mask,

76 +)

77 from vllm.renderers import TokenizeParams

78 from vllm.sequence import IntermediateTensors

79 from vllm.tokenizers import cached_tokenizer_from_config

vllm/model_executor/models/qwen2_5_vl.py

+6 -6

@@ -67,12 +67,6 @@ from vllm.model_executor.layers.rotary_embedding.common import (

67 )

68 from vllm.model_executor.models.module_mapping import MultiModelKeys

69 from vllm.multimodal import MULTIMODAL_REGISTRY

70 -from vllm.multimodal.evs import (

71 - compute_mrope_for_media,

72 - compute_retained_tokens_count,

73 - compute_retention_mask,

74 - recompute_mrope_positions,

75 -)

70 from vllm.multimodal.inputs import (

71 MultiModalFeatureSpec,

72 MultiModalFieldConfig,

@@ -80,6 +74,12 @@ from vllm.multimodal.inputs import (

74 )

75 from vllm.multimodal.parse import MultiModalDataItems

76 from vllm.multimodal.processing import PromptReplacement, PromptUpdate

77 +from vllm.multimodal.video_prune.evs import (

78 + compute_mrope_for_media,

79 + compute_retained_tokens_count,

80 + compute_retention_mask,

81 + recompute_mrope_positions,

82 +)

83 from vllm.platforms import current_platform

84 from vllm.sequence import IntermediateTensors

85 from vllm.utils.tensor_schema import TensorSchema, TensorShape

vllm/model_executor/models/qwen3_vl.py

+61 -24

@@ -50,7 +50,12 @@ from transformers.video_utils import VideoMetadata

51 from vllm.compilation.decorators import support_torch_compile

52 from vllm.config import VllmConfig

53 -from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions

53 +from vllm.config.multimodal import (

54 + BaseDummyOptions,

55 + MultiModalConfig,

56 + VideoDummyOptions,

57 + VideoPruningMethod,

58 +)

59 from vllm.distributed import get_pp_group, parallel_state

60 from vllm.inputs import MultiModalDataDict

61 from vllm.logger import init_logger

@@ -69,12 +74,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope

74 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead

75 from vllm.model_executor.models.module_mapping import MultiModelKeys

76 from vllm.multimodal import MULTIMODAL_REGISTRY

72 -from vllm.multimodal.evs import (

73 - compute_mrope_for_media,

74 - compute_retained_tokens_count,

75 - compute_retention_mask,

76 - recompute_mrope_positions,

77 -)

77 from vllm.multimodal.inputs import (

78 MultiModalFeatureSpec,

79 MultiModalFieldConfig,

@@ -92,6 +91,18 @@ from vllm.multimodal.processing import (

91 PromptUpdate,

92 PromptUpdateDetails,

93 )

94 +from vllm.multimodal.video_prune.evs import (

95 + compute_mrope_for_media,

96 + compute_retained_tokens_count,

97 + compute_retention_mask,

98 + recompute_mrope_positions,

99 +)

100 +from vllm.multimodal.video_prune.vidcom2 import (

101 + compute_retained_tokens_count as vidcom2_compute_retained_tokens_count,

102 +)

103 +from vllm.multimodal.video_prune.vidcom2 import (

104 + compute_retention_mask as vidcom2_compute_retention_mask,

105 +)

106 from vllm.sequence import IntermediateTensors

107 from vllm.tokenizers.protocol import TokenizerLike

108 from vllm.tokenizers.registry import cached_tokenizer_from_config

@@ -1256,7 +1267,7 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])

1267 hf_config = self.info.get_hf_config()

1268 tokenizer = self.info.get_tokenizer()

1269 merge_size = hf_config.vision_config.spatial_merge_size

1259 - video_pruning_rate = self.info.ctx.get_mm_config().video_pruning_rate

1270 + pruning_spec = self.info.ctx.get_mm_config().get_video_pruning_spec()

1271 vision_start_token_id = hf_config.vision_start_token_id

1272 vision_end_token_id = hf_config.vision_end_token_id

1273 video_token_id = hf_config.video_token_id

@@ -1339,11 +1350,18 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])

1350 merge_size**2

1351 )

1352

1342 - if video_pruning_rate is not None and video_pruning_rate > 0.0:

1343 - num_tokens = compute_retained_tokens_count(

1353 + # Apply video pruning (EVS or VidCom2) if enabled.

1354 + if pruning_spec is not None:

1355 + method, prune_q = pruning_spec

1356 + count_fn = (

1357 + vidcom2_compute_retained_tokens_count

1358 + if method == "vidcom2"

1359 + else compute_retained_tokens_count

1360 + )

1361 + num_tokens = count_fn(

1362 tokens_per_frame=tokens_per_frame_base,

1363 num_frames=num_frames,

1346 - q=video_pruning_rate,

1364 + q=prune_q,

1365 )

1366 tokens_per_frame = [num_tokens] + [0] * (num_frames - 1)

1367 select_token_id = False

@@ -1459,16 +1477,22 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])

1477 f"video length ({grid_thw[0]})."

1478 )

1479

1462 - # Compute tokens per frame, with EVS support

1480 + # Compute tokens per frame, with EVS / VidCom2 support

1481 num_frames = int(grid_thw[0])

1482 tokens_per_frame_base = int(grid_thw[1:].prod()) // merge_length

1483

1466 - video_pruning_rate = self.info.ctx.get_mm_config().video_pruning_rate

1467 - if video_pruning_rate is not None and video_pruning_rate > 0.0:

1468 - num_tokens = compute_retained_tokens_count(

1484 + pruning_spec = self.info.ctx.get_mm_config().get_video_pruning_spec()

1485 + if pruning_spec is not None:

1486 + method, prune_q = pruning_spec

1487 + count_fn = (

1488 + vidcom2_compute_retained_tokens_count

1489 + if method == "vidcom2"

1490 + else compute_retained_tokens_count

1491 + )

1492 + num_tokens = count_fn(

1493 tokens_per_frame=tokens_per_frame_base,

1494 num_frames=num_frames,

1471 - q=video_pruning_rate,

1495 + q=prune_q,

1496 )

1497 tokens_per_frame = [num_tokens] + [0] * (num_frames - 1)

1498 select_token_id = False

@@ -1701,6 +1725,8 @@ class Qwen3VLForConditionalGeneration(

1725

1726 supports_encoder_tp_data = True

1727

1728 + supported_video_pruning_methods = ("evs", "vidcom2")

1729 +

1730 # To ensure correct weight loading and mapping.

1731 hf_to_vllm_mapper = WeightsMapper(

1732 orig_to_new_prefix={

@@ -1719,6 +1745,17 @@ class Qwen3VLForConditionalGeneration(

1745

1746 raise ValueError("Only image or video modality is supported")

1747

1748 + def _init_video_pruning(self, multimodal_config: MultiModalConfig) -> None:

1749 + pruning_spec = multimodal_config.get_video_pruning_spec()

1750 + if pruning_spec is None:

1751 + self.video_pruning_method: VideoPruningMethod | None = None

1752 + self.video_pruning_rate = multimodal_config.video_pruning_rate

1753 + else:

1754 + self.video_pruning_method, self.video_pruning_rate = pruning_spec

1755 + self.is_multimodal_pruning_enabled = (

1756 + multimodal_config.is_multimodal_pruning_enabled()

1757 + )

1758 +

1759 def __init__(self, *, vllm_config: VllmConfig, prefix: str = "model"):

1760 super().__init__()

1761 config: Qwen3VLConfig = vllm_config.model_config.hf_config

@@ -1730,10 +1767,7 @@ class Qwen3VLForConditionalGeneration(

1767 self._tokenizer = cached_tokenizer_from_config(vllm_config.model_config)

1768 self.multimodal_config = multimodal_config

1769 self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"

1733 - self.video_pruning_rate = multimodal_config.video_pruning_rate

1734 - self.is_multimodal_pruning_enabled = (

1735 - multimodal_config.is_multimodal_pruning_enabled()

1736 - )

1770 + self._init_video_pruning(multimodal_config)

1771

1772 self.use_deepstack = hasattr(config.vision_config, "deepstack_visual_indexes")

1773 self.deepstack_num_level = (

@@ -1848,7 +1882,7 @@ class Qwen3VLForConditionalGeneration(

1882 EncoderCudaGraphConfig,

1883 )

1884

1851 - # When EVS pruning is enabled, embed_multimodal post-processes both

1885 + # When video pruning is enabled, embed_multimodal post-processes both

1886 # image and video embeddings (mrope positions are appended for image,

1887 # prune+append for video). The encoder CUDA graph path bypasses that

1888 # post-process, producing inconsistent embedding formats vs eager. So

@@ -2291,9 +2325,12 @@ class Qwen3VLForConditionalGeneration(

2325

2326 t, h, w = size

2327 if self.is_multimodal_pruning_enabled:

2294 - # For each video, compute retention mask using EVS.

2295 - # retention_mask: [11424].

2296 - retention_mask = compute_retention_mask(

2328 + # Compute the retention mask for each video (EVS or VidCom2).

2329 + if self.video_pruning_method == "vidcom2":

2330 + mask_fn = vidcom2_compute_retention_mask

2331 + else:

2332 + mask_fn = compute_retention_mask

2333 + retention_mask = mask_fn(

2334 emb,

2335 size,

2336 spatial_merge_size=self.visual.spatial_merge_size,

vllm/model_executor/models/qwen3_vl_moe.py

+1 -4

@@ -220,10 +220,7 @@ class Qwen3VLMoeForConditionalGeneration(

220 self._tokenizer = cached_tokenizer_from_config(vllm_config.model_config)

221 self.multimodal_config = multimodal_config

222 self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"

223 - self.video_pruning_rate = multimodal_config.video_pruning_rate

224 - self.is_multimodal_pruning_enabled = (

225 - multimodal_config.is_multimodal_pruning_enabled()

226 - )

223 + self._init_video_pruning(multimodal_config)

224

225 self.use_deepstack = hasattr(config.vision_config, "deepstack_visual_indexes")

226 self.deepstack_num_level = (

vllm/model_executor/models/registry.py

@@ -797,6 +797,7 @@ class _ModelInfo:

797 supports_replayssm: bool

798 supports_transcription: bool

799 supports_transcription_only: bool

800 + supported_video_pruning_methods: tuple[str, ...]

801

802 @staticmethod

803 def from_model_cls(model: type[nn.Module]) -> "_ModelInfo":

@@ -827,6 +828,9 @@ class _ModelInfo:

828 supports_transcription(model) and model.supports_transcription_only

829 ),

830 has_noops=has_noops(model),

831 + supported_video_pruning_methods=getattr(

832 + model, "supported_video_pruning_methods", ()

833 + ),

834 )

835

836

vllm/multimodal/video_prune/__init__.py new

@@ -0,0 +1,2 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

vllm/multimodal/video_prune/evs.py renamed

vllm/multimodal/video_prune/vidcom2.py new

+124

@@ -0,0 +1,124 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +

4 +# VidCom2 (Video Compression Commander) video token pruning.

5 +# Liu et al., EMNLP 2025 — https://arxiv.org/abs/2505.14454

6 +# Adapted from the reference implementation:

7 +# https://github.com/xuyang-liu16/VidCom2 (Apache-2.0,

9 +

10 +import torch

11 +import torch.nn.functional as F

12 +

13 +# Multi-scale Gaussian bandwidths from the reference implementation.

14 +_ALPHAS: tuple[float, ...] = tuple(2.0**k for k in range(-3, 2))

15 +_LOW_VAR_CHANNEL_RATIO: float = 0.5

16 +_SOFTMAX_TEMPERATURE: float = 0.01

17 +

18 +

19 +def compute_retained_tokens_count(

20 + tokens_per_frame: int, num_frames: int, q: float

21 +) -> int:

22 + """Number of video tokens retained after VidCom2 pruning.

23 +

24 + The target is `(1 - q) * total_tokens`, i.e. a retention ratio of

25 + `1 - q` averaged across frames. Because the per-frame budget is floored

26 + at one token, the global minimum is `num_frames` (one token per frame).

27 + """

28 + total_tokens = tokens_per_frame * num_frames

29 + base_num = int(total_tokens * (1.0 - q))

30 + return max(num_frames, min(base_num, total_tokens))

31 +

32 +

33 +def compute_retention_mask(

34 + video_embeds: torch.Tensor,

35 + video_size_thw: torch.LongTensor | tuple[int, int, int],

36 + spatial_merge_size: int,

37 + q: float,

38 +) -> torch.Tensor:

39 + """Compute the VidCom2 retention mask for a single video.

40 +

41 + Args:

42 + video_embeds: `(T*H*W/merge^2, hidden_size)` post-ViT token features.

43 + video_size_thw: `(T, H, W)` grid dimensions.

44 + spatial_merge_size: ViT spatial merge factor (e.g. 2).

45 + q: Pruning fraction in `[0, 1)`; retention ratio is `1 - q`.

46 +

47 + Returns:

48 + Flat bool tensor of shape `(T*H*W/merge^2,)`, True for retained

49 + tokens. The True count equals `compute_retained_tokens_count` so

50 + placeholders sized at prompt-processing time match exactly.

51 + """

52 + T, H, W = map(int, video_size_thw)

53 + rows = H // spatial_merge_size

54 + cols = W // spatial_merge_size

55 + tokens_per_frame = rows * cols

56 + total_tokens = T * tokens_per_frame

57 +

58 + device = video_embeds.device

59 + if tokens_per_frame == 0 or total_tokens == 0:

60 + return torch.ones(0, dtype=torch.bool, device=device)

61 +

62 + target_retained = compute_retained_tokens_count(

63 + tokens_per_frame=tokens_per_frame, num_frames=T, q=q

64 + )

65 + target_retained = min(target_retained, total_tokens)

66 +

67 + # 1. Score in the lowest-variance half of channels.

68 + variances = video_embeds.var(dim=0, unbiased=False)

69 + k_channels = max(1, int(video_embeds.size(-1) * _LOW_VAR_CHANNEL_RATIO))

70 + _, low_var_idx = torch.topk(variances, k=k_channels, largest=False)

71 + sel = video_embeds.index_select(-1, low_var_idx)

72 +

73 + # 2. Multi-scale Gaussian similarity to video and per-frame centers.

74 + frames = sel.view(T, tokens_per_frame, sel.size(-1))

75 + frames = F.normalize(frames, dim=-1)

76 + vid_center = frames.mean(dim=(0, 1), keepdim=True) # (1, 1, C)

77 + frame_center = frames.mean(dim=1, keepdim=True) # (T, 1, C)

78 + v_score = _multi_scale_gaussian(frames, vid_center)

79 + f_score = _multi_scale_gaussian(frames, frame_center)

80 + # Higher similarity = more redundant; lowest-similarity tokens are kept.

81 + similarity = v_score + f_score # (T, tpf)

82 +

83 + # 3. Per-frame dynamic budget: distinctive frames get a larger share.

84 + base = 1.0 - q

85 + frame_scores = -v_score.mean(dim=-1) # (T,)

86 + probs = F.softmax((frame_scores - frame_scores.max()) / _SOFTMAX_TEMPERATURE, dim=0)

87 + scales = (base * (1.0 + probs - probs.mean())).clamp(max=1.0)

88 + ks = (scales * tokens_per_frame).round().long().clamp(min=1, max=tokens_per_frame)

89 +

90 + # 4. Retain the smallest-similarity tokens per frame.

91 + mask_2d = torch.zeros(T, tokens_per_frame, dtype=torch.bool, device=device)

92 + for i in range(T):

93 + k_i = int(ks[i].item())

94 + if k_i <= 0:

95 + continue

96 + _, idx = torch.topk(similarity[i], k=k_i, largest=False, sorted=False)

97 + mask_2d[i].scatter_(0, idx, True)

98 +

99 + # 5. Reconcile rounding/clamp drift to the exact target count by score.

100 + flat_mask = mask_2d.view(-1)

101 + flat_sim = similarity.view(-1)

102 + current = int(flat_mask.sum().item())

103 + if current > target_retained:

104 + drop_n = current - target_retained

105 + retained_idx = flat_mask.nonzero(as_tuple=False).squeeze(-1)

106 + retained_sim = flat_sim[retained_idx]

107 + _, worst = torch.topk(retained_sim, k=drop_n, largest=True, sorted=False)

108 + flat_mask[retained_idx[worst]] = False

109 + elif current < target_retained:

110 + add_n = target_retained - current

111 + available_idx = (~flat_mask).nonzero(as_tuple=False).squeeze(-1)

112 + if available_idx.numel() > 0:

113 + available_sim = flat_sim[available_idx]

114 + add_n = min(add_n, available_idx.numel())

115 + _, best = torch.topk(available_sim, k=add_n, largest=False, sorted=False)

116 + flat_mask[available_idx[best]] = True

117 +

118 + return flat_mask

119 +

120 +

121 +def _multi_scale_gaussian(x: torch.Tensor, center: torch.Tensor) -> torch.Tensor:

122 + """Sum Gaussian kernels over `_ALPHAS`; `(T, N, C) -> (T, N)` scores."""

123 + dist_sq = ((x - center) ** 2).sum(dim=-1)

124 + return sum(torch.exp(-dist_sq / (2.0 * a)) for a in _ALPHAS)

vllm/transformers_utils/processors/nano_nemotron_vl.py

+1 -1

@@ -23,9 +23,9 @@ from PIL import Image

23 from transformers import BatchFeature, PretrainedConfig, TensorType

25 from vllm.model_executor.models.parakeet import ParakeetExtractor

26 -from vllm.multimodal.evs import compute_retained_tokens_count

26 from vllm.multimodal.inputs import AudioItem

27 from vllm.multimodal.processing.processor import PromptUpdateDetails

28 +from vllm.multimodal.video_prune.evs import compute_retained_tokens_count

29 from vllm.tokenizers.hf import HfTokenizer

31 from .internvl import calculate_internvl_targets, get_internvl_target_ratios