@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 70c73df69e

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 7, 07:45 UTC

[Bugfix] Fix EVS implementation for Qwen3 VL (#33607)

Signed-off-by: 2ez4bz <133824995+2ez4bz@users.noreply.github.com>

William Zhang committed Mar 3, 2026 at 18:18 UTC 70c73df69ee28fec37781d9bc82a994619ab95b1

7 files changed +895 -272

tests/model_executor/test_qwen3_vl_mrope.py new

+237

@@ -0,0 +1,237 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +import dataclasses

4 +import random

5 +from dataclasses import dataclass

6 +

7 +import pytest

8 +import torch

9 +

10 +from vllm.model_executor.models.qwen3_vl import Qwen3VLForConditionalGeneration

11 +from vllm.multimodal.inputs import (

12 + MultiModalFeatureSpec,

13 + MultiModalFieldElem,

14 + MultiModalKwargsItem,

15 + PlaceholderRange,

16 +)

17 +

18 +

19 +@pytest.fixture(autouse=True, scope="module")

20 +def _force_cpu_default_device():

21 + # _get_mrope_input_positions returns CPU tensors (via torch.from_numpy).

22 + # Ensure the default device is CPU so the rest of the test tensors match.

23 + original = torch.get_default_device()

24 + torch.set_default_device("cpu")

25 + yield

26 + torch.set_default_device(original)

27 +

28 +

29 +IMAGE_TOKEN_ID = 999

30 +VIDEO_TOKEN_ID = 888

31 +VISION_START_TOKEN_ID = 777

32 +VISION_END_TOKEN_ID = 778

33 +

34 +

35 +@dataclass

36 +class DummyVisionConfig:

37 + spatial_merge_size: int = 1

38 +

39 +

40 +@dataclass

41 +class DummyConfig:

42 + image_token_id: int = IMAGE_TOKEN_ID

43 + video_token_id: int = VIDEO_TOKEN_ID

44 + vision_start_token_id: int = VISION_START_TOKEN_ID

45 + vision_end_token_id: int = VISION_END_TOKEN_ID

46 + vision_config: DummyVisionConfig = dataclasses.field(

47 + default_factory=DummyVisionConfig

48 + )

49 +

50 +

51 +def make_video_embedding(

52 + t, h, w, interleave_text_tokens: tuple[int, int], video_pruning_rate: float = 0.0

53 +):

54 + """

55 + Helper function to make a video embedding for a given video size and pruning rate.

56 +

57 + Args:

58 + t: Number of frames.

59 + h: Number of rows.

60 + w: Number of columns.

61 + interleave_text_tokens: Tuple of minimum and maximum number of text tokens to

62 + interleave with the video.

63 + video_pruning_rate: Pruning rate for the video.

64 +

65 + Returns:

66 + Tuple of (unpruned_tokens_sequence, pruned_tokens_sequence, retention_mask)

67 + """

68 + unpruned_tokens_sequence = []

69 + population = list(range(1, 100))

70 +

71 + for _ in range(t):

72 + num_prefix_tokens = random.randint(

73 + interleave_text_tokens[0], interleave_text_tokens[1]

74 + )

75 +

76 + prefix_tokens = random.choices(population, k=num_prefix_tokens)

77 + vision_tokens = (

78 + [VISION_START_TOKEN_ID] + [VIDEO_TOKEN_ID] * h * w + [VISION_END_TOKEN_ID]

79 + )

80 +

81 + unpruned_tokens_sequence.extend(prefix_tokens)

82 + unpruned_tokens_sequence.extend(vision_tokens)

83 +

84 + unpruned_tokens_sequence = torch.tensor(unpruned_tokens_sequence, dtype=torch.long)

85 + video_token_mask = unpruned_tokens_sequence == VIDEO_TOKEN_ID

86 +

87 + pruning_mask = torch.bernoulli(video_token_mask.float() * video_pruning_rate).bool() # type: ignore[attr-defined]

88 + # Sanity check that we don't prune what should not be pruned.

89 + assert not pruning_mask[~video_token_mask].any()

90 +

91 + retention_mask = ~pruning_mask

92 + pruned_tokens_sequence = unpruned_tokens_sequence[retention_mask]

93 + return unpruned_tokens_sequence, pruned_tokens_sequence, retention_mask

94 +

95 +

96 +@pytest.mark.parametrize("spatial_merge_size", [1, 2])

97 +@pytest.mark.parametrize("grid_thw", [[3, 8, 7], [128, 10, 12]])

98 +@pytest.mark.parametrize("num_prefix_tokens", [1, 11])

99 +@pytest.mark.parametrize("num_suffix_tokens", [0, 7])

100 +@pytest.mark.parametrize("video_pruning_rate", [0, 0.25, 0.75])

101 +@pytest.mark.parametrize("interleave_text_tokens", [(0, 0), (1, 4)])

102 +def test_match_qwen3vl_mrope_evs_on(

103 + spatial_merge_size: int,

104 + num_prefix_tokens: int,

105 + grid_thw: tuple[int, int, int],

106 + num_suffix_tokens: int,

107 + video_pruning_rate: float,

108 + interleave_text_tokens: tuple[int, int],

109 +):

110 + hf_config = DummyConfig()

111 + hf_config.vision_config.spatial_merge_size = spatial_merge_size

112 +

113 + t, h, w = grid_thw

114 + population = list(range(1, 100))

115 + prefix_tokens = random.choices(population, k=num_prefix_tokens)

116 + suffix_tokens = random.choices(population, k=num_suffix_tokens)

117 +

118 + video_tokens, video_tokens_pruned, retention_mask = make_video_embedding(

119 + t,

120 + h // spatial_merge_size,

121 + w // spatial_merge_size,

122 + interleave_text_tokens=interleave_text_tokens,

123 + video_pruning_rate=video_pruning_rate,

124 + )

125 + assert len(video_tokens) == len(retention_mask)

126 +

127 + input_tokens = prefix_tokens + video_tokens.tolist() + suffix_tokens

128 + input_tokens_pruned = prefix_tokens + video_tokens_pruned.tolist() + suffix_tokens

129 +

130 + whole_sequence_retention_mask = torch.cat(

131 + [

132 + torch.ones(len(prefix_tokens), dtype=torch.bool),

133 + retention_mask,

134 + torch.ones(len(suffix_tokens), dtype=torch.bool),

135 + ],

136 + dim=0,

137 + )

138 +

139 + # Build the GT mrope for unpruned input.

140 + mm_feature = MultiModalFeatureSpec(

141 + data=MultiModalKwargsItem(

142 + {

143 + "video_grid_thw": MultiModalFieldElem(

144 + data=torch.tensor(grid_thw),

145 + field=None, # HACK.

146 + ),

147 + }

148 + ),

149 + modality="video",

150 + identifier="DUMMY",

151 + mm_position=PlaceholderRange(offset=0, length=len(input_tokens)),

152 + )

153 + expected_mrope, _ = Qwen3VLForConditionalGeneration._get_mrope_input_positions(

154 + input_tokens=input_tokens,

155 + mm_features=[mm_feature],

156 + config=hf_config,

157 + )

158 +

159 + # Compute mrope for a video-only media (unpruned).

160 + mm_feature = MultiModalFeatureSpec(

161 + data=MultiModalKwargsItem(

162 + {

163 + "video_grid_thw": MultiModalFieldElem(

164 + data=torch.tensor(grid_thw),

165 + field=None, # HACK.

166 + ),

167 + }

168 + ),

169 + modality="video",

170 + identifier="DUMMY",

171 + mm_position=PlaceholderRange(offset=0, length=video_tokens.numel()),

172 + )

173 + video_mrope, _ = Qwen3VLForConditionalGeneration._get_mrope_input_positions(

174 + input_tokens=video_tokens.tolist(),

175 + mm_features=[mm_feature],

176 + config=hf_config,

177 + )

178 + video_mrope = video_mrope.permute(1, 0) # [N, 3]

179 + hidden_size = 16

180 +

181 + is_video_embed = torch.isin(

182 + video_tokens_pruned, torch.tensor([VIDEO_TOKEN_ID], dtype=torch.long)

183 + )

184 +

185 + expanded_positions = torch.full(

186 + (len(video_tokens_pruned), 5),

187 + fill_value=-100,

188 + device=video_mrope.device,

189 + dtype=torch.long,

190 + )

191 + expanded_positions[is_video_embed, :3] = video_mrope[retention_mask][is_video_embed]

192 + expanded_positions[~is_video_embed, :3] = video_mrope[retention_mask][

193 + ~is_video_embed

194 + ]

195 +

196 + is_vision_start = video_tokens_pruned == VISION_START_TOKEN_ID

197 + expanded_positions[..., 3] = is_vision_start

198 + expanded_positions[..., 4] = is_video_embed

199 +

200 + # Check that all positions were filled, since we initialized them as negative.

201 + assert (expanded_positions >= 0).all()

202 +

203 + video_embeddings = torch.empty(

204 + (len(video_tokens_pruned), hidden_size), device=video_mrope.device

205 + )

206 +

207 + video_embeddings = torch.cat(

208 + [

209 + video_embeddings,

210 + expanded_positions.float(),

211 + ],

212 + dim=1,

213 + )

214 + multimodal_embeddings = [video_embeddings]

215 +

216 + expected_mrope_masked = expected_mrope[:, whole_sequence_retention_mask]

217 +

218 + # Initialize computed_mrope with sequential positions for all prefix tokens

219 + computed_mrope = torch.empty((3, len(input_tokens_pruned)), dtype=torch.long)

220 + computed_mrope[:, 0 : len(prefix_tokens)] = expected_mrope[

221 + :, 0 : len(prefix_tokens)

222 + ]

223 +

224 + # Paranoia check that computed_mrope is wrong.

225 + assert not torch.equal(computed_mrope, expected_mrope_masked)

226 +

227 + _, actual_mrope, _ = Qwen3VLForConditionalGeneration._recompute_mrope_positions(

228 + input_ids=input_tokens_pruned,

229 + multimodal_embeddings=multimodal_embeddings,

230 + mrope_positions=computed_mrope,

231 + num_computed_tokens=len(prefix_tokens),

232 + vision_start_token_id=hf_config.vision_start_token_id,

233 + image_token_id=hf_config.image_token_id,

234 + video_token_id=hf_config.video_token_id,

235 + )

236 +

237 + assert torch.equal(actual_mrope, expected_mrope_masked)

vllm/model_executor/models/qwen2_5_vl.py

@@ -195,6 +195,8 @@ class Qwen2_5_VLVideoPixelInputs(TensorSchema):

195 - second_per_grid_ts: The video time interval (in seconds) for each

196 grid along the temporal dimension in the 3D position IDs. Returned

197 when `videos` is not `None`.

198 + - timestamps: List of timestamp values (in seconds) for each frame

199 + after merging. Length equals the temporal dimension after merging.

200 """

201

202 type: Literal["pixel_values_videos"]

@@ -214,6 +216,8 @@ class Qwen2_5_VLVideoPixelInputs(TensorSchema):

216 TensorShape("nv"),

217 ]

218

219 + timestamps: list[list[float]] | None = None

220 +

221

222 class Qwen2_5_VLVideoEmbeddingInputs(TensorSchema):

223 """

@@ -232,6 +236,8 @@ class Qwen2_5_VLVideoEmbeddingInputs(TensorSchema):

236 - second_per_grid_ts: The video time interval (in seconds) for each

237 grid along the temporal dimension in the 3D position IDs. Returned

238 when `videos` is not `None`.

239 + - timestamps: List of timestamp values (in seconds) for each frame

240 + after merging. Length equals the temporal dimension after merging.

241 """

242

243 type: Literal["video_embeds"]

@@ -250,6 +256,7 @@ class Qwen2_5_VLVideoEmbeddingInputs(TensorSchema):

256 torch.Tensor | None,

257 TensorShape("nv"),

258 ] = None

259 + timestamps: list[list[float]] | None = None

260

261

262 Qwen2_5_VLVideoInputs: TypeAlias = (

vllm/model_executor/models/qwen2_vl.py

@@ -755,6 +755,7 @@ def _create_qwen2vl_field_factory(

755 "video", video_embed_grid_sizes

756 ),

757 video_grid_thw=MultiModalFieldConfig.batched("video", keep_on_cpu=True),

758 + timestamps=MultiModalFieldConfig.batched("video", keep_on_cpu=True),

759 )

760

761 return _qwen2vl_field_config

vllm/model_executor/models/qwen3_5.py

+13 -8

@@ -628,6 +628,9 @@ class Qwen3_5MoeForCausalLM(Qwen3_5ForCausalLMBase, QwenNextMixtureOfExperts):

628 dummy_inputs=Qwen3VLDummyInputsBuilder,

629 )

630 class Qwen3_5ForConditionalGeneration(Qwen3VLForConditionalGeneration, IsHybrid):

631 + # Qwen3.5 does not support multimodal pruning (EVS).

632 + supports_multimodal_pruning = False

633 +

634 packed_modules_mapping = Qwen3VLForConditionalGeneration.packed_modules_mapping | {

635 "in_proj_qkvz": ["in_proj_qkv", "in_proj_z"],

636 "in_proj_ba": ["in_proj_b", "in_proj_a"],

@@ -643,10 +646,8 @@ class Qwen3_5ForConditionalGeneration(Qwen3VLForConditionalGeneration, IsHybrid)

646 self.config = config

647 self.multimodal_config = multimodal_config

648 self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"

646 - self.video_pruning_rate = multimodal_config.video_pruning_rate

647 - self.is_multimodal_pruning_enabled = (

648 - multimodal_config.is_multimodal_pruning_enabled()

649 - )

649 + # Qwen3.5 does not support multimodal pruning (EVS).

650 + self.is_multimodal_pruning_enabled = False

651

652 with self._mark_tower_model(vllm_config, {"image", "video"}):

653 self.visual = Qwen3_VisionTransformer(

@@ -693,6 +694,12 @@ class Qwen3_5ForConditionalGeneration(Qwen3VLForConditionalGeneration, IsHybrid)

694

695 return inputs_embeds

696

697 + def recompute_mrope_positions(self, *args, **kwargs):

698 + raise NotImplementedError(

699 + "Qwen3.5 does not support multimodal pruning (EVS). "

700 + "recompute_mrope_positions should never be called."

701 + )

702 +

703 def forward(

704 self,

705 input_ids: torch.Tensor,

@@ -851,10 +858,8 @@ class Qwen3_5MoeForConditionalGeneration(

858 self.config = config

859 self.multimodal_config = multimodal_config

860 self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"

854 - self.video_pruning_rate = multimodal_config.video_pruning_rate

855 - self.is_multimodal_pruning_enabled = (

856 - multimodal_config.is_multimodal_pruning_enabled()

857 - )

861 + # Qwen3.5 does not support multimodal pruning (EVS).

862 + self.is_multimodal_pruning_enabled = False

863

864 with self._mark_tower_model(vllm_config, {"image", "video"}):

865 self.visual = Qwen3_VisionTransformer(

vllm/model_executor/models/qwen3_vl.py

+565 -256

@@ -79,6 +79,7 @@ from vllm.multimodal.inputs import (

79 MultiModalDataDict,

80 MultiModalFeatureSpec,

81 MultiModalFieldConfig,

82 + MultiModalFieldElem,

83 MultiModalKwargsItem,

84 MultiModalKwargsItems,

85 PlaceholderRange,

@@ -93,6 +94,8 @@ from vllm.multimodal.processing import (

94 PromptUpdateDetails,

95 )

96 from vllm.sequence import IntermediateTensors

97 +from vllm.tokenizers.protocol import TokenizerLike

98 +from vllm.tokenizers.registry import cached_tokenizer_from_config

99 from vllm.utils.collection_utils import is_list_of

100 from vllm.utils.math_utils import round_up

101

@@ -763,7 +766,6 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):

766 def _get_video_second_idx(

767 self,

768 metadata: dict[str, Any],

766 - out_item: MultiModalKwargsItem,

769 do_sample_frames: bool | None = None,

770 sampled_fps: float | None = None,

771 ) -> list[int]:

@@ -956,6 +958,7 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])

958 if videos := mm_data.pop("videos", []):

959 video_grid_thw_lst = []

960 pixel_values_videos_lst = []

961 + timestamps_per_video = []

962

963 for item in videos:

964 video_array, metadata = item

@@ -979,6 +982,14 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])

982 **{k: metadata[k] for k in metadata if k != "do_sample_frames"}

983 )

984

985 + # Compute timestamps here where we have access to metadata

986 + timestamps = self.info._get_video_second_idx(

987 + metadata=metadata,

988 + do_sample_frames=video_mm_kwargs["do_sample_frames"],

989 + sampled_fps=video_mm_kwargs.get("fps"),

990 + )

991 + timestamps_per_video.append(timestamps)

992 +

993 video_mm_data = dict()

994 video_mm_data["videos"] = [[video_array]]

995 video_mm_data["video_metadata"] = [[metadata]]

@@ -989,6 +1000,49 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])

1000 mm_kwargs=video_mm_kwargs,

1001 tok_kwargs=tok_kwargs,

1002 )

1003 +

1004 + merge_size = processor.video_processor.merge_size

1005 + # Get video grid info for EVS calculation.

1006 + video_grid_thw = video_outputs["video_grid_thw"]

1007 + num_frames = int(video_grid_thw[0, 0])

1008 + tokens_per_frame_base = int(video_grid_thw[0, 1:].prod()) // (

1009 + merge_size**2

1010 + )

1011 +

1012 + # Apply EVS if enabled.

1013 + video_pruning_rate = self.info.ctx.get_mm_config().video_pruning_rate

1014 + if video_pruning_rate is not None and video_pruning_rate > 0.0:

1015 + num_tokens = compute_retained_tokens_count(

1016 + tokens_per_frame=tokens_per_frame_base,

1017 + num_frames=num_frames,

1018 + q=video_pruning_rate,

1019 + )

1020 + # Here we just need placeholders that won't actually be replaced -

1021 + # we just need to make sure the total number of tokens is correct

1022 + # assign all tokens to the first frame.

1023 + tokens_per_frame = [num_tokens] + [0] * (num_frames - 1)

1024 + select_token_id = False

1025 + else:

1026 + tokens_per_frame = [tokens_per_frame_base] * num_frames

1027 + select_token_id = True

1028 +

1029 + # Generate the video replacement with EVS-adjusted token counts

1030 + tokenizer = self.info.get_tokenizer()

1031 + hf_config = self.info.get_hf_config()

1032 + video_repl = Qwen3VLMultiModalProcessor.get_video_repl(

1033 + tokens_per_frame=tokens_per_frame,

1034 + timestamps=timestamps,

1035 + tokenizer=tokenizer,

1036 + vision_start_token_id=hf_config.vision_start_token_id,

1037 + vision_end_token_id=hf_config.vision_end_token_id,

1038 + video_token_id=hf_config.video_token_id,

1039 + select_token_id=select_token_id,

1040 + )

1041 +

1042 + # Convert token IDs to text for the HF processor flow

1043 + video_placeholder = tokenizer.decode(

1044 + video_repl.full, skip_special_tokens=False

1045 + )

1046 input_ids = video_outputs.pop("input_ids")

1047 video_placeholder = processor.tokenizer.batch_decode(input_ids)[0]

1048 prompt = prompt.replace(

@@ -1002,6 +1056,7 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])

1056 video_outputs = dict(

1057 pixel_values_videos=torch.cat(pixel_values_videos_lst),

1058 video_grid_thw=torch.cat(video_grid_thw_lst),

1059 + timestamps=timestamps_per_video,

1060 )

1061 else:

1062 video_outputs = dict()

@@ -1057,60 +1112,42 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])

1112 grid_thw = out_item["video_grid_thw"].data

1113 assert isinstance(grid_thw, torch.Tensor)

1114

1060 - video, metadata = mm_items["video"][item_idx]

1061 - do_sample_frames = hf_processor_mm_kwargs.get("do_sample_frames")

1115 sampled_fps = hf_processor_mm_kwargs.get("fps")

1116 if is_list_of(sampled_fps, float):

1117 sampled_fps = sampled_fps[item_idx]

1065 - timestamps = self.info._get_video_second_idx(

1066 - metadata, out_item, do_sample_frames, sampled_fps

1067 - )

1118

1119 + timestamps = out_item["timestamps"].data

1120 assert len(timestamps) == grid_thw[0], (

1121 f"The timestamps length({len(timestamps)}) should be equal "

1122 f"video length ({grid_thw[0]})."

1123 )

1124

1074 - frames_idx_token = [

1075 - tokenizer.encode(f"<{curr_time:.1f} seconds>", add_special_tokens=False)

1076 - for curr_time in timestamps

1077 - ]

1078 - tokens_per_frame = int(grid_thw[1:].prod()) // merge_length

1079 - per_frame_token_counts = [tokens_per_frame for _ in frames_idx_token]

1125 + # Compute tokens per frame, with EVS support

1126 + num_frames = int(grid_thw[0])

1127 + tokens_per_frame_base = int(grid_thw[1:].prod()) // merge_length

1128

1129 video_pruning_rate = self.info.ctx.get_mm_config().video_pruning_rate

1130 if video_pruning_rate is not None and video_pruning_rate > 0.0:

1083 - total_retained = compute_retained_tokens_count(

1084 - tokens_per_frame,

1085 - len(frames_idx_token),

1086 - video_pruning_rate,

1131 + num_tokens = compute_retained_tokens_count(

1132 + tokens_per_frame=tokens_per_frame_base,

1133 + num_frames=num_frames,

1134 + q=video_pruning_rate,

1135 )

1088 - if len(frames_idx_token) == 0:

1089 - per_frame_token_counts = []

1090 - elif len(frames_idx_token) == 1:

1091 - per_frame_token_counts = [tokens_per_frame]

1092 - else:

1093 - first_frame_tokens = tokens_per_frame

1094 - remaining_tokens = max(total_retained - first_frame_tokens, 0)

1095 - base = remaining_tokens // (len(frames_idx_token) - 1)

1096 - remainder = remaining_tokens % (len(frames_idx_token) - 1)

1097 - per_frame_token_counts = [first_frame_tokens]

1098 - for frame_idx in range(1, len(frames_idx_token)):

1099 - extra = base + (1 if (frame_idx - 1) < remainder else 0)

1100 - per_frame_token_counts.append(extra)

1101 -

1102 - placeholder = []

1103 - for frame_idx, timestamp_tokens in enumerate(frames_idx_token):

1104 - placeholder.extend(timestamp_tokens)

1105 - tokens_this_frame = per_frame_token_counts[

1106 - frame_idx if frame_idx < len(per_frame_token_counts) else -1

1107 - ]

1108 - placeholder.extend(

1109 - [vision_start_token_id]

1110 - + [video_token_id] * tokens_this_frame

1111 - + [vision_end_token_id]

1112 - )

1113 - return PromptUpdateDetails.select_token_id(placeholder, video_token_id)

1136 + tokens_per_frame = [num_tokens] + [0] * (num_frames - 1)

1137 + select_token_id = False

1138 + else:

1139 + tokens_per_frame = [tokens_per_frame_base] * num_frames

1140 + select_token_id = True

1141 +

1142 + return Qwen3VLMultiModalProcessor.get_video_repl(

1143 + tokens_per_frame=tokens_per_frame,

1144 + timestamps=timestamps,

1145 + tokenizer=tokenizer,

1146 + vision_start_token_id=vision_start_token_id,

1147 + vision_end_token_id=vision_end_token_id,

1148 + video_token_id=video_token_id,

1149 + select_token_id=select_token_id,

1150 + )

1151

1152 return [

1153 PromptReplacement(

@@ -1127,6 +1164,69 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])

1164 ),

1165 ]

1166

1167 + @staticmethod

1168 + def get_video_repl(

1169 + *,

1170 + tokens_per_frame: list[int],

1171 + timestamps: list[float | int],

1172 + tokenizer: TokenizerLike,

1173 + vision_start_token_id: int,

1174 + vision_end_token_id: int,

1175 + video_token_id: int,

1176 + select_token_id: bool = False,

1177 + ) -> PromptUpdateDetails[list[int]]:

1178 + """Build prompt replacement for a video in Qwen3VL format.

1179 +

1180 + The replacement structure for each frame is:

1181 + timestamp_tokens + vision_start_token + video_tokens + vision_end_token

1182 +

1183 + Args:

1184 + tokens_per_frame: Number of video tokens per frame (can vary per frame for

1185 + EVS).

1186 + timestamps: List of timestamps in seconds for each frame

1187 + tokenizer: Tokenizer to encode timestamp strings

1188 + vision_start_token_id: Token ID for vision start marker

1189 + vision_end_token_id: Token ID for vision end marker

1190 + video_token_id: Token ID for video content

1191 +

1192 + Returns:

1193 + PromptUpdateDetails with full token sequence

1194 + """

1195 + assert len(timestamps) == len(tokens_per_frame), (

1196 + "timestamps and tokens_per_frame must have the same length"

1197 + )

1198 +

1199 + # Tokenize timestamp strings independently to avoid tokenizer merging

1200 + # tokens across boundaries.

1201 + # TODO: switch to `_seq2tokens` which has some caching.

1202 + timestamp_token_ids = [

1203 + tokenizer.encode(f"<{timestamp:.1f} seconds>", add_special_tokens=False)

1204 + for timestamp in timestamps

1205 + ]

1206 +

1207 + # Build the full token sequence

1208 + all_token_ids = []

1209 + for frame_timestamp_ids, num_tokens in zip(

1210 + timestamp_token_ids, tokens_per_frame

1211 + ):

1212 + # Add timestamp tokens

1213 + all_token_ids.extend(frame_timestamp_ids)

1214 +

1215 + # Add vision tokens: vision_start + video_tokens + vision_end

1216 + all_token_ids.append(vision_start_token_id)

1217 + all_token_ids.extend([video_token_id] * num_tokens)

1218 + all_token_ids.append(vision_end_token_id)

1219 +

1220 + if select_token_id:

1221 + return PromptUpdateDetails.select_token_id(all_token_ids, video_token_id)

1222 +

1223 + # NOTE: we use `from_seq` instead of `select_token_id` because we want all

1224 + # tokens in the placeholder to be initially marked as candidates. Then

1225 + # in `get_input_embeddings``, we refine the mask to only replace

1226 + # `video_token_id` / `image_token_id`` positions with video/image embeddings,

1227 + # keeping text embeddings for timestamps and structural tokens.

1228 + return PromptUpdateDetails.from_seq(all_token_ids)

1229 +

1230

1231 @support_torch_compile(

1232 dynamic_arg_dims={

@@ -1280,6 +1380,7 @@ class Qwen3VLForConditionalGeneration(

1380 multimodal_config = vllm_config.model_config.multimodal_config

1381

1382 self.config = config

1383 + self._tokenizer = cached_tokenizer_from_config(vllm_config.model_config)

1384 self.multimodal_config = multimodal_config

1385 self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"

1386 self.video_pruning_rate = multimodal_config.video_pruning_rate

@@ -1419,6 +1520,7 @@ class Qwen3VLForConditionalGeneration(

1520 video_embeds = kwargs.pop("video_embeds", None)

1521 video_grid_thw = kwargs.pop("video_grid_thw", None)

1522 second_per_grid_ts = kwargs.pop("second_per_grid_ts", None)

1523 + timestamps = kwargs.pop("timestamps", None)

1524

1525 if pixel_values_videos is None and video_embeds is None:

1526 return None

@@ -1429,6 +1531,7 @@ class Qwen3VLForConditionalGeneration(

1531 pixel_values_videos=pixel_values_videos,

1532 video_grid_thw=video_grid_thw,

1533 second_per_grid_ts=second_per_grid_ts,

1534 + timestamps=timestamps,

1535 )

1536

1537 if video_embeds is not None:

@@ -1436,6 +1539,7 @@ class Qwen3VLForConditionalGeneration(

1539 type="video_embeds",

1540 video_embeds=video_embeds,

1541 video_grid_thw=video_grid_thw,

1542 + timestamps=timestamps,

1543 )

1544

1545 def _process_image_input(

@@ -1502,19 +1606,29 @@ class Qwen3VLForConditionalGeneration(

1606

1607 Returns:

1608 Tuple of image embeddings for each image item.

1505 - Resulting embeddings will have extra 4 channels for

1506 - computed mrope positions.

1609 + Resulting embeddings will have extra 5 channels for

1610 + computed mrope positions, consistent with video embeddings.

1611 """

1508 - merge_size = self.visual.spatial_merge_size

1509 - grid_thw = image_input["image_grid_thw"]

1510 - grid_thw_list = grid_thw.tolist()

1511 - image_embeds_out = []

1512 - for emb, size in zip(image_embeds_split, grid_thw_list):

1513 - positions = compute_mrope_for_media(size, merge_size).to(emb.device)

1514 - emb = torch.cat([emb, positions], dim=1)

1515 - image_embeds_out.append(emb)

1516 - image_embeds_split = image_embeds_out

1517 - return tuple(image_embeds_split)

1612 + if self.is_multimodal_pruning_enabled:

1613 + merge_size = self.visual.spatial_merge_size

1614 + grid_thw = image_input["image_grid_thw"]

1615 + grid_thw_list = grid_thw.tolist()

1616 + image_embeds_out = []

1617 + for emb, size in zip(image_embeds_split, grid_thw_list):

1618 + positions = compute_mrope_for_media(size, merge_size).to(emb.device)

1619 + positions = torch.cat(

1620 + [

1621 + positions,

1622 + torch.zeros_like(

1623 + positions[:, 0:1]

1624 + ), # Dummy extra fifth channel

1625 + ],

1626 + dim=1,

1627 + )

1628 + emb = torch.cat([emb, positions], dim=1)

1629 + image_embeds_out.append(emb)

1630 + image_embeds_split = tuple(image_embeds_out)

1631 + return image_embeds_split

1632

1633 def _postprocess_video_embeds_evs(

1634 self,

@@ -1531,62 +1645,218 @@ class Qwen3VLForConditionalGeneration(

1645

1646 Returns:

1647 Tuple of video embeddings for each video item.

1534 - Resulting embeddings will have extra 4 channels for

1535 - computed mrope positions.

1648 + Resulting embeddings will have extra 5 channels for computed mrope

1649 + positions, and whether the index corresponds to a video embedding.

1650 """

1651 grid_thw = video_input["video_grid_thw"]

1652 assert grid_thw.ndim == 2

1653 grid_thw_list = grid_thw.tolist()

1654 merge_size = self.visual.spatial_merge_size

1655

1542 - # Cast to long to match the original code

1543 - # https://github.com/huggingface/transformers/blob/41980ce93e775f6c88500c51c8db7946fc6a2add/src/transformers/models/qwen2_5_vl/modular_qwen2_5_vl.py#L491 # noqa

1544 - second_per_grid_ts = video_input.get("second_per_grid_ts")

1545 - if second_per_grid_ts is None:

1546 - # For Qwen3-VL, second_per_grid_ts might not be available

1547 - # Use default value of 1.0 for each video

1548 - second_per_grid_ts = torch.ones(len(grid_thw_list), dtype=torch.long)

1656 + # Apply EVS to each video.

1657 + video_embeds_out = []

1658 + for video_idx, (emb, size) in enumerate(zip(video_embeds_split, grid_thw_list)):

1659 + # Compute positions.

1660 + timestamps = video_input.timestamps[video_idx]

1661 + num_frames = len(timestamps)

1662 +

1663 + t, h, w = size

1664 + if self.is_multimodal_pruning_enabled:

1665 + # For each video, compute retention mask using EVS.

1666 + # retention_mask: [11424].

1667 + retention_mask = compute_retention_mask(

1668 + emb,

1669 + size,

1670 + spatial_merge_size=self.visual.spatial_merge_size,

1671 + q=self.video_pruning_rate,

1672 + )

1673 + # Apply retention mask.

1674 + emb = emb[retention_mask]

1675 +

1676 + # Calculate the actual number of retained tokens per frame.

1677 + num_frames, rows, cols = (

1678 + t,

1679 + h // merge_size,

1680 + w // merge_size,

1681 + )

1682 + retention_mask_thw = retention_mask.reshape(num_frames, rows, cols)

1683 + num_tokens_per_frame = (

1684 + retention_mask_thw.sum(dim=(1, 2)).long().tolist()

1685 + )

1686 + else:

1687 + feature_size = emb.shape[0] // num_frames

1688 + num_tokens_per_frame = [feature_size] * num_frames

1689 + retention_mask = None

1690 +

1691 + emb = self._create_final_video_embeddings(

1692 + video_embeddings=emb,

1693 + num_tokens_per_frame=num_tokens_per_frame,

1694 + timestamps=timestamps,

1695 + video_grid_thw=size,

1696 + retention_mask=retention_mask,

1697 + )

1698 +

1699 + video_embeds_out.append(emb)

1700 +

1701 + return tuple(video_embeds_out)

1702 +

1703 + def _create_final_video_embeddings(

1704 + self,

1705 + video_embeddings: torch.Tensor,

1706 + num_tokens_per_frame: list[int],

1707 + timestamps: list[float],

1708 + video_grid_thw: list[int],

1709 + retention_mask: torch.Tensor,

1710 + ) -> torch.Tensor:

1711 + """Create final embeddings that combine video embeddings with

1712 + text embeddings of indicator tokens.

1713 +

1714 + These final embeddings contain:

1715 + - Actual video embeddings in positions corresponding to video content

1716 + - Text embeddings for indicator tokens (<img>, </img>, and

1717 + frame separation text) in their respective positions

1718 +

1719 + These embeddings will replace the placeholder embeddings to create

1720 + input_embeds for the LLM.

1721 + """

1722 + device = video_embeddings.device

1723 +

1724 + # Generate video replacement token IDs using get_video_repl

1725 + # This tokenizes each frame separator independently, then uses pre-tokenized

1726 + # special tokens to ensure consistent tokenization regardless of

1727 + # num_tokens_per_frame values.

1728 + video_repl = Qwen3VLMultiModalProcessor.get_video_repl(

1729 + tokens_per_frame=num_tokens_per_frame,

1730 + tokenizer=self._tokenizer,

1731 + timestamps=timestamps,

1732 + vision_start_token_id=self.config.vision_start_token_id,

1733 + vision_end_token_id=self.config.vision_end_token_id,

1734 + video_token_id=self.config.video_token_id,

1735 + select_token_id=self.is_multimodal_pruning_enabled,

1736 + )

1737 +

1738 + repl_token_ids = torch.tensor(video_repl.full, device=device)

1739 + embed_token_id = _cached_tensor(self.config.video_token_id, device=device)

1740 + is_video_embed = torch.isin(repl_token_ids, embed_token_id)

1741 +

1742 + # Get text embeddings for indicator tokens (has only `visual_dim``).

1743 + text_embeddings = self.get_language_model().embed_input_ids(repl_token_ids)

1744 +

1745 + if self.use_deepstack:

1746 + (

1747 + deepstack_input_embeds,

1748 + multimodal_embeddings,

1749 + ) = self._compute_deepstack_embeds(

1750 + inputs_embeds=text_embeddings,

1751 + multimodal_embeddings=[video_embeddings],

1752 + is_multimodal=is_video_embed,

1753 + )

1754 else:

1550 - second_per_grid_ts = second_per_grid_ts.long()

1551 - tokens_per_second = getattr(self.config.vision_config, "tokens_per_second", 1.0)

1755 + deepstack_input_embeds = None

1756 + multimodal_embeddings = [video_embeddings]

1757

1553 - video_embeds_out = []

1554 - for emb, size, video_second_per_grid_t in zip(

1555 - video_embeds_split, grid_thw_list, second_per_grid_ts

1556 - ):

1557 - # For each video, we compute retention mask using EVS

1558 - retention_mask = compute_retention_mask(

1559 - emb,

1560 - size,

1561 - spatial_merge_size=self.visual.spatial_merge_size,

1562 - q=self.video_pruning_rate,

1758 + merged_embeddings = _merge_multimodal_embeddings(

1759 + inputs_embeds=text_embeddings,

1760 + multimodal_embeddings=multimodal_embeddings,

1761 + is_multimodal=is_video_embed,

1762 + )

1763 +

1764 + to_concat = [merged_embeddings]

1765 + if deepstack_input_embeds is not None:

1766 + to_concat.append(

1767 + deepstack_input_embeds.permute(1, 0, 2).reshape(

1768 + deepstack_input_embeds.shape[1], -1

1769 + )

1770 )

1771

1565 - # Debug logging for EVS pruning

1566 - logger.debug(

1567 - "EVS: Video tokens pruned from %d to %d (T=%d,H=%d,W=%d, "

1568 - "pruning_rate=%.2f, reduction=%.1f%%)",

1569 - emb.shape[0],

1570 - retention_mask.sum().item(),

1571 - size[0],

1572 - size[1],

1573 - size[2],

1574 - self.video_pruning_rate,

1575 - (1 - retention_mask.float().mean().item()) * 100,

1772 + expanded_positions = None

1773 + if self.is_multimodal_pruning_enabled:

1774 + is_vision_start = repl_token_ids.eq(self.config.vision_start_token_id)

1775 + expanded_positions = self._get_expanded_positions(

1776 + device=merged_embeddings.device,

1777 + seq_len=merged_embeddings.shape[0],

1778 + video_grid_thw=video_grid_thw,

1779 + num_tokens_per_frame=num_tokens_per_frame,

1780 + timestamps=timestamps,

1781 + is_video_embed=is_video_embed,

1782 + is_vision_start=is_vision_start,

1783 + retention_mask=retention_mask,

1784 )

1785 + to_concat.append(expanded_positions)

1786

1578 - positions = compute_mrope_for_media(

1579 - size,

1580 - merge_size,

1581 - tokens_per_second=tokens_per_second,

1582 - video_second_per_grid=video_second_per_grid_t.item(),

1583 - ).to(emb.device)

1787 + final_video_embeddings = torch.cat(to_concat, dim=-1)

1788

1585 - emb = emb[retention_mask]

1586 - positions = positions[retention_mask]

1587 - emb = torch.cat([emb, positions], dim=1)

1588 - video_embeds_out.append(emb)

1589 - return tuple(video_embeds_out)

1789 + return final_video_embeddings

1790 +

1791 + def _get_expanded_positions(

1792 + self,

1793 + device,

1794 + seq_len,

1795 + video_grid_thw,

1796 + num_tokens_per_frame,

1797 + timestamps,

1798 + is_video_embed,

1799 + is_vision_start,

1800 + retention_mask,

1801 + ):

1802 + embed_token_id = _cached_tensor(self.config.video_token_id, device=device)

1803 +

1804 + # Expand positions to match the full sequence length

1805 + # (includes both video tokens and indicator tokens)

1806 + # Shape: [full_length, 5] where positions are filled for video tokens

1807 + # and zeros for indicator tokens.

1808 + # Channel 3 flags VISION_START tokens so that

1809 + # recompute_mrope_positions can reliably count timestamp tokens

1810 + # (even when early frames have all video tokens pruned).

1811 + # Channel 4 flags video-embedding tokens.

1812 + expanded_positions = torch.zeros(

1813 + seq_len,

1814 + 5, # [t_index, h_index, w_index, is_vision_start, is_video]

1815 + device=device,

1816 + dtype=torch.long,

1817 + )

1818 + _, h, w = video_grid_thw

1819 + merge_size = self.visual.spatial_merge_size

1820 + num_frames = len(num_tokens_per_frame)

1821 + unpruned_token_ids = Qwen3VLMultiModalProcessor.get_video_repl(

1822 + tokens_per_frame=[(h // merge_size) * (w // merge_size)] * num_frames,

1823 + tokenizer=self._tokenizer,

1824 + timestamps=timestamps,

1825 + vision_start_token_id=self.config.vision_start_token_id,

1826 + vision_end_token_id=self.config.vision_end_token_id,

1827 + video_token_id=self.config.video_token_id,

1828 + ).full

1829 + unpruned_token_ids_tensor = torch.tensor(unpruned_token_ids, device=device)

1830 + mm_feature = MultiModalFeatureSpec(

1831 + data=MultiModalKwargsItem(

1832 + {

1833 + "video_grid_thw": MultiModalFieldElem(

1834 + data=torch.tensor(video_grid_thw),

1835 + field=None, # HACK.

1836 + ),

1837 + }

1838 + ),

1839 + modality="video",

1840 + identifier="DUMMY",

1841 + mm_position=PlaceholderRange(offset=0, length=len(unpruned_token_ids)),

1842 + )

1843 + original_mrope = (

1844 + self.get_mrope_input_positions(

1845 + input_tokens=unpruned_token_ids,

1846 + mm_features=[mm_feature],

1847 + )[0]

1848 + .to(device)

1849 + .permute(1, 0)

1850 + )

1851 + full_is_video_embed = unpruned_token_ids_tensor == embed_token_id

1852 + expanded_positions[is_video_embed, :3] = original_mrope[full_is_video_embed][

1853 + retention_mask

1854 + ]

1855 + expanded_positions[~is_video_embed, :3] = original_mrope[~full_is_video_embed]

1856 + expanded_positions[..., 3] = is_vision_start

1857 + expanded_positions[..., 4] = is_video_embed

1858 +

1859 + return expanded_positions

1860

1861 def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:

1862 mm_input_by_modality = {}

@@ -1607,66 +1877,77 @@ class Qwen3VLForConditionalGeneration(

1877 )

1878 return mm_input_by_modality

1879

1610 - def iter_mm_grid_hw(

1611 - self, input_tokens: list[int], mm_features: list[MultiModalFeatureSpec]

1612 - ) -> Iterator[tuple[int, int, int]]:

1613 - """

1614 - Iterate over multimodal features and yield grid information.

1615 -

1616 - For videos with EVS (Efficient Video Sampling) enabled, this function

1617 - computes the offset based on the pruned token count rather than relying

1618 - on input_tokens.index(), which would fail when tokens are pruned.

1880 + @staticmethod

1881 + def _iter_mm_grid_hw(

1882 + input_tokens: list[int],

1883 + mm_features: list[MultiModalFeatureSpec],

1884 + video_token_id: int,

1885 + vision_start_token_id: int,

1886 + vision_end_token_id: int,

1887 + spatial_merge_size: int,

1888 + ) -> Iterator[tuple[int, int, int, int]]:

1889 + """Iterate over multimodal features and yield position info.

1890

1891 Args:

1621 - input_tokens: List of token IDs in the prompt

1622 - mm_features: List of multimodal feature specifications

1892 + input_tokens: List of token IDs in the input sequence.

1893 + mm_features: List of multimodal feature specifications containing

1894 + image/video data and position information.

1895 + video_token_id: Token ID used for video tokens.

1896 + vision_start_token_id: Token ID marking the start of a vision sequence.

1897 + vision_end_token_id: Token ID marking the end of a vision sequence.

1898 + spatial_merge_size: Size of the spatial merge operation used to

1899 + compute logical grid dimensions from the original feature grid.

1900

1901 Yields:

1625 - Tuple of (offset, grid_h, grid_w) for each frame/image

1902 + offset: Position of the first video/image token in the sequence.

1903 + llm_grid_h: Logical grid height (may not match actual token count with EVS).

1904 + llm_grid_w: Logical grid width (may not match actual token count with EVS).

1905 + actual_num_tokens: Actual number of video/image tokens in the placeholder.

1906 """

1627 - video_token_id = self.config.video_token_id

1628 - spatial_merge_size = self.config.vision_config.spatial_merge_size

1907 for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset):

1908 offset = mm_feature.mm_position.offset

1909 if mm_feature.modality == "image":

1910 t, h, w = mm_feature.data["image_grid_thw"].data.tolist()

1911 assert t == 1, f"Image must have 1 frame, got {t}"

1634 - yield offset, h // spatial_merge_size, w // spatial_merge_size

1912 + llm_grid_h = h // spatial_merge_size

1913 + llm_grid_w = w // spatial_merge_size

1914 + yield offset, llm_grid_h, llm_grid_w, llm_grid_h * llm_grid_w

1915 elif mm_feature.modality == "video":

1916 t, h, w = mm_feature.data["video_grid_thw"].data.tolist()

1917 llm_grid_h = h // spatial_merge_size

1918 llm_grid_w = w // spatial_merge_size

1919

1640 - # Check if EVS (Efficient Video Sampling) is enabled

1641 - is_evs_enabled = (

1642 - hasattr(self, "video_pruning_rate")

1643 - and self.video_pruning_rate is not None

1644 - and self.video_pruning_rate > 0.0

1645 - )

1646 -

1647 - if is_evs_enabled:

1648 - frame_offsets = self._extract_frame_offsets_from_mask(

1649 - mm_feature.mm_position, t

1650 - )

1651 - if frame_offsets is not None:

1652 - for rel_offset in frame_offsets:

1653 - yield offset + rel_offset, llm_grid_h, llm_grid_w

1654 - continue

1655 -

1656 - # If EVS is enabled but mask is missing, this indicates a bug

1657 - # in the prompt processing pipeline. The is_embed mask should

1658 - # always be present when video_pruning_rate > 0.

1659 - raise RuntimeError(

1660 - f"EVS is enabled (pruning_rate={self.video_pruning_rate}) "

1661 - "but is_embed mask is missing from mm_position. "

1662 - "This indicates a bug in prompt processing."

1663 - )

1664 - else:

1665 - # Non-EVS mode: Use original logic with input_tokens.index()

1666 - for _ in range(t):

1667 - offset = input_tokens.index(video_token_id, offset)

1668 - yield offset, llm_grid_h, llm_grid_w

1669 - offset += llm_grid_h * llm_grid_w

1920 + for _ in range(t):

1921 + # When EVS is enabled, some frames may have 0 video tokens in the

1922 + # placeholder. We use `vision_start_token_id` to locate each frame

1923 + # since it is always present for every frame.

1924 + # We then look for the first `video_token_id` after

1925 + # `vision_start_token_id` and before `vision_end_token_id`.

1926 + offset = input_tokens.index(vision_start_token_id, offset)

1927 + vision_end_offset = input_tokens.index(vision_end_token_id, offset)

1928 +

1929 + try:

1930 + actual_num_tokens = 0

1931 + video_offset = input_tokens.index(

1932 + video_token_id, offset, vision_end_offset

1933 + )

1934 + # NOTE: looking at the

1935 + # `Qwen3VLMultiModalProcessor.get_video_repl` code, we can

1936 + # see that we can use the below formula to get the token

1937 + # count, since everything in between `video_offset` and

1938 + # `vision_end_offset` is populated as `video_token_id`.

1939 + # This saves us from manually counting the number tokens

1940 + # that match `video_token_id` in between.

1941 + actual_num_tokens += vision_end_offset - video_offset

1942 + except ValueError:

1943 + # No `video_token_id` in this frame (EVS with 0 tokens for

1944 + # this frame) -> use `offset + 1`` to move past

1945 + # `vision_start_token_id`.

1946 + video_offset = offset + 1

1947 +

1948 + yield video_offset, llm_grid_h, llm_grid_w, actual_num_tokens

1949 + # Move offset past this frame for next iteration.

1950 + offset = vision_end_offset + 1

1951 else:

1952 raise ValueError(f"Unsupported modality: {mm_feature.modality}")

1953

@@ -1771,13 +2052,100 @@ class Qwen3VLForConditionalGeneration(

2052

2053 return [len(seg) for seg in segments]

2054

2055 + def get_mrope_input_positions(

2056 + self,

2057 + input_tokens: list[int],

2058 + mm_features: list[MultiModalFeatureSpec],

2059 + ) -> tuple[torch.Tensor, int]:

2060 + return self._get_mrope_input_positions(

2061 + input_tokens=input_tokens,

2062 + mm_features=mm_features,

2063 + config=self.config,

2064 + )

2065 +

2066 + @staticmethod

2067 + def _get_mrope_input_positions(

2068 + input_tokens: list[int],

2069 + mm_features: list[MultiModalFeatureSpec],

2070 + config: Qwen3VLConfig,

2071 + ):

2072 + llm_pos_ids_list = []

2073 + st = 0

2074 + for (

2075 + offset,

2076 + llm_grid_h,

2077 + llm_grid_w,

2078 + actual_num_tokens,

2079 + ) in Qwen3VLForConditionalGeneration._iter_mm_grid_hw(

2080 + input_tokens,

2081 + mm_features,

2082 + video_token_id=config.video_token_id,

2083 + vision_start_token_id=config.vision_start_token_id,

2084 + vision_end_token_id=config.vision_end_token_id,

2085 + spatial_merge_size=config.vision_config.spatial_merge_size,

2086 + ):

2087 + # Skip frames with 0 tokens (EVS placeholder with tokens lumped elsewhere)

2088 + if actual_num_tokens == 0:

2089 + continue

2090 +

2091 + text_len = offset - st

2092 + st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0

2093 + llm_pos_ids_list.append(

2094 + np.broadcast_to(np.arange(text_len), (3, text_len)) + st_idx

2095 + )

2096 +

2097 + # Check if this is a "lumped placeholder" (all tokens from multiple frames

2098 + # assigned to the 0-th frame - see

2099 + # `Qwen3VLMultiModalProcessor.get_video_repl`.

2100 + expected_tokens_per_frame = llm_grid_h * llm_grid_w

2101 + if actual_num_tokens > expected_tokens_per_frame:

2102 + # Lumped placeholder: create grid positions for all "logical" frames

2103 + # represented.

2104 + num_logical_frames = actual_num_tokens // expected_tokens_per_frame

2105 + remainder = actual_num_tokens % expected_tokens_per_frame

2106 +

2107 + # Create positions for complete frames.

2108 + for _ in range(num_logical_frames):

2109 + grid_indices = np.indices((1, llm_grid_h, llm_grid_w)).reshape(

2110 + 3, -1

2111 + )

2112 + llm_pos_ids_list.append(grid_indices + text_len + st_idx)

2113 + st_idx = llm_pos_ids_list[-1].max() + 1

2114 + text_len = 0 # No text between frames within the lump

2115 +

2116 + # Handle remainder tokens if any (partial frame).

2117 + # NOTE: this should never be the case. Should we have an assert?

2118 + if remainder > 0:

2119 + # Create a partial grid - take first 'remainder' positions

2120 + full_grid = np.indices((1, llm_grid_h, llm_grid_w)).reshape(3, -1)

2121 + grid_indices = full_grid[:, :remainder]

2122 + llm_pos_ids_list.append(grid_indices + text_len + st_idx)

2123 + else:

2124 + # Normal case: frame has exactly the expected tokens (after actual EVS

2125 + # pruning).

2126 + grid_indices = np.indices((1, llm_grid_h, llm_grid_w)).reshape(3, -1)

2127 + llm_pos_ids_list.append(grid_indices + text_len + st_idx)

2128 +

2129 + st = offset + actual_num_tokens

2130 +

2131 + if st < len(input_tokens):

2132 + st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0

2133 + text_len = len(input_tokens) - st

2134 + llm_pos_ids_list.append(

2135 + np.broadcast_to(np.arange(text_len), (3, text_len)) + st_idx

2136 + )

2137 +

2138 + llm_positions = np.concatenate(llm_pos_ids_list, axis=1).reshape(3, -1)

2139 + mrope_position_delta = (llm_positions.max() + 1 - len(input_tokens)).item()

2140 + return torch.from_numpy(llm_positions), mrope_position_delta

2141 +

2142 def recompute_mrope_positions(

2143 self,

2144 input_ids: list[int],

1777 - multimodal_embeddings: tuple[torch.Tensor, ...],

2145 + multimodal_embeddings: MultiModalEmbeddings,

2146 mrope_positions: torch.LongTensor,

2147 num_computed_tokens: int,

1780 - ) -> tuple[tuple[torch.Tensor, ...], torch.Tensor, int]:

2148 + ) -> tuple[MultiModalEmbeddings, torch.Tensor, int]:

2149 """

2150 Update part of input mrope positions (starting with

2151 num_computed_tokens index). Original mrope_positions are computed

@@ -1786,9 +2154,10 @@ class Qwen3VLForConditionalGeneration(

2154 mrope_positions before we feed it to LLM.

2155

2156 Args:

1789 - input_ids: (N,) All input tokens of the prompt (Containing

1790 - entire sequence).

1791 - multimodal_embeddings: Tuple of multimodal embeddings.

2157 + input_ids: (N,) All input tokens of the prompt containing

2158 + entire sequence.

2159 + multimodal_embeddings: Tuple of multimodal embeddings that

2160 + fits into the prefill chunk that is being processed.

2161 mrope_positions: Existing mrope positions (3, N) for entire

2162 sequence

2163 num_computed_tokens: A number of computed tokens so far.

@@ -1797,10 +2166,26 @@ class Qwen3VLForConditionalGeneration(

2166 Tuple of (multimodal_embeddings, mrope_positions,

2167 mrope_position_delta).

2168 """

1800 - image_token_id = self.config.image_token_id

1801 - video_token_id = self.config.video_token_id

1802 - vision_start_token_id = self.config.vision_start_token_id

2169 + return self._recompute_mrope_positions(

2170 + input_ids=input_ids,

2171 + multimodal_embeddings=multimodal_embeddings,

2172 + mrope_positions=mrope_positions,

2173 + num_computed_tokens=num_computed_tokens,

2174 + image_token_id=self.config.image_token_id,

2175 + video_token_id=self.config.video_token_id,

2176 + vision_start_token_id=self.config.vision_start_token_id,

2177 + )

2178

2179 + @staticmethod

2180 + def _recompute_mrope_positions(

2181 + input_ids: list[int],

2182 + multimodal_embeddings: MultiModalEmbeddings,

2183 + mrope_positions: torch.LongTensor,

2184 + num_computed_tokens: int,

2185 + vision_start_token_id: int,

2186 + image_token_id: int,

2187 + video_token_id: int,

2188 + ) -> tuple[MultiModalEmbeddings, torch.Tensor, int]:

2189 # Device

2190 device = (

2191 multimodal_embeddings[0].device

@@ -1811,10 +2196,21 @@ class Qwen3VLForConditionalGeneration(

2196 # Tensors

2197 input_ids_t = torch.as_tensor(input_ids, device=device, dtype=torch.long)

2198

1814 - mm_embeddings_out = [mm[:, :-4] for mm in multimodal_embeddings]

1815 - mm_embeddings_pos = [

1816 - mm[:, -4:].permute(1, 0).long() for mm in multimodal_embeddings

1817 - ]

2199 + mm_embeddings_out = []

2200 + mm_embeddings_pos = []

2201 + # Strip position information from embeddings (last 5 channels)

2202 + # For Qwen3 VL, handle potentially empty frames (from unpacking)

2203 + for mm in multimodal_embeddings:

2204 + if mm.shape[0] > 0: # Only process non-empty frames

2205 + mm_embeddings_out.append(mm[:, :-5])

2206 + mm_embeddings_pos.append(mm[:, -5:].permute(1, 0).long())

2207 + else:

2208 + # Empty frame - keep as is

2209 + mm_embeddings_out.append(mm)

2210 + # Create empty position tensor with correct shape

2211 + mm_embeddings_pos.append(

2212 + torch.empty(5, 0, device=device, dtype=torch.long)

2213 + )

2214

2215 positions, mrope_positions_delta = recompute_mrope_positions(

2216 input_ids_t,

@@ -1828,107 +2224,14 @@ class Qwen3VLForConditionalGeneration(

2224

2225 return tuple(mm_embeddings_out), positions, mrope_positions_delta

2226

1831 - def get_mrope_input_positions(

1832 - self,

1833 - input_tokens: list[int],

1834 - mm_features: list[MultiModalFeatureSpec],

1835 - ) -> tuple[torch.Tensor, int]:

1836 - # Pre-collect actual frame token counts for EVS mode

1837 - frame_token_counts_map = {}

1838 - for mm_feature in mm_features:

1839 - if mm_feature.modality == "video":

1840 - is_evs_enabled = (

1841 - hasattr(self, "video_pruning_rate")

1842 - and self.video_pruning_rate is not None

1843 - and self.video_pruning_rate > 0.0

1844 - )

1845 - if is_evs_enabled:

1846 - t = mm_feature.data["video_grid_thw"].data.tolist()[0]

1847 - token_counts = self._get_actual_frame_token_counts(

1848 - mm_feature.mm_position, t

1849 - )

1850 - assert token_counts is not None, (

1851 - "EVS enabled but failed to extract frame token counts "

1852 - "from is_embed mask"

1853 - )

1854 - frame_token_counts_map[mm_feature.mm_position.offset] = token_counts

1855 -

1856 - llm_pos_ids_list = []

1857 - st = 0

1858 - frame_counts_idx = {}

1859 -

1860 - for offset, llm_grid_h, llm_grid_w in self.iter_mm_grid_hw(

1861 - input_tokens, mm_features

1862 - ):

1863 - text_len = offset - st

1864 - st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0

1865 -

1866 - # Determine actual token count for this frame

1867 - base_offset = None

1868 - for feat_offset in frame_token_counts_map:

1869 - if offset >= feat_offset:

1870 - base_offset = feat_offset

1871 -

1872 - if base_offset is not None:

1873 - # EVS mode: use actual token count from is_embed mask

1874 - assert base_offset in frame_token_counts_map, (

1875 - f"Found base_offset {base_offset} but not in frame_token_counts_map"

1876 - )

1877 -

1878 - if base_offset not in frame_counts_idx:

1879 - frame_counts_idx[base_offset] = 0

1880 -

1881 - counts = frame_token_counts_map[base_offset]

1882 - idx = frame_counts_idx[base_offset]

1883 -

1884 - assert idx < len(counts), (

1885 - f"EVS frame index {idx} out of range (total frames: {len(counts)})"

1886 - )

1887 -

1888 - actual_frame_tokens = counts[idx]

1889 - frame_counts_idx[base_offset] += 1

1890 - else:

1891 - # Non-EVS mode (or image): use theoretical grid size

1892 - actual_frame_tokens = llm_grid_h * llm_grid_w

1893 -

1894 - # Add text segment

1895 - text_positions = (

1896 - np.broadcast_to(np.arange(text_len), (3, text_len)) + st_idx

1897 - )

1898 - llm_pos_ids_list.append(text_positions)

1899 - st_idx += text_len

1900 -

1901 - # Add frame segment with actual token count (not theoretical)

1902 - grid_indices = np.indices((1, llm_grid_h, llm_grid_w)).reshape(3, -1)

1903 - # Only take the first actual_frame_tokens positions

1904 - frame_positions = grid_indices[:, :actual_frame_tokens] + st_idx

1905 - llm_pos_ids_list.append(frame_positions)

1906 -

1907 - # Update st using actual token count

1908 - st = offset + actual_frame_tokens

1909 -

1910 - # Handle final text segment

1911 - if st < len(input_tokens):

1912 - st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0

1913 - text_len = len(input_tokens) - st

1914 - final_text_positions = (

1915 - np.broadcast_to(np.arange(text_len), (3, text_len)) + st_idx

1916 - )

1917 - llm_pos_ids_list.append(final_text_positions)

1918 -

1919 - llm_positions = np.concatenate(llm_pos_ids_list, axis=1).reshape(3, -1)

1920 - mrope_position_delta = (llm_positions.max() + 1 - len(input_tokens)).item()

1921 -

1922 - return torch.from_numpy(llm_positions), mrope_position_delta

1923 -

2227 def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings | None:

2228 mm_input_by_modality = self._parse_and_validate_multimodal_inputs(**kwargs)

2229 if not mm_input_by_modality:

2230 return None

2231

2232 # The result multimodal_embeddings is tuple of tensors, with each

1930 - # tensor correspoending to a multimodal data item (image or video).

1931 - multimodal_embeddings: tuple[torch.Tensor, ...] = ()

2233 + # tensor corresponding to a multimodal data item (image or video).

2234 + multimodal_embeddings: list[torch.Tensor] = []

2235

2236 # NOTE: It is important to iterate over the keys in this dictionary

2237 # to preserve the order of the modalities.

@@ -1936,19 +2239,20 @@ class Qwen3VLForConditionalGeneration(

2239 multimodal_input = mm_input_by_modality[modality]

2240 if modality == "image":

2241 image_embeddings = self._process_image_input(multimodal_input)

1939 - if self.is_multimodal_pruning_enabled:

1940 - image_embeddings = self._postprocess_image_embeds_evs(

1941 - image_embeddings, multimodal_input

1942 - )

1943 - multimodal_embeddings += tuple(image_embeddings)

2242 + image_embeddings = self._postprocess_image_embeds_evs(

2243 + image_embeddings, multimodal_input

2244 + )

2245 + multimodal_embeddings.extend(image_embeddings)

2246 if modality == "video":

2247 video_embeddings = self._process_video_input(multimodal_input)

2248 if self.is_multimodal_pruning_enabled:

2249 video_embeddings = self._postprocess_video_embeds_evs(

2250 video_embeddings, multimodal_input

2251 )

1950 - multimodal_embeddings += tuple(video_embeddings)

1951 - return multimodal_embeddings

2252 + multimodal_embeddings.extend(video_embeddings)

2253 +

2254 + embeddings_tuple = tuple(multimodal_embeddings)

2255 + return embeddings_tuple

2256

2257 def _compute_deepstack_embeds(

2258 self,

@@ -2128,3 +2432,8 @@ class Qwen3VLForConditionalGeneration(

2432 vision_config = hf_config.vision_config

2433 merge_size = vision_config.spatial_merge_size

2434 return num_vision_tokens // merge_size**2

2435 +

2436 +

2437 +@lru_cache

2438 +def _cached_tensor(x, device) -> torch.Tensor:

2439 + return torch.tensor(x, device=device)

vllm/model_executor/models/qwen3_vl_moe.py

@@ -45,6 +45,7 @@ from vllm.model_executor.model_loader.weight_utils import (

45 )

46 from vllm.multimodal import MULTIMODAL_REGISTRY

47 from vllm.sequence import IntermediateTensors

48 +from vllm.tokenizers.registry import cached_tokenizer_from_config

50 from .interfaces import MixtureOfExperts

51 from .qwen3_moe import (

@@ -415,6 +416,7 @@ class Qwen3VLMoeForConditionalGeneration(

416 multimodal_config = vllm_config.model_config.multimodal_config

417

418 self.config = config

419 + self._tokenizer = cached_tokenizer_from_config(vllm_config.model_config)

420 self.multimodal_config = multimodal_config

421 self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"

422 self.video_pruning_rate = multimodal_config.video_pruning_rate

vllm/multimodal/evs.py

+70 -8

@@ -170,9 +170,9 @@ def recompute_mrope_positions(

170 multimodal_embeddings may contain zero, some or even some part of all

171 multimodal_embeddings for a given prompt.

172

173 - Each multimodal_positions has 4 extra channels

174 - (First 3 channels corresponds to original 3 mrope positions, last channel

175 - is the maximum width of the media repeated). Provided multimodal_positions

173 + Each multimodal_positions has 4 or 5 extra channels

174 + (first 3 channels correspond to the original 3 mrope positions;

175 + remaining channels vary by model — see below). Provided multimodal_positions

176 do not reflect location of media position in sequence - they are computed

177 like the media is in the 0-th position in the sequence.

178

@@ -186,6 +186,16 @@ def recompute_mrope_positions(

186 Args:

187 input_ids: (N,) All input tokens of the prompt (entire sequence).

188 multimodal_positions: List of mrope positions for each media.

189 + If a given element is of shape (4, N), it is assumed to only describe

190 + positions for video / image embeddings. This is the case of e.g. Qwen2.5 VL,

191 + where each multimodal input is a contiguous chunk of embeddings.

192 + The expected channels are [t, h, w, max_width].

193 + If it is of shape (5, N), it is assumed to possibly describe positions for

194 + both video / image embeddings, as well as text embeddings. This is the case

195 + of e.g. Qwen3 VL, where each video inputs are comprised of individual

196 + frames' embeddings, interleaved with embeddings for timestamp tokens,

197 + and vision start / end tokens. The expected channels are

198 + [t, h, w, is_vision_start, is_vision].

199 mrope_positions: Existing mrope positions (4, N) for entire sequence.

200 num_computed_tokens: A number of computed tokens so far.

201 vision_start_token_id: Token indicating start of vision media.

@@ -233,6 +243,21 @@ def recompute_mrope_positions(

243 # - Current prefill chunk has no vision start indexes at all

244 # - Vision start token appeared in previous prefill round

245 # - Regular case

246 + has_video_tokens = False

247 + num_timestamp_tokens = 0

248 + if mm_pos.shape[0] == 5 and mm_pos.shape[1] > 0:

249 + # mm_pos[4, :] indicates which positions are for video embeddings.

250 + # If there are no video embeddings, skip timestamp adjustment.

251 + has_video_tokens = torch.any(mm_pos[4, :]).item()

252 + if has_video_tokens:

253 + # Channel 3 flags VISION_START tokens. Timestamp tokens

254 + # precede the first VISION_START, so its index gives us the

255 + # exact timestamp count. This is robust even when early

256 + # frames have all their video tokens pruned (which would

257 + # push argmax(channel 4) far into a later frame).

258 + first_vs = (mm_pos[3, :] == 1).nonzero(as_tuple=True)[0]

259 + num_timestamp_tokens = first_vs[0].item() if len(first_vs) > 0 else 0

260 +

261 seen_vision_start_indices = vision_start_indices[

262 vision_start_indices < num_computed_tokens

263 ]

@@ -249,6 +274,18 @@ def recompute_mrope_positions(

274 in_the_middle_of_media = (

275 seen_mm_tokens > seem_mm_tokens_before_last_vision_start

276 )

277 + # For Qwen3 VL, we can be inside a media segment even before any

278 + # video tokens appear (timestamp tokens are text). If we've passed

279 + # the last vision_start token but haven't reached the first video

280 + # embedding, treat this as "in the middle of media".

281 + if (

282 + not in_the_middle_of_media

283 + and has_video_tokens

284 + and num_computed_tokens > last_vision_start_token

285 + and num_computed_tokens

286 + <= last_vision_start_token + num_timestamp_tokens + 1

287 + ):

288 + in_the_middle_of_media = True

289

290 if in_the_middle_of_media:

291 mm_embeddings_seen = (

@@ -274,14 +311,39 @@ def recompute_mrope_positions(

311 mm_embeddings_seen = 0

312 global_mm_start = next_vision_start_token

313

277 - # Offset right after vision_start_token

278 - base = positions[-1, global_mm_start] + 1

279 - local_start = global_mm_start + 1 + mm_embeddings_seen

314 + # For Qwen3 VL, mm_pos includes timestamp tokens before vision_start

315 + # when starting a new media. Adjust global_mm_start to point to where

316 + # the sequence actually begins (before timestamp tokens).

317 + adjusted_for_timestamps = False

318 + if mm_pos.shape[0] == 5 and mm_embeddings_seen == 0 and has_video_tokens:

319 + # NOTE: -1 is because there is a vision start token right after

320 + # timestamp tokens before any video embeddings appear.

321 +

322 + # Adjust global_mm_start to point to the first timestamp token

323 + # instead of the vision_start token.

324 + global_mm_start -= num_timestamp_tokens

325 + adjusted_for_timestamps = True

326 +

327 + # Offset calculation depends on whether we adjusted for timestamp tokens

328 + if adjusted_for_timestamps:

329 + # Start from position before the first timestamp token

330 + base = positions[-1, global_mm_start - 1] + 1

331 + local_start = global_mm_start + mm_embeddings_seen

332 + else:

333 + # Original logic: start after vision_start_token

334 + base = positions[-1, global_mm_start] + 1

335 + local_start = global_mm_start + 1 + mm_embeddings_seen

336 +

337 local_end = local_start + mm_pos.shape[1]

338 positions[:, local_start:local_end] = mm_pos[0:3] + base

339

283 - # mm_pos[3, 0] is the max width of the media

284 - offset = mm_pos[3, 0] + base

340 + # For Qwen3 VL (5-channel), use the maximum position reached across

341 + # all tokens (both video and text) in all dimensions (t, h, w).

342 + # For Qwen2.5 VL (4-channel), mm_pos[3, 0] is the max width.

343 + if mm_pos.shape[0] == 5:

344 + offset = mm_pos[0:3, :].max() + base + 1

345 + else:

346 + offset = mm_pos[3, 0] + base

347

348 text_pos_sum = torch.cumsum(text_mask[local_end:].long(), dim=0)

349