[Perf] Optimize Qwen3-VL multi-video prompt processing (#46026)
Signed-off-by: Sirius29 <422058530@qq.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
L丶 committed
Jun 20, 2026 at 22:09 UTC
d272418f459a82e1012b60116ac00659a7017cde
2 files changed
+124
-26
tests/models/multimodal/processing/test_qwen3_vl.py
+46
@@ -92,3 +92,49 @@ def test_processor_num_frames_timestamp(
92
assert len(video_phs) == 1, (
93
f"Expected exactly 1 video placeholder, got {len(video_phs)}"
94
)
95
+
96
+
97
+@pytest.mark.parametrize("model_id", [MODEL_ID])
98
+@pytest.mark.parametrize("num_videos", [2, 4])
99
+def test_processor_multi_video(
100
+ model_id: str,
101
+ num_videos: int,
102
+) -> None:
103
+ """Verify that multi-video processing produces correct placeholders.
104
+
105
+ This exercises the token-level replacement path in
106
+ ``_call_hf_processor`` which avoids the quadratic text-level
107
+ prompt expansion.
108
+ """
109
+ ctx = build_model_context(
110
+ model_id,
111
+ limit_mm_per_prompt={"image": 0, "video": num_videos},
112
+ )
113
+ processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
114
+
115
+ prompt = "<|vision_start|><|video_pad|><|vision_end|>" * num_videos
116
+ mm_data = {"video": [_build_video_mm_data(num_frames=8)["video"][0]] * num_videos}
117
+
118
+ processed = processor(
119
+ prompt,
120
+ mm_items=processor.info.parse_mm_data(mm_data),
121
+ hf_processor_mm_kwargs={"num_frames": 8},
122
+ )
123
+
124
+ token_ids = processed["prompt_token_ids"]
125
+ assert len(token_ids) > 0
126
+
127
+ video_phs = processed["mm_placeholders"].get("video", [])
128
+ assert len(video_phs) == num_videos, (
129
+ f"Expected {num_videos} video placeholders, got {len(video_phs)}"
130
+ )
131
+
132
+ # All placeholders should have the same length (same video params)
133
+ # and must not overlap.
134
+ lengths = {ph.length for ph in video_phs}
135
+ assert len(lengths) == 1, f"Placeholder lengths differ: {lengths}"
136
+ for i in range(1, len(video_phs)):
137
+ prev_end = video_phs[i - 1].offset + video_phs[i - 1].length
138
+ assert video_phs[i].offset >= prev_end, (
139
+ f"Placeholder {i} overlaps with placeholder {i - 1}"
140
+ )
vllm/model_executor/models/qwen3_vl.py
+78
-26
@@ -1202,6 +1202,49 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]):
1202
return video_items
1203
1204
1205
+def _replace_video_token_placeholders(
1206
+ prompt_ids: list[int],
1207
+ target: list[int],
1208
+ replacements: list[list[int]],
1209
+) -> list[int]:
1210
+ """Replace each 3-token video placeholder with its expanded sequence.
1211
+
1212
+ Args:
1213
+ prompt_ids: Token IDs of the original (unexpanded) prompt.
1214
+ target: 3-element list ``[vision_start_id, video_pad_id,
1215
+ vision_end_id]`` to search for.
1216
+ replacements: Per-video expanded token sequences, in prompt order.
1217
+
1218
+ Returns:
1219
+ Token IDs with every placeholder triplet replaced.
1220
+ """
1221
+ result: list[int] = []
1222
+ repl_idx = 0
1223
+ i = 0
1224
+ n = len(prompt_ids)
1225
+ t0, t1, t2 = target
1226
+ num_repl = len(replacements)
1227
+
1228
+ while i < n:
1229
+ if (
1230
+ i + 2 < n
1231
+ and prompt_ids[i] == t0
1232
+ and prompt_ids[i + 1] == t1
1233
+ and prompt_ids[i + 2] == t2
1234
+ ):
1235
+ result.extend(replacements[repl_idx])
1236
+ repl_idx += 1
1237
+ i += 3
1238
+ else:
1239
+ result.append(prompt_ids[i])
1240
+ i += 1
1241
+
1242
+ assert repl_idx == num_repl, (
1243
+ f"Found {repl_idx} video placeholders but expected {num_repl}"
1244
+ )
1245
+ return result
1246
+
1247
+
1248
class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]):
1249
def _call_hf_processor(
1250
self,
@@ -1211,15 +1254,23 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
1254
tok_kwargs: Mapping[str, object],
1255
) -> BatchFeature:
1256
mm_data = dict(mm_data)
1214
- processor = self.info.get_hf_processor(**mm_kwargs)
1257
1258
# Separate video processing from image processing. Because the videos
1259
# are processed into several image patches
1260
+ video_input_ids_lst: list[list[int]] = []
1261
if videos := mm_data.pop("videos", []):
1262
video_grid_thw_lst = []
1263
pixel_values_videos_lst = []
1264
timestamps_per_video = []
1265
1266
+ hf_config = self.info.get_hf_config()
1267
+ tokenizer = self.info.get_tokenizer()
1268
+ merge_size = hf_config.vision_config.spatial_merge_size
1269
+ video_pruning_rate = self.info.ctx.get_mm_config().video_pruning_rate
1270
+ vision_start_token_id = hf_config.vision_start_token_id
1271
+ vision_end_token_id = hf_config.vision_end_token_id
1272
+ video_token_id = hf_config.video_token_id
1273
+
1274
for item in videos:
1275
video_array, metadata = item
1276
@@ -1269,55 +1320,38 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
1320
tok_kwargs=tok_kwargs,
1321
)
1322
1272
- merge_size = processor.video_processor.merge_size
1273
- # Get video grid info for EVS calculation.
1323
+ # Discard HF output input_ids — we use get_video_repl below
1324
+ # to generate the correct (EVS-adjusted) token sequence.
1325
+ video_outputs.pop("input_ids", None)
1326
+
1327
video_grid_thw = video_outputs["video_grid_thw"]
1328
num_frames = int(video_grid_thw[0, 0])
1329
tokens_per_frame_base = int(video_grid_thw[0, 1:].prod()) // (
1330
merge_size**2
1331
)
1332
1280
- # Apply EVS if enabled.
1281
- video_pruning_rate = self.info.ctx.get_mm_config().video_pruning_rate
1333
if video_pruning_rate is not None and video_pruning_rate > 0.0:
1334
num_tokens = compute_retained_tokens_count(
1335
tokens_per_frame=tokens_per_frame_base,
1336
num_frames=num_frames,
1337
q=video_pruning_rate,
1338
)
1288
- # Here we just need placeholders that won't actually be replaced -
1289
- # we just need to make sure the total number of tokens is correct
1290
- # assign all tokens to the first frame.
1339
tokens_per_frame = [num_tokens] + [0] * (num_frames - 1)
1340
select_token_id = False
1341
else:
1342
tokens_per_frame = [tokens_per_frame_base] * num_frames
1343
select_token_id = True
1344
1297
- # Generate the video replacement with EVS-adjusted token counts
1298
- tokenizer = self.info.get_tokenizer()
1299
- hf_config = self.info.get_hf_config()
1345
video_repl = Qwen3VLMultiModalProcessor.get_video_repl(
1346
tokens_per_frame=tokens_per_frame,
1347
timestamps=timestamps,
1348
tokenizer=tokenizer,
1304
- vision_start_token_id=hf_config.vision_start_token_id,
1305
- vision_end_token_id=hf_config.vision_end_token_id,
1306
- video_token_id=hf_config.video_token_id,
1349
+ vision_start_token_id=vision_start_token_id,
1350
+ vision_end_token_id=vision_end_token_id,
1351
+ video_token_id=video_token_id,
1352
select_token_id=select_token_id,
1353
)
1309
-
1310
- # Convert token IDs to text for the HF processor flow
1311
- video_placeholder = tokenizer.decode(
1312
- video_repl.full, skip_special_tokens=False
1313
- )
1314
- input_ids = video_outputs.pop("input_ids")
1315
- video_placeholder = processor.tokenizer.batch_decode(input_ids)[0]
1316
- prompt = prompt.replace(
1317
- "<|vision_start|><|video_pad|><|vision_end|>",
1318
- video_placeholder,
1319
- 1,
1320
- )
1354
+ video_input_ids_lst.append(list(video_repl.full))
1355
1356
video_grid_thw_lst.append(video_outputs["video_grid_thw"])
1357
pixel_values_videos_lst.append(video_outputs["pixel_values_videos"])
@@ -1335,6 +1369,24 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
1369
mm_kwargs=mm_kwargs,
1370
tok_kwargs=tok_kwargs,
1371
)
1372
+
1373
+ # Replace each placeholder triplet with pre-computed video tokens.
1374
+ if video_input_ids_lst:
1375
+ hf_config = self.info.get_hf_config()
1376
+ video_target = [
1377
+ hf_config.vision_start_token_id,
1378
+ hf_config.video_token_id,
1379
+ hf_config.vision_end_token_id,
1380
+ ]
1381
+ input_ids = processed_outputs.pop("input_ids")
1382
+ if not isinstance(input_ids, list):
1383
+ input_ids = input_ids.tolist()
1384
+ (prompt_ids,) = input_ids
1385
+ expanded_ids = _replace_video_token_placeholders(
1386
+ prompt_ids, video_target, video_input_ids_lst
1387
+ )
1388
+ processed_outputs["input_ids"] = [expanded_ids]
1389
+
1390
combined_outputs = dict(
1391
processed_outputs,
1392
**video_outputs,