A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[VLM] Update Qwen3-VL max_num_video_tokens calculation for configurable video profiling (#25557)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Signed-off-by: Roger Wang <hey@rogerw.io> Co-authored-by: Roger Wang <hey@rogerw.io>
Isotr0py committed
Sep 28, 2025 at 12:21 UTC
0efd540dbc5405ada2f57f09d2a376aecad576dc
2 files changed
+74
-9
vllm/model_executor/models/qwen2_vl.py
+9
-4
@@ -79,7 +79,7 @@ from .vision import get_vit_attn_backend, run_dp_sharded_mrope_vision_model
79
logger = init_logger(__name__)
80
81
# For profile run
82
-_MAX_FRAMES_PER_VIDEO = 32
82
+_MAX_FRAMES_PER_VIDEO = 14
83
84
# === Vision Inputs === #
85
@@ -932,6 +932,7 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
932
_, num_image_tokens = self._get_vision_info(
933
image_width=image_width,
934
image_height=image_height,
935
+ num_frames=1,
936
image_processor=image_processor,
937
)
938
return num_image_tokens
@@ -956,6 +957,7 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
957
max_image_size, _ = self._get_vision_info(
958
image_width=9999999,
959
image_height=9999999,
960
+ num_frames=1,
961
image_processor=None,
962
)
963
return max_image_size
@@ -969,10 +971,12 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
971
image_processor=None,
972
)
973
972
- def _get_max_video_frames(self, max_tokens: int) -> int:
974
+ def _get_max_video_frames(self,
975
+ max_tokens: int,
976
+ start_num_frames: int = 1) -> int:
977
target_width, target_height = self.get_image_size_with_most_features()
978
975
- num_frames = 0
979
+ num_frames = start_num_frames
980
981
while True:
982
next_num_frames = num_frames + 1
@@ -994,12 +998,13 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
998
self,
999
seq_len: int,
1000
mm_counts: Mapping[str, int],
1001
+ max_frames_per_video: int = _MAX_FRAMES_PER_VIDEO,
1002
) -> int:
1003
max_videos = mm_counts.get("video", 0)
1004
1005
max_total_frames = self._get_max_video_frames(seq_len)
1006
max_frames_per_video = min(max_total_frames // max(max_videos, 1),
1002
- _MAX_FRAMES_PER_VIDEO)
1007
+ max_frames_per_video)
1008
1009
return max(max_frames_per_video, 1)
1010
vllm/model_executor/models/qwen3_vl.py
+65
-5
@@ -33,11 +33,14 @@ import torch.nn as nn
33
import torch.nn.functional as F
34
from transformers import BatchFeature
35
from transformers.models.qwen2_vl import Qwen2VLImageProcessorFast
36
-from transformers.models.qwen2_vl.image_processing_qwen2_vl import smart_resize
36
+from transformers.models.qwen2_vl.image_processing_qwen2_vl import (
37
+ smart_resize as image_smart_resize)
38
from transformers.models.qwen3_vl import (Qwen3VLProcessor,
39
Qwen3VLVideoProcessor)
40
from transformers.models.qwen3_vl.configuration_qwen3_vl import (
41
Qwen3VLConfig, Qwen3VLVisionConfig)
42
+from transformers.models.qwen3_vl.video_processing_qwen3_vl import (
43
+ smart_resize as video_smart_resize)
44
from transformers.video_utils import VideoMetadata
45
46
from vllm.attention.layer import check_upstream_fa_availability
@@ -85,6 +88,9 @@ from .vision import get_vit_attn_backend, run_dp_sharded_mrope_vision_model
88
89
logger = init_logger(__name__)
90
91
+# Official recommended max pixels is 24576 * 32 * 32
92
+_MAX_FRAMES_PER_VIDEO = 24576
93
+
94
95
class Qwen3_VisionPatchEmbed(nn.Module):
96
@@ -593,11 +599,16 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
599
image_height: int,
600
num_frames: int = 2,
601
do_resize: bool = True,
596
- image_processor: Optional[Qwen2VLImageProcessorFast],
602
+ image_processor: Optional[Union[Qwen2VLImageProcessorFast,
603
+ Qwen3VLVideoProcessor]],
604
) -> tuple[ImageSize, int]:
598
- if image_processor is None:
605
+ if image_processor is None and num_frames > 1:
606
+ image_processor = self.get_video_processor()
607
+ elif image_processor is None:
608
image_processor = self.get_image_processor()
609
610
+ is_video = isinstance(image_processor, Qwen3VLVideoProcessor)
611
+
612
hf_config = self.get_hf_config()
613
vision_config = hf_config.vision_config
614
patch_size = vision_config.patch_size
@@ -605,12 +616,22 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
616
temporal_patch_size = vision_config.temporal_patch_size
617
618
if do_resize:
619
+ if is_video:
620
+ smart_resize = video_smart_resize
621
+ extra_kwargs = {
622
+ "num_frames": num_frames,
623
+ "temporal_factor": temporal_patch_size
624
+ }
625
+ else:
626
+ smart_resize = image_smart_resize
627
+ extra_kwargs = {}
628
resized_height, resized_width = smart_resize(
629
height=image_height,
630
width=image_width,
631
factor=patch_size * merge_size,
632
min_pixels=image_processor.size["shortest_edge"],
633
max_pixels=image_processor.size["longest_edge"],
634
+ **extra_kwargs,
635
)
636
preprocessed_size = ImageSize(width=resized_width,
637
height=resized_height)
@@ -629,6 +650,39 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
650
651
return preprocessed_size, num_vision_tokens
652
653
+ def _get_max_video_frames(self,
654
+ max_tokens: int,
655
+ start_num_frames: int = 2) -> int:
656
+ return super()._get_max_video_frames(max_tokens,
657
+ start_num_frames=start_num_frames)
658
+
659
+ def get_num_frames_with_most_features(
660
+ self,
661
+ seq_len: int,
662
+ mm_counts: Mapping[str, int],
663
+ ) -> int:
664
+ return super().get_num_frames_with_most_features(
665
+ seq_len, mm_counts, max_frames_per_video=_MAX_FRAMES_PER_VIDEO)
666
+
667
+ def get_max_video_tokens(
668
+ self,
669
+ seq_len: int,
670
+ mm_counts: Mapping[str, int],
671
+ ) -> int:
672
+ target_width, target_height = self.get_image_size_with_most_features()
673
+ video_soft_tokens = self.get_num_video_tokens(
674
+ image_width=target_width,
675
+ image_height=target_height,
676
+ num_frames=self.get_num_frames_with_most_features(
677
+ seq_len, mm_counts),
678
+ image_processor=None,
679
+ )
680
+
681
+ # NOTE: By default in Qwen3-VL, one video token is converted to
682
+ # "<{timestamp} seconds>" (on average 9.5 tokens) + vision_start_token + video_token + vision_end_token # noqa: E501
683
+ formatted_video_soft_tokens = video_soft_tokens * 12.5
684
+ return int(formatted_video_soft_tokens)
685
+
686
def _calculate_timestamps(self, indices: list[int] | torch.Tensor,
687
video_fps: float, merge_size: int):
688
if not isinstance(indices, list):
@@ -698,6 +752,12 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]):
752
self.info.get_image_size_with_most_features())
753
target_num_frames = self.info.get_num_frames_with_most_features(
754
seq_len, mm_counts)
755
+ target_video_size, _ = self.info._get_vision_info(
756
+ image_width=target_width,
757
+ image_height=target_height,
758
+ num_frames=target_num_frames,
759
+ image_processor=self.info.get_video_processor(),
760
+ )
761
return {
762
"image":
763
self._get_dummy_images(width=target_width,
@@ -705,8 +765,8 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]):
765
num_images=num_images),
766
"video":
767
self._get_dummy_videos(
708
- width=target_width,
709
- height=target_height,
768
+ width=target_video_size.width,
769
+ height=target_video_size.height,
770
num_frames=target_num_frames,
771
num_videos=num_videos,
772
),