A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 16:00 UTC
[Bugfix] fix encoder cache hang in Qwen3VL (#32684)
Signed-off-by: JJJYmmm <92386084+JJJYmmm@users.noreply.github.com> Signed-off-by: Roger Wang <hey@rogerw.io> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
JJJYmmm committed
Jan 25, 2026 at 13:17 UTC
7e67df5570e2afc6a92833096898689c91244fb6
3 files changed
+54
-43
tests/models/multimodal/processing/test_tensor_schema.py
+2
-17
@@ -48,18 +48,6 @@ VideoInput: TypeAlias = (
48
AudioInput = list[tuple[np.ndarray, int]]
49
50
51
-MM_OPTIONS_OVERRIDES = {
52
- # Qwen3-VL's default profiling video size (64x64) can cause trouble
53
- # after resizing, so we override it here for testing.
54
- "qwen3_vl": dict(
55
- video=VideoDummyOptions(num_frames=128, width=256, height=256),
56
- ),
57
- "qwen3_vl_moe": dict(
58
- video=VideoDummyOptions(num_frames=128, width=256, height=256),
59
- ),
60
-}
61
-
62
-
51
def _resize_data(
52
_data: Image.Image | np.ndarray, size_factor: float
53
) -> Image.Image | np.ndarray:
@@ -73,12 +61,12 @@ def _resize_data(
61
elif is_list_of(_data, Image.Image):
62
W, H = next(iter(_data)).width, next(iter(_data)).height
63
T = len(_data)
76
- T, W, H = map(lambda x: max(int(x * size_factor), 1), (T, W, H))
64
+ T, W, H = map(lambda x: max(int(x * size_factor), 2), (T, W, H))
65
return [d.resize((W, H)) for d in _data[:T]]
66
# Video input with numpy arrays
67
elif isinstance(_data, np.ndarray) and _data.ndim >= 4:
68
T, H, W, C = _data.shape[-4:]
81
- T, H, W = map(lambda x: max(int(x * size_factor), 1), (T, H, W))
69
+ T, H, W = map(lambda x: max(int(x * size_factor), 2), (T, H, W))
70
return _data[..., :T, :H, :W, :C]
71
# Audio input
72
elif isinstance(_data, np.ndarray) and _data.ndim == 1:
@@ -103,8 +91,6 @@ def create_batched_mm_kwargs(
91
processor: BaseMultiModalProcessor,
92
size_factors: tuple[float, ...] = (1.0, 0.5, 0.25),
93
) -> Iterable[tuple[str, int, BatchedTensorInputs]]:
106
- model_type = model_config.hf_config.model_type
107
-
94
processing_info = processor.info
95
dummy_inputs = processor.dummy_inputs
96
supported_mm_limits = processing_info.get_supported_mm_limits()
@@ -115,7 +101,6 @@ def create_batched_mm_kwargs(
101
processor_inputs = dummy_inputs.get_dummy_processor_inputs(
102
seq_len=model_config.max_model_len,
103
mm_counts=mm_counts,
118
- mm_options=MM_OPTIONS_OVERRIDES.get(model_type),
104
)
105
mm_data = processor_inputs.mm_data
106
resized_mm_data = {
vllm/model_executor/models/qwen2_vl.py
+8
-3
@@ -892,7 +892,9 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
892
)
893
return num_video_tokens
894
895
- def get_image_size_with_most_features(self) -> ImageSize:
895
+ def get_image_size_with_most_features(
896
+ self, max_pixels: int | None = None
897
+ ) -> ImageSize:
898
# NOTE: Simply processing a huge size with _get_vision_info might not give a
899
# size that maximizes the number of featrues, i.e., the number of (merged)
900
# patches. This is because the number of patches limits the allowed aspect
@@ -910,8 +912,11 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
912
vision_config = hf_config.vision_config
913
patch_size = vision_config.patch_size
914
merge_size = vision_config.spatial_merge_size
913
- image_processor = self.get_image_processor()
914
- max_pixels = image_processor.max_pixels or image_processor.size["longest_edge"]
915
+ if max_pixels is None:
916
+ image_processor = self.get_image_processor()
917
+ max_pixels = (
918
+ image_processor.max_pixels or image_processor.size["longest_edge"]
919
+ )
920
unit = patch_size * merge_size
921
max_seq_len = max_pixels // (unit * unit)
922
vllm/model_executor/models/qwen3_vl.py
+44
-23
@@ -91,6 +91,7 @@ from vllm.multimodal.processing import (
91
)
92
from vllm.sequence import IntermediateTensors
93
from vllm.utils.collection_utils import is_list_of
94
+from vllm.utils.math_utils import round_up
95
from vllm.v1.attention.backends.registry import AttentionBackendEnum
96
97
from .interfaces import (
@@ -129,8 +130,9 @@ from .vision import (
130
131
logger = init_logger(__name__)
132
132
-# Official recommended max frames is 2048
133
-_MAX_FRAMES_PER_VIDEO = 2048
133
+# We use 2048 dummy video frames that would generate vision embeddings
134
+# of the maximum size.
135
+DUMMY_VIDEO_NUM_FRAMES = 2048
136
137
138
class Qwen3_VisionPatchEmbed(nn.Module):
@@ -662,7 +664,7 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
664
else:
665
preprocessed_size = ImageSize(width=image_width, height=image_height)
666
665
- padded_num_frames = num_frames + num_frames % temporal_patch_size
667
+ padded_num_frames = round_up(num_frames, temporal_patch_size)
668
669
grid_t = max(padded_num_frames // temporal_patch_size, 1)
670
grid_h = preprocessed_size.height // patch_size
@@ -684,7 +686,7 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
686
mm_counts: Mapping[str, int],
687
) -> int:
688
return super().get_num_frames_with_most_features(
687
- seq_len, mm_counts, max_frames_per_video=_MAX_FRAMES_PER_VIDEO
689
+ seq_len, mm_counts, max_frames_per_video=DUMMY_VIDEO_NUM_FRAMES
690
)
691
692
def get_max_video_tokens(
@@ -692,11 +694,17 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
694
seq_len: int,
695
mm_counts: Mapping[str, int],
696
) -> int:
695
- target_width, target_height = self.get_image_size_with_most_features()
697
+ video_processor = self.get_video_processor()
698
+ video_max_pixels = video_processor.size["longest_edge"]
699
+ # video_max_pixels contains the temporal compression factor,
700
+ # so we divide by 2 to get the maximum number of image pixels.
701
+ target_width, target_height = self.get_image_size_with_most_features(
702
+ max_pixels=video_max_pixels // video_processor.temporal_patch_size
703
+ )
704
num_video_soft_tokens = self.get_num_video_tokens(
705
image_width=target_width,
706
image_height=target_height,
699
- num_frames=self.get_num_frames_with_most_features(seq_len, mm_counts),
707
+ num_frames=2,
708
image_processor=None,
709
)
710
return num_video_soft_tokens
@@ -779,11 +787,12 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]):
787
image_overrides = mm_options.get("image") if mm_options else None
788
video_overrides = mm_options.get("video") if mm_options else None
789
782
- target_width, target_height = self.info.get_image_size_with_most_features()
783
- target_num_frames = self.info.get_num_frames_with_most_features(
784
- seq_len, mm_counts
790
+ target_image_width, target_image_height = (
791
+ self.info.get_image_size_with_most_features()
792
)
793
794
+ # treat videos as special images
795
+ target_num_frames = 2
796
if video_overrides:
797
assert isinstance(video_overrides, VideoDummyOptions)
798
num_frames_override = video_overrides.num_frames
@@ -804,48 +813,60 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]):
813
target_num_frames = min(target_num_frames, num_frames_override)
814
target_num_frames = max(target_num_frames, 2)
815
816
+ video_processor = self.info.get_video_processor()
817
+ video_max_pixels = video_processor.size["longest_edge"]
818
+ # video_max_pixels contains the temporal compression factor,
819
+ # so we divide by 2 to get the maximum number of image pixels.
820
+ target_video_width, target_video_height = (
821
+ self.info.get_image_size_with_most_features(
822
+ max_pixels=video_max_pixels // video_processor.temporal_patch_size
823
+ )
824
+ )
825
target_video_size, _ = self.info._get_vision_info(
808
- image_width=target_width,
809
- image_height=target_height,
826
+ image_width=target_video_width,
827
+ image_height=target_video_height,
828
num_frames=target_num_frames,
811
- image_processor=self.info.get_video_processor(),
829
+ image_processor=video_processor,
830
)
831
# NOTE: we need to do this check here since Qwen3-VL resizes video
832
# frames depending on how many frames there are.
815
- width, height = target_video_size.width, target_video_size.height
833
+ target_video_width, target_video_height = (
834
+ target_video_size.width,
835
+ target_video_size.height,
836
+ )
837
if video_overrides:
838
assert isinstance(video_overrides, VideoDummyOptions)
839
width_override = video_overrides.width
840
if width_override:
820
- if width_override > width:
841
+ if width_override > target_video_width:
842
logger.warning(
843
"video.width override (%d) exceeds model's "
844
"maximum width (%d), will be ignored",
845
width_override,
825
- width,
846
+ target_video_width,
847
)
827
- width = min(width, width_override)
848
+ target_video_width = min(target_video_width, width_override)
849
height_override = video_overrides.height
850
if height_override:
830
- if height_override > height:
851
+ if height_override > target_video_height:
852
logger.warning(
853
"video.height override (%d) exceeds model's "
854
"maximum height (%d), will be ignored",
855
height_override,
835
- height,
856
+ target_video_height,
857
)
837
- height = min(height, height_override)
858
+ target_video_height = min(target_video_height, height_override)
859
860
return {
861
"image": self._get_dummy_images(
841
- width=target_width,
842
- height=target_height,
862
+ width=target_image_width,
863
+ height=target_image_height,
864
num_images=num_images,
865
overrides=image_overrides,
866
),
867
"video": self._get_dummy_videos(
847
- width=width,
848
- height=height,
868
+ width=target_video_width,
869
+ height=target_video_height,
870
num_frames=target_num_frames,
871
num_videos=num_videos,
872
),