[Bugfix][Multimodal] Release Qwen2.5-VL and Qwen3-VL RoPE caches with the model (#54346)
Signed-off-by: waizuichougou <2082431897@qq.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
waizuichougou committed
Aug 30, 2026 at 11:10 UTC
b016ed8ea3dd8cfec9173e8d5a87ef12adc03096
2 files changed
+20
-7
vllm/model_executor/models/qwen2_5_vl.py
+10
-3
@@ -27,7 +27,7 @@
27
"""Inference-only Qwen2.5-VL model compatible with HuggingFace weights."""
28
29
from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
30
-from functools import lru_cache, partial
30
+from functools import partial
31
from typing import Annotated, Any, Literal, TypeAlias
32
33
import einops
@@ -82,6 +82,7 @@ from vllm.multimodal.video_prune.evs import (
82
)
83
from vllm.platforms import current_platform
84
from vllm.sequence import IntermediateTensors
85
+from vllm.utils.cache import LRUCache
86
from vllm.utils.tensor_schema import TensorSchema, TensorShape
87
from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d
88
from vllm.v1.attention.backends.registry import AttentionBackendEnum
@@ -647,6 +648,7 @@ class Qwen2_5_VisionTransformer(nn.Module):
648
self.spatial_merge_size = vision_config.spatial_merge_size
649
self.fullatt_block_indexes = vision_config.fullatt_block_indexes
650
self.spatial_merge_unit = self.spatial_merge_size**2
651
+ self._rope_by_thw_cache = LRUCache(capacity=1024)
652
use_data_parallel = is_vit_use_data_parallel()
653
self.tp_size = (
654
1
@@ -795,8 +797,11 @@ class Qwen2_5_VisionTransformer(nn.Module):
797
798
return index_new, cu_seqlens_tmp
799
798
- @lru_cache(maxsize=1024) # noqa: B019
800
def get_rope_by_thw(self, t, h, w):
801
+ cache_key = (t, h, w)
802
+ if cache_key in self._rope_by_thw_cache:
803
+ return self._rope_by_thw_cache[cache_key]
804
+
805
window_index_thw, cu_seqlens_window_thw = self.get_window_index_thw(t, h, w)
806
cos_thw, sin_thw = self.rotary_pos_emb_thw(t, h, w)
807
@@ -809,13 +814,15 @@ class Qwen2_5_VisionTransformer(nn.Module):
814
cu_seqlens_thw = torch.repeat_interleave(
815
torch.tensor([h * w], dtype=torch.int32), t
816
)
812
- return (
817
+ result = (
818
cos_thw,
819
sin_thw,
820
window_index_thw,
821
cu_seqlens_window_thw,
822
cu_seqlens_thw,
823
)
824
+ self._rope_by_thw_cache[cache_key] = result
825
+ return result
826
827
def compute_attn_mask_seqlen(
828
self,
vllm/model_executor/models/qwen3_vl.py
+10
-4
@@ -110,6 +110,7 @@ from vllm.sequence import IntermediateTensors
110
from vllm.tokenizers.protocol import TokenizerLike
111
from vllm.tokenizers.registry import cached_tokenizer_from_config
112
from vllm.triton_utils import HAS_TRITON, tl, triton
113
+from vllm.utils.cache import LRUCache
114
from vllm.utils.collection_utils import is_list_of
115
from vllm.utils.math_utils import round_up
116
from vllm.utils.torch_utils import PIN_MEMORY
@@ -581,6 +582,7 @@ class Qwen3_VisionTransformer(nn.Module):
582
else []
583
)
584
self.num_grid_per_side = int(self.num_position_embeddings**0.5)
585
+ self._rot_pos_ids_cache = LRUCache(capacity=1024)
586
587
use_data_parallel = is_vit_use_data_parallel()
588
self.tp_size = (
@@ -672,9 +674,11 @@ class Qwen3_VisionTransformer(nn.Module):
674
def device(self) -> torch.device:
675
return self.patch_embed.proj.weight.device
676
675
- @staticmethod
676
- @lru_cache(maxsize=1024)
677
- def rot_pos_ids(h: int, w: int, spatial_merge_size: int) -> torch.Tensor:
677
+ def rot_pos_ids(self, h: int, w: int, spatial_merge_size: int) -> torch.Tensor:
678
+ cache_key = (h, w, spatial_merge_size)
679
+ if cache_key in self._rot_pos_ids_cache:
680
+ return self._rot_pos_ids_cache[cache_key]
681
+
682
hpos_ids = np.broadcast_to(np.arange(h).reshape(h, 1), (h, w))
683
h_div = h // spatial_merge_size
684
w_div = w // spatial_merge_size
@@ -697,7 +701,9 @@ class Qwen3_VisionTransformer(nn.Module):
701
wpos_ids = wpos_ids.transpose(0, 2, 1, 3)
702
wpos_ids = wpos_ids.flatten()
703
700
- return torch.from_numpy(np.stack([hpos_ids, wpos_ids], axis=-1))
704
+ result = torch.from_numpy(np.stack([hpos_ids, wpos_ids], axis=-1))
705
+ self._rot_pos_ids_cache[cache_key] = result
706
+ return result
707
708
def rot_pos_emb(self, grid_thw: list[list[int]]):
709
max_grid_size = max(max(h, w) for _, h, w in grid_thw)