A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Models] Prevent CUDA sync in Qwen2.5-VL (#24741)
Signed-off-by: Lukas Geiger <lukas.geiger94@gmail.com>
Lukas Geiger committed
Sep 12, 2025 at 17:03 UTC
b0d1213ac3952227834f7d075fa1a2879b468c69
1 file changed
+4
-1
vllm/model_executor/models/qwen2_5_vl.py
+4
-1
@@ -64,6 +64,7 @@ from vllm.multimodal.utils import run_dp_sharded_mrope_vision_model
64
from vllm.platforms import _Backend
65
from vllm.sequence import IntermediateTensors
66
from vllm.transformers_utils.config import uses_mrope
67
+from vllm.utils import is_pin_memory_available
68
from vllm.utils.tensor_schema import TensorSchema, TensorShape
69
70
from .interfaces import (MultiModalEmbeddings, SupportsLoRA,
@@ -737,7 +738,7 @@ class Qwen2_5_VisionTransformer(nn.Module):
738
@staticmethod
739
def invert_permutation(perm: torch.Tensor) -> torch.Tensor:
740
# building the inverse permutation in O(n) time
740
- inv = torch.empty_like(perm)
741
+ inv = torch.empty_like(perm, pin_memory=is_pin_memory_available())
742
inv[perm] = torch.arange(perm.numel(),
743
device=perm.device,
744
dtype=perm.dtype)
@@ -808,6 +809,8 @@ class Qwen2_5_VisionTransformer(nn.Module):
809
non_blocking=True)
810
window_index = window_index.to(device=hidden_states.device,
811
non_blocking=True)
812
+ reverse_indices = reverse_indices.to(device=hidden_states.device,
813
+ non_blocking=True)
814
815
hidden_states = hidden_states.reshape(
816
seq_len // self.spatial_merge_unit, self.spatial_merge_unit, -1)