A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[BugFix][QWEN-VL]fix wrong apply_rotary_emb_torch selection introduced by #24642 (#26123)
Signed-off-by: Chendi Xue <Chendi.Xue@intel.com> Signed-off-by: Chendi.Xue <chendi.xue@intel.com> Co-authored-by: Roger Wang <hey@rogerw.io>
Chendi.Xue committed
Oct 3, 2025 at 10:52 UTC
dd96465fd7447c0de06af796b6c9bd3949913ea6
2 files changed
+10
-4
vllm/model_executor/layers/rotary_embedding/common.py
+8
-3
@@ -4,7 +4,7 @@
4
import math
5
from functools import cache
6
from importlib.util import find_spec
7
-from typing import Callable
7
+from typing import Callable, Optional
8
9
import torch
10
@@ -72,7 +72,9 @@ def apply_rotary_emb_dispatch(x: torch.Tensor, cos: torch.Tensor,
72
73
74
@cache
75
-def dispatch_rotary_emb_function() -> Callable[..., torch.Tensor]:
75
+def dispatch_rotary_emb_function(
76
+ default: Optional[Callable[..., torch.Tensor]] = None
77
+) -> Callable[..., torch.Tensor]:
78
if current_platform.is_cuda():
79
return apply_rotary_emb
80
@@ -85,7 +87,10 @@ def dispatch_rotary_emb_function() -> Callable[..., torch.Tensor]:
87
"flash_attn is not installed. Falling back to PyTorch "
88
"implementation for rotary embeddings.")
89
88
- return apply_rotary_emb_torch
90
+ if default is not None:
91
+ return default
92
+ else:
93
+ return apply_rotary_emb_torch
94
95
96
# yarn functions
vllm/model_executor/models/qwen2_vl.py
+2
-1
@@ -276,7 +276,8 @@ def apply_rotary_emb_torch(x: torch.Tensor,
276
277
def apply_rotary_pos_emb_vision(t: torch.Tensor,
278
freqs: torch.Tensor) -> torch.Tensor:
279
- rotary_emb_function = dispatch_rotary_emb_function()
279
+ rotary_emb_function = dispatch_rotary_emb_function(
280
+ default=apply_rotary_emb_torch)
281
t_ = t.float()
282
cos = freqs.cos()
283
sin = freqs.sin()