A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 11:00 UTC
[Misc] Use `apply_rotary_emb` from vllm_flash_attn for Qwen2-VL vision RoPE (#17726)
Signed-off-by: Isotr0py <2037008807@qq.com>
Isotr0py committed
May 7, 2025 at 12:51 UTC
c3e9d5060e896597c7428c2dc35223c8e4cd4698
2 files changed
+6
-12
vllm/model_executor/models/qwen2_5_vl.py
+2
-7
@@ -297,13 +297,8 @@ class Qwen2_5_VisionAttention(nn.Module):
297
q, k, v = (rearrange(x, "s b ... -> b s ...").contiguous()
298
for x in (q, k, v))
299
if rotary_pos_emb is not None:
300
- use_flash_attn = self.attn_backend == _Backend.FLASH_ATTN
301
- q = apply_rotary_pos_emb_vision(q,
302
- rotary_pos_emb,
303
- use_flash_attn=use_flash_attn)
304
- k = apply_rotary_pos_emb_vision(k,
305
- rotary_pos_emb,
306
- use_flash_attn=use_flash_attn)
300
+ q = apply_rotary_pos_emb_vision(q, rotary_pos_emb)
301
+ k = apply_rotary_pos_emb_vision(k, rotary_pos_emb)
302
303
if self.attn_backend == _Backend.FLASH_ATTN:
304
# from vllm_flash_attn.flash_attn_interface import (
vllm/model_executor/models/qwen2_vl.py
+4
-5
@@ -64,7 +64,7 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor,
64
BaseProcessingInfo, PromptReplacement,
65
PromptUpdate)
66
from vllm.multimodal.profiling import BaseDummyInputsBuilder
67
-from vllm.platforms import _Backend
67
+from vllm.platforms import _Backend, current_platform
68
from vllm.sequence import IntermediateTensors
69
from vllm.transformers_utils.config import uses_mrope
70
from vllm.transformers_utils.processor import (
@@ -230,14 +230,13 @@ def apply_rotary_emb_torch(x: torch.Tensor,
230
231
232
def apply_rotary_pos_emb_vision(t: torch.Tensor,
233
- freqs: torch.Tensor,
234
- use_flash_attn=False) -> torch.Tensor:
233
+ freqs: torch.Tensor) -> torch.Tensor:
234
t_ = t.float()
235
cos = freqs.cos()
236
sin = freqs.sin()
237
apply_rotary_emb = apply_rotary_emb_torch
239
- if use_flash_attn:
240
- from flash_attn.layers.rotary import apply_rotary_emb
238
+ if current_platform.is_cuda():
239
+ from vllm.vllm_flash_attn.layers.rotary import apply_rotary_emb
240
output = apply_rotary_emb(t_, cos, sin).type_as(t)
241
return output
242