A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 21:00 UTC
[Bugfix] Initialize attention bias on the same device as Query/Key/Value for QwenVL Series (#14031)
Yang Liu committed
Feb 28, 2025 at 23:36 UTC
9b61dd41e7f515c5c6a141a802cda86075bd9625
2 files changed
+4
-2
vllm/model_executor/models/qwen2_5_vl.py
+2
-1
@@ -323,7 +323,8 @@ class Qwen2_5_VisionAttention(nn.Module):
323
324
seqlens = (cu_seqlens[1:] - cu_seqlens[:-1]).tolist()
325
attn_bias = BlockDiagonalMask.from_seqlens(q_seqlen=seqlens,
326
- kv_seqlen=None)
326
+ kv_seqlen=None,
327
+ device=q.device)
328
329
context_layer = xops.memory_efficient_attention_forward(
330
q, k, v, attn_bias=attn_bias, p=0, scale=None)
vllm/model_executor/models/qwen2_vl.py
+2
-1
@@ -367,7 +367,8 @@ class Qwen2VisionAttention(nn.Module):
367
368
seqlens = (cu_seqlens[1:] - cu_seqlens[:-1]).tolist()
369
attn_bias = BlockDiagonalMask.from_seqlens(q_seqlen=seqlens,
370
- kv_seqlen=None)
370
+ kv_seqlen=None,
371
+ device=q.device)
372
373
context_layer = xops.memory_efficient_attention_forward(
374
q, k, v, attn_bias=attn_bias, p=0, scale=None)