A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 18:30 UTC
[Bugfix] Fix incorrect qwen2.5-vl attention mask pre-computation (#15200)
Signed-off-by: Isotr0py <2037008807@qq.com>
Isotr0py committed
Mar 21, 2025 at 10:18 UTC
1e508343e1ec6c5466586a9bcb7b229f0d879c76
3 files changed
+37
-4
tests/models/decoder_only/vision_language/test_models.py
+13
@@ -508,6 +508,19 @@ VLM_TEST_SETTINGS = {
508
limit_mm_per_prompt={"image": 4},
509
)],
510
),
511
+ # regression test for https://github.com/vllm-project/vllm/issues/15122
512
+ "qwen2_5_vl-windows-attention": VLMTestInfo(
513
+ models=["Qwen/Qwen2.5-VL-3B-Instruct"],
514
+ test_type=VLMTestType.CUSTOM_INPUTS,
515
+ max_model_len=4096,
516
+ max_num_seqs=2,
517
+ auto_cls=AutoModelForVision2Seq,
518
+ vllm_output_post_proc=model_utils.qwen2_vllm_to_hf_output,
519
+ custom_test_opts=[CustomTestOptions(
520
+ inputs=custom_inputs.windows_attention_image_qwen2_5_vl(),
521
+ limit_mm_per_prompt={"image": 1},
522
+ )],
523
+ ),
524
}
525
# yapf: enable
526
tests/models/decoder_only/vision_language/vlm_utils/custom_inputs.py
+18
@@ -1,7 +1,11 @@
1
# SPDX-License-Identifier: Apache-2.0
2
"""Custom input builders for edge-cases in different models."""
3
+from io import BytesIO
4
from typing import Callable
5
6
+import requests
7
+from PIL import Image
8
+
9
from vllm.multimodal.image import rescale_image_size
10
from vllm.multimodal.video import (rescale_video_size, resize_video,
11
sample_frames_from_video)
@@ -102,3 +106,17 @@ def different_patch_input_cases_internvl():
106
build_single_image_inputs(images, formatted_sprompts, wrapped_sf),
107
build_multi_image_inputs([images], formatted_mprompts, wrapped_sf),
108
]
109
+
110
+
111
+def windows_attention_image_qwen2_5_vl():
112
+ # image from regression issue: https://github.com/vllm-project/vllm/issues/15122
113
+ image_url = "https://aomediacodec.github.io/av1-avif/testFiles/Link-U/hato.jpg"
114
+ image = Image.open(BytesIO(requests.get(image_url).content))
115
+
116
+ question = "Describe the image."
117
+ img_prompt = "<|vision_start|><|image_pad|><|vision_end|>"
118
+ prompt = (f"<|im_start|>User\n{img_prompt}{question}<|im_end|>\n"
119
+ "<|im_start|>assistant\n")
120
+
121
+ wrapped_sf = ImageSizeWrapper(type=SizeType.SIZE_FACTOR, data=[0.5])
122
+ return build_single_image_inputs([image], [prompt], wrapped_sf)
vllm/model_executor/models/qwen2_5_vl.py
+6
-4
@@ -647,15 +647,17 @@ class Qwen2_5_VisionTransformer(nn.Module):
647
648
max_seqlen = None
649
seqlens = None
650
- if self.attn_backend == _Backend.FLASH_ATTN:
651
- max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max().item()
652
- elif self.attn_backend == _Backend.XFORMERS:
653
- seqlens = (cu_seqlens[1:] - cu_seqlens[:-1]).tolist()
650
for layer_num, blk in enumerate(self.blocks):
651
if layer_num in self.fullatt_block_indexes:
652
cu_seqlens_now = cu_seqlens
653
else:
654
cu_seqlens_now = cu_window_seqlens
655
+ # pre-compute cu_seqlens for window attn
656
+ if self.attn_backend == _Backend.FLASH_ATTN:
657
+ max_seqlen = (cu_seqlens_now[1:] -
658
+ cu_seqlens_now[:-1]).max().item()
659
+ elif self.attn_backend == _Backend.XFORMERS:
660
+ seqlens = (cu_seqlens_now[1:] - cu_seqlens_now[:-1]).tolist()
661
hidden_states = blk(
662
hidden_states,
663
cu_seqlens=cu_seqlens_now,