A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[V1] Support sliding window attention (#9679)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Woosuk Kwon committed
Oct 24, 2024 at 22:20 UTC
9645b9f646024b1e416ed5a61cfba7d14d54b571
1 file changed
+4
-8
vllm/v1/attention/backends/flash_attn.py
+4
-8
@@ -82,8 +82,10 @@ class FlashAttentionImpl(AttentionImpl):
82
if alibi_slopes is not None:
83
alibi_slopes = torch.tensor(alibi_slopes, dtype=torch.float32)
84
self.alibi_slopes = alibi_slopes
85
- self.sliding_window = ((sliding_window, sliding_window)
86
- if sliding_window is not None else (-1, -1))
85
+ if sliding_window is None:
86
+ self.sliding_window = (-1, -1)
87
+ else:
88
+ self.sliding_window = (sliding_window - 1, 0)
89
self.kv_cache_dtype = kv_cache_dtype
90
if logits_soft_cap is None:
91
# In flash-attn, setting logits_soft_cap as 0 means no soft cap.
@@ -93,12 +95,6 @@ class FlashAttentionImpl(AttentionImpl):
95
assert self.num_heads % self.num_kv_heads == 0
96
self.num_queries_per_kv = self.num_heads // self.num_kv_heads
97
96
- if sliding_window is not None:
97
- # NOTE(woosuk): flash-attn's sliding window does not work with
98
- # paged KV cache.
99
- raise ValueError(
100
- "Sliding window is not supported in FlashAttention.")
101
-
98
support_head_sizes = FlashAttentionBackend.get_supported_head_sizes()
99
if head_size not in support_head_sizes:
100
raise ValueError(