@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9645b9f646
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[V1] Support sliding window attention (#9679)

Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Woosuk Kwon committed Oct 24, 2024 at 22:20 UTC 9645b9f646024b1e416ed5a61cfba7d14d54b571
1 file changed +4 -8
vllm/v1/attention/backends/flash_attn.py
+4 -8
@@ -82,8 +82,10 @@ class FlashAttentionImpl(AttentionImpl):
82 if alibi_slopes is not None:
83 alibi_slopes = torch.tensor(alibi_slopes, dtype=torch.float32)
84 self.alibi_slopes = alibi_slopes
85 - self.sliding_window = ((sliding_window, sliding_window)
86 - if sliding_window is not None else (-1, -1))
85 + if sliding_window is None:
86 + self.sliding_window = (-1, -1)
87 + else:
88 + self.sliding_window = (sliding_window - 1, 0)
89 self.kv_cache_dtype = kv_cache_dtype
90 if logits_soft_cap is None:
91 # In flash-attn, setting logits_soft_cap as 0 means no soft cap.
@@ -93,12 +95,6 @@ class FlashAttentionImpl(AttentionImpl):
95 assert self.num_heads % self.num_kv_heads == 0
96 self.num_queries_per_kv = self.num_heads // self.num_kv_heads
97
96 - if sliding_window is not None:
97 - # NOTE(woosuk): flash-attn's sliding window does not work with
98 - # paged KV cache.
99 - raise ValueError(
100 - "Sliding window is not supported in FlashAttention.")
101 -
98 support_head_sizes = FlashAttentionBackend.get_supported_head_sizes()
99 if head_size not in support_head_sizes:
100 raise ValueError(