@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4fa3e33349
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Kernel] Support sliding window in flash attention backend (#9403)

Chen Zhang committed Oct 20, 2024 at 10:57 UTC 4fa3e3334978dce74eba296ee8cc2e970ed20e5e
13 files changed +41 -61
tests/kernels/test_attention_selector.py
+15 -20
@@ -20,21 +20,21 @@ def test_env(name: str, device: str, monkeypatch):
20
21 if device == "cpu":
22 with patch("vllm.attention.selector.is_cpu", return_value=True):
23 - backend = which_attn_to_use(16, None, torch.float16, torch.float16,
24 - 16, False)
23 + backend = which_attn_to_use(16, torch.float16, torch.float16, 16,
24 + False)
25 assert backend.name == "TORCH_SDPA"
26 elif device == "hip":
27 with patch("vllm.attention.selector.is_hip", return_value=True):
28 - backend = which_attn_to_use(16, None, torch.float16, torch.float16,
29 - 16, False)
28 + backend = which_attn_to_use(16, torch.float16, torch.float16, 16,
29 + False)
30 assert backend.name == "ROCM_FLASH"
31 elif device == "openvino":
32 with patch("vllm.attention.selector.is_openvino", return_value=True):
33 - backend = which_attn_to_use(16, None, torch.float16, torch.float16,
34 - 16, False)
33 + backend = which_attn_to_use(16, torch.float16, torch.float16, 16,
34 + False)
35 assert backend.name == "OPENVINO"
36 else:
37 - backend = which_attn_to_use(16, None, torch.float16, torch.float16, 16,
37 + backend = which_attn_to_use(16, torch.float16, torch.float16, 16,
38 False)
39 assert backend.name == name
40
@@ -46,37 +46,32 @@ def test_flash_attn(monkeypatch):
46
47 # Unsupported CUDA arch
48 with patch("torch.cuda.get_device_capability", return_value=(7, 5)):
49 - backend = which_attn_to_use(16, None, torch.float16, None, 16, False)
49 + backend = which_attn_to_use(16, torch.float16, None, 16, False)
50 assert backend.name != STR_FLASH_ATTN_VAL
51
52 # Unsupported data type
53 - backend = which_attn_to_use(16, None, torch.float8_e4m3fn, None, 16, False)
53 + backend = which_attn_to_use(16, torch.float8_e4m3fn, None, 16, False)
54 assert backend.name != STR_FLASH_ATTN_VAL
55
56 # Unsupported kv cache data type
57 - backend = which_attn_to_use(16, None, torch.float16, "fp8", 16, False)
57 + backend = which_attn_to_use(16, torch.float16, "fp8", 16, False)
58 assert backend.name != STR_FLASH_ATTN_VAL
59
60 # Unsupported block size
61 - backend = which_attn_to_use(16, None, torch.float16, None, 8, False)
62 - assert backend.name != STR_FLASH_ATTN_VAL
63 -
64 - # Unsupported sliding window
65 - backend = which_attn_to_use(16, 1, torch.float16, None, 16, False)
61 + backend = which_attn_to_use(16, torch.float16, None, 8, False)
62 assert backend.name != STR_FLASH_ATTN_VAL
63
64 # flash-attn is not installed
65 with patch.dict('sys.modules', {'vllm_flash_attn': None}):
70 - backend = which_attn_to_use(16, None, torch.float16, None, 16, False)
66 + backend = which_attn_to_use(16, torch.float16, None, 16, False)
67 assert backend.name != STR_FLASH_ATTN_VAL
68
69 # Unsupported head size
74 - backend = which_attn_to_use(17, None, torch.float16, None, 16, False)
70 + backend = which_attn_to_use(17, torch.float16, None, 16, False)
71 assert backend.name != STR_FLASH_ATTN_VAL
72
73 # Attention-free models should bypass env and use PlaceholderAttention
78 - backend = which_attn_to_use(16, None, torch.float16, torch.float16, 16,
79 - True)
74 + backend = which_attn_to_use(16, torch.float16, torch.float16, 16, True)
75 assert backend.name != STR_FLASH_ATTN_VAL
76
77
@@ -84,4 +79,4 @@ def test_invalid_env(monkeypatch):
79 """Throw an exception if the backend name is invalid."""
80 override_backend_env_variable(monkeypatch, STR_INVALID_VAL)
81 with pytest.raises(ValueError):
87 - which_attn_to_use(16, None, torch.float16, None, 16, False)
82 + which_attn_to_use(16, torch.float16, None, 16, False)
tests/kernels/test_flash_attn.py
+16 -13
@@ -78,6 +78,7 @@ def ref_paged_attn(
78 @pytest.mark.parametrize("dtype", DTYPES)
79 @pytest.mark.parametrize("soft_cap", [None, 10.0, 50.0])
80 @pytest.mark.parametrize("num_blocks", NUM_BLOCKS)
81 +@pytest.mark.parametrize("sliding_window", [None, 256])
82 @torch.inference_mode()
83 def test_flash_attn_with_paged_kv(
84 kv_lens: List[int],
@@ -87,6 +88,7 @@ def test_flash_attn_with_paged_kv(
88 block_size: int,
89 soft_cap: Optional[float],
90 num_blocks: int,
91 + sliding_window: Optional[int],
92 ) -> None:
93 torch.set_default_device("cuda")
94 seed_everything(0)
@@ -96,6 +98,8 @@ def test_flash_attn_with_paged_kv(
98 assert num_query_heads % num_kv_heads == 0
99 max_kv_len = max(kv_lens)
100 scale = head_size**-0.5
101 + window_size = ((sliding_window - 1, 0) if sliding_window is not None else
102 + (-1, -1))
103
104 query = torch.randn(num_seqs, num_query_heads, head_size, dtype=dtype)
105 key_cache = torch.randn(num_blocks,
@@ -121,18 +125,18 @@ def test_flash_attn_with_paged_kv(
125 block_table=block_tables,
126 cache_seqlens=kv_lens_tensor,
127 softcap=soft_cap if soft_cap is not None else 0,
128 + window_size=window_size,
129 ).squeeze(1)
130
126 - ref_output = ref_paged_attn(
127 - query=query,
128 - key_cache=key_cache,
129 - value_cache=value_cache,
130 - query_lens=[1] * num_seqs,
131 - kv_lens=kv_lens,
132 - block_tables=block_tables,
133 - scale=scale,
134 - soft_cap=soft_cap,
135 - )
131 + ref_output = ref_paged_attn(query=query,
132 + key_cache=key_cache,
133 + value_cache=value_cache,
134 + query_lens=[1] * num_seqs,
135 + kv_lens=kv_lens,
136 + block_tables=block_tables,
137 + scale=scale,
138 + soft_cap=soft_cap,
139 + sliding_window=sliding_window)
140 torch.testing.assert_close(output, ref_output, atol=2e-2, rtol=1e-2), \
141 f"{torch.max(torch.abs(output - ref_output))}"
142
@@ -141,7 +145,7 @@ def test_flash_attn_with_paged_kv(
145 @pytest.mark.parametrize("num_heads", NUM_HEADS)
146 @pytest.mark.parametrize("head_size", HEAD_SIZES)
147 @pytest.mark.parametrize("block_size", BLOCK_SIZES)
144 -@pytest.mark.parametrize("sliding_window", [None])
148 +@pytest.mark.parametrize("sliding_window", [None, 256])
149 @pytest.mark.parametrize("dtype", DTYPES)
150 @pytest.mark.parametrize("soft_cap", [None, 10.0, 50.0])
151 @pytest.mark.parametrize("num_blocks", NUM_BLOCKS)
@@ -166,8 +170,7 @@ def test_varlen_with_paged_kv(
170 assert num_query_heads % num_kv_heads == 0
171 max_query_len = max(query_lens)
172 max_kv_len = max(kv_lens)
169 - window_size = ((sliding_window,
170 - sliding_window) if sliding_window is not None else
173 + window_size = ((sliding_window - 1, 0) if sliding_window is not None else
174 (-1, -1))
175 scale = head_size**-0.5
176
vllm/attention/backends/flash_attn.py
+5 -8
@@ -524,8 +524,8 @@ class FlashAttentionImpl(AttentionImpl):
524 if alibi_slopes is not None:
525 alibi_slopes = torch.tensor(alibi_slopes, dtype=torch.float32)
526 self.alibi_slopes = alibi_slopes
527 - self.sliding_window = ((sliding_window, sliding_window)
528 - if sliding_window is not None else (-1, -1))
527 + self.sliding_window = ((sliding_window - 1,
528 + 0) if sliding_window is not None else (-1, -1))
529 self.kv_cache_dtype = kv_cache_dtype
530 if logits_soft_cap is None:
531 # In flash-attn, setting logits_soft_cap as 0 means no soft cap.
@@ -535,12 +535,6 @@ class FlashAttentionImpl(AttentionImpl):
535 assert self.num_heads % self.num_kv_heads == 0
536 self.num_queries_per_kv = self.num_heads // self.num_kv_heads
537
538 - if sliding_window is not None:
539 - # NOTE(woosuk): flash-attn's sliding window does not work with
540 - # paged KV cache.
541 - raise ValueError(
542 - "Sliding window is not supported in FlashAttention.")
543 -
538 support_head_sizes = FlashAttentionBackend.get_supported_head_sizes()
539 if head_size not in support_head_sizes:
540 raise ValueError(
@@ -704,6 +698,7 @@ def unified_flash_attention(
698 max_seqlen_k=max_seq_len,
699 softmax_scale=softmax_scale,
700 causal=True,
701 + window_size=window_size,
702 alibi_slopes=alibi_slopes,
703 block_table=prefill_meta.block_tables,
704 softcap=logits_soft_cap,
@@ -725,6 +720,7 @@ def unified_flash_attention(
720 max_seqlen_k=decode_meta.max_decode_seq_len,
721 softmax_scale=softmax_scale,
722 causal=True,
723 + window_size=window_size,
724 alibi_slopes=alibi_slopes,
725 softcap=logits_soft_cap,
726 block_table=decode_meta.block_tables,
@@ -739,6 +735,7 @@ def unified_flash_attention(
735 cache_seqlens=decode_meta.seq_lens_tensor,
736 softmax_scale=softmax_scale,
737 causal=True,
738 + window_size=window_size,
739 alibi_slopes=alibi_slopes,
740 softcap=logits_soft_cap,
741 ).squeeze(1)
vllm/attention/layer.py
+3 -4
@@ -78,10 +78,9 @@ class Attention(nn.Module):
78 # During model initialization, the default dtype is set as the model
79 # weight and activation dtype.
80 dtype = torch.get_default_dtype()
81 - attn_backend = get_attn_backend(head_size, sliding_window, dtype,
82 - kv_cache_dtype, block_size,
83 - is_attention_free, blocksparse_params
84 - is not None)
81 + attn_backend = get_attn_backend(head_size, dtype, kv_cache_dtype,
82 + block_size, is_attention_free,
83 + blocksparse_params is not None)
84 impl_cls = attn_backend.get_impl_cls()
85 self.impl = impl_cls(num_heads, head_size, scale, num_kv_heads,
86 alibi_slopes, sliding_window, kv_cache_dtype,
vllm/attention/selector.py
+2 -8
@@ -90,7 +90,6 @@ def get_global_forced_attn_backend() -> Optional[_Backend]:
90 @lru_cache(maxsize=None)
91 def get_attn_backend(
92 head_size: int,
93 - sliding_window: Optional[int],
93 dtype: torch.dtype,
94 kv_cache_dtype: Optional[str],
95 block_size: int,
@@ -105,8 +104,8 @@ def get_attn_backend(
104 BlocksparseFlashAttentionBackend)
105 return BlocksparseFlashAttentionBackend
106
108 - backend = which_attn_to_use(head_size, sliding_window, dtype,
109 - kv_cache_dtype, block_size, is_attention_free)
107 + backend = which_attn_to_use(head_size, dtype, kv_cache_dtype, block_size,
108 + is_attention_free)
109 if backend == _Backend.FLASH_ATTN:
110 from vllm.attention.backends.flash_attn import ( # noqa: F401
111 FlashAttentionBackend)
@@ -155,7 +154,6 @@ def get_attn_backend(
154
155 def which_attn_to_use(
156 head_size: int,
158 - sliding_window: Optional[int],
157 dtype: torch.dtype,
158 kv_cache_dtype: Optional[str],
159 block_size: int,
@@ -243,10 +241,6 @@ def which_attn_to_use(
241 "Cannot use FlashAttention-2 backend for block size not "
242 "divisible by 16.")
243 selected_backend = _Backend.XFORMERS
246 - elif sliding_window is not None:
247 - logger.info(
248 - "Cannot use FlashAttention-2 backend due to sliding window.")
249 - selected_backend = _Backend.XFORMERS
244
245 # FlashAttn is valid for the model, checking if the package is installed.
246 if selected_backend == _Backend.FLASH_ATTN:
vllm/worker/cache_engine.py
-1
@@ -53,7 +53,6 @@ class CacheEngine:
53
54 # Get attention backend.
55 self.attn_backend = get_attn_backend(self.head_size,
56 - model_config.get_sliding_window(),
56 model_config.dtype,
57 cache_config.cache_dtype,
58 self.block_size,
vllm/worker/cpu_model_runner.py
-1
@@ -420,7 +420,6 @@ class CPUModelRunner(ModelRunnerBase[ModelInputForCPU]):
420 self.block_size = cache_config.block_size
421 self.attn_backend = get_attn_backend(
422 self.model_config.get_head_size(),
423 - self.model_config.get_sliding_window(),
423 self.model_config.dtype,
424 self.kv_cache_dtype,
425 self.block_size,
vllm/worker/cpu_worker.py
-1
@@ -57,7 +57,6 @@ class CPUCacheEngine:
57 # Get attention backend.
58 self.attn_backend = get_attn_backend(
59 self.model_config.get_head_size(),
60 - self.model_config.get_sliding_window(),
60 self.model_config.dtype,
61 cache_config.cache_dtype,
62 self.block_size,
vllm/worker/model_runner.py
-1
@@ -1011,7 +1011,6 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1011
1012 self.attn_backend = get_attn_backend(
1013 self.model_config.get_head_size(),
1014 - self.model_config.get_sliding_window(),
1014 self.model_config.dtype,
1015 self.kv_cache_dtype,
1016 self.block_size,
vllm/worker/openvino_model_runner.py
-1
@@ -75,7 +75,6 @@ class OpenVINOModelRunner:
75
76 self.attn_backend = get_attn_backend(
77 self.model_config.get_head_size(),
78 - self.model_config.get_sliding_window(),
78 self.model_config.dtype,
79 self.kv_cache_dtype,
80 self.block_size,
vllm/worker/openvino_worker.py
-1
@@ -71,7 +71,6 @@ class OpenVINOCacheEngine:
71 # Get attention backend.
72 self.attn_backend = get_attn_backend(
73 self.head_size,
74 - self.model_config.get_sliding_window(),
74 self.model_config.dtype,
75 self.cache_config.cache_dtype,
76 self.block_size,
vllm/worker/tpu_model_runner.py
-1
@@ -114,7 +114,6 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
114 dtype=np.int32)
115 self.attn_backend = get_attn_backend(
116 self.model_config.get_head_size(),
117 - self.model_config.get_sliding_window(),
117 self.model_config.dtype,
118 self.cache_config.cache_dtype,
119 self.block_size,
vllm/worker/xpu_model_runner.py
-1
@@ -374,7 +374,6 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPUWithSamplingMetadata]):
374
375 self.attn_backend = get_attn_backend(
376 self.model_config.get_head_size(),
377 - self.model_config.get_sliding_window(),
377 self.model_config.dtype,
378 self.kv_cache_dtype,
379 self.block_size,