A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Kernel] Support sliding window in flash attention backend (#9403)
Chen Zhang committed
Oct 20, 2024 at 10:57 UTC
4fa3e3334978dce74eba296ee8cc2e970ed20e5e
13 files changed
+41
-61
tests/kernels/test_attention_selector.py
+15
-20
@@ -20,21 +20,21 @@ def test_env(name: str, device: str, monkeypatch):
20
21
if device == "cpu":
22
with patch("vllm.attention.selector.is_cpu", return_value=True):
23
- backend = which_attn_to_use(16, None, torch.float16, torch.float16,
24
- 16, False)
23
+ backend = which_attn_to_use(16, torch.float16, torch.float16, 16,
24
+ False)
25
assert backend.name == "TORCH_SDPA"
26
elif device == "hip":
27
with patch("vllm.attention.selector.is_hip", return_value=True):
28
- backend = which_attn_to_use(16, None, torch.float16, torch.float16,
29
- 16, False)
28
+ backend = which_attn_to_use(16, torch.float16, torch.float16, 16,
29
+ False)
30
assert backend.name == "ROCM_FLASH"
31
elif device == "openvino":
32
with patch("vllm.attention.selector.is_openvino", return_value=True):
33
- backend = which_attn_to_use(16, None, torch.float16, torch.float16,
34
- 16, False)
33
+ backend = which_attn_to_use(16, torch.float16, torch.float16, 16,
34
+ False)
35
assert backend.name == "OPENVINO"
36
else:
37
- backend = which_attn_to_use(16, None, torch.float16, torch.float16, 16,
37
+ backend = which_attn_to_use(16, torch.float16, torch.float16, 16,
38
False)
39
assert backend.name == name
40
@@ -46,37 +46,32 @@ def test_flash_attn(monkeypatch):
46
47
# Unsupported CUDA arch
48
with patch("torch.cuda.get_device_capability", return_value=(7, 5)):
49
- backend = which_attn_to_use(16, None, torch.float16, None, 16, False)
49
+ backend = which_attn_to_use(16, torch.float16, None, 16, False)
50
assert backend.name != STR_FLASH_ATTN_VAL
51
52
# Unsupported data type
53
- backend = which_attn_to_use(16, None, torch.float8_e4m3fn, None, 16, False)
53
+ backend = which_attn_to_use(16, torch.float8_e4m3fn, None, 16, False)
54
assert backend.name != STR_FLASH_ATTN_VAL
55
56
# Unsupported kv cache data type
57
- backend = which_attn_to_use(16, None, torch.float16, "fp8", 16, False)
57
+ backend = which_attn_to_use(16, torch.float16, "fp8", 16, False)
58
assert backend.name != STR_FLASH_ATTN_VAL
59
60
# Unsupported block size
61
- backend = which_attn_to_use(16, None, torch.float16, None, 8, False)
62
- assert backend.name != STR_FLASH_ATTN_VAL
63
-
64
- # Unsupported sliding window
65
- backend = which_attn_to_use(16, 1, torch.float16, None, 16, False)
61
+ backend = which_attn_to_use(16, torch.float16, None, 8, False)
62
assert backend.name != STR_FLASH_ATTN_VAL
63
64
# flash-attn is not installed
65
with patch.dict('sys.modules', {'vllm_flash_attn': None}):
70
- backend = which_attn_to_use(16, None, torch.float16, None, 16, False)
66
+ backend = which_attn_to_use(16, torch.float16, None, 16, False)
67
assert backend.name != STR_FLASH_ATTN_VAL
68
69
# Unsupported head size
74
- backend = which_attn_to_use(17, None, torch.float16, None, 16, False)
70
+ backend = which_attn_to_use(17, torch.float16, None, 16, False)
71
assert backend.name != STR_FLASH_ATTN_VAL
72
73
# Attention-free models should bypass env and use PlaceholderAttention
78
- backend = which_attn_to_use(16, None, torch.float16, torch.float16, 16,
79
- True)
74
+ backend = which_attn_to_use(16, torch.float16, torch.float16, 16, True)
75
assert backend.name != STR_FLASH_ATTN_VAL
76
77
@@ -84,4 +79,4 @@ def test_invalid_env(monkeypatch):
79
"""Throw an exception if the backend name is invalid."""
80
override_backend_env_variable(monkeypatch, STR_INVALID_VAL)
81
with pytest.raises(ValueError):
87
- which_attn_to_use(16, None, torch.float16, None, 16, False)
82
+ which_attn_to_use(16, torch.float16, None, 16, False)
tests/kernels/test_flash_attn.py
+16
-13
@@ -78,6 +78,7 @@ def ref_paged_attn(
78
@pytest.mark.parametrize("dtype", DTYPES)
79
@pytest.mark.parametrize("soft_cap", [None, 10.0, 50.0])
80
@pytest.mark.parametrize("num_blocks", NUM_BLOCKS)
81
+@pytest.mark.parametrize("sliding_window", [None, 256])
82
@torch.inference_mode()
83
def test_flash_attn_with_paged_kv(
84
kv_lens: List[int],
@@ -87,6 +88,7 @@ def test_flash_attn_with_paged_kv(
88
block_size: int,
89
soft_cap: Optional[float],
90
num_blocks: int,
91
+ sliding_window: Optional[int],
92
) -> None:
93
torch.set_default_device("cuda")
94
seed_everything(0)
@@ -96,6 +98,8 @@ def test_flash_attn_with_paged_kv(
98
assert num_query_heads % num_kv_heads == 0
99
max_kv_len = max(kv_lens)
100
scale = head_size**-0.5
101
+ window_size = ((sliding_window - 1, 0) if sliding_window is not None else
102
+ (-1, -1))
103
104
query = torch.randn(num_seqs, num_query_heads, head_size, dtype=dtype)
105
key_cache = torch.randn(num_blocks,
@@ -121,18 +125,18 @@ def test_flash_attn_with_paged_kv(
125
block_table=block_tables,
126
cache_seqlens=kv_lens_tensor,
127
softcap=soft_cap if soft_cap is not None else 0,
128
+ window_size=window_size,
129
).squeeze(1)
130
126
- ref_output = ref_paged_attn(
127
- query=query,
128
- key_cache=key_cache,
129
- value_cache=value_cache,
130
- query_lens=[1] * num_seqs,
131
- kv_lens=kv_lens,
132
- block_tables=block_tables,
133
- scale=scale,
134
- soft_cap=soft_cap,
135
- )
131
+ ref_output = ref_paged_attn(query=query,
132
+ key_cache=key_cache,
133
+ value_cache=value_cache,
134
+ query_lens=[1] * num_seqs,
135
+ kv_lens=kv_lens,
136
+ block_tables=block_tables,
137
+ scale=scale,
138
+ soft_cap=soft_cap,
139
+ sliding_window=sliding_window)
140
torch.testing.assert_close(output, ref_output, atol=2e-2, rtol=1e-2), \
141
f"{torch.max(torch.abs(output - ref_output))}"
142
@@ -141,7 +145,7 @@ def test_flash_attn_with_paged_kv(
145
@pytest.mark.parametrize("num_heads", NUM_HEADS)
146
@pytest.mark.parametrize("head_size", HEAD_SIZES)
147
@pytest.mark.parametrize("block_size", BLOCK_SIZES)
144
-@pytest.mark.parametrize("sliding_window", [None])
148
+@pytest.mark.parametrize("sliding_window", [None, 256])
149
@pytest.mark.parametrize("dtype", DTYPES)
150
@pytest.mark.parametrize("soft_cap", [None, 10.0, 50.0])
151
@pytest.mark.parametrize("num_blocks", NUM_BLOCKS)
@@ -166,8 +170,7 @@ def test_varlen_with_paged_kv(
170
assert num_query_heads % num_kv_heads == 0
171
max_query_len = max(query_lens)
172
max_kv_len = max(kv_lens)
169
- window_size = ((sliding_window,
170
- sliding_window) if sliding_window is not None else
173
+ window_size = ((sliding_window - 1, 0) if sliding_window is not None else
174
(-1, -1))
175
scale = head_size**-0.5
176
vllm/attention/backends/flash_attn.py
+5
-8
@@ -524,8 +524,8 @@ class FlashAttentionImpl(AttentionImpl):
524
if alibi_slopes is not None:
525
alibi_slopes = torch.tensor(alibi_slopes, dtype=torch.float32)
526
self.alibi_slopes = alibi_slopes
527
- self.sliding_window = ((sliding_window, sliding_window)
528
- if sliding_window is not None else (-1, -1))
527
+ self.sliding_window = ((sliding_window - 1,
528
+ 0) if sliding_window is not None else (-1, -1))
529
self.kv_cache_dtype = kv_cache_dtype
530
if logits_soft_cap is None:
531
# In flash-attn, setting logits_soft_cap as 0 means no soft cap.
@@ -535,12 +535,6 @@ class FlashAttentionImpl(AttentionImpl):
535
assert self.num_heads % self.num_kv_heads == 0
536
self.num_queries_per_kv = self.num_heads // self.num_kv_heads
537
538
- if sliding_window is not None:
539
- # NOTE(woosuk): flash-attn's sliding window does not work with
540
- # paged KV cache.
541
- raise ValueError(
542
- "Sliding window is not supported in FlashAttention.")
543
-
538
support_head_sizes = FlashAttentionBackend.get_supported_head_sizes()
539
if head_size not in support_head_sizes:
540
raise ValueError(
@@ -704,6 +698,7 @@ def unified_flash_attention(
698
max_seqlen_k=max_seq_len,
699
softmax_scale=softmax_scale,
700
causal=True,
701
+ window_size=window_size,
702
alibi_slopes=alibi_slopes,
703
block_table=prefill_meta.block_tables,
704
softcap=logits_soft_cap,
@@ -725,6 +720,7 @@ def unified_flash_attention(
720
max_seqlen_k=decode_meta.max_decode_seq_len,
721
softmax_scale=softmax_scale,
722
causal=True,
723
+ window_size=window_size,
724
alibi_slopes=alibi_slopes,
725
softcap=logits_soft_cap,
726
block_table=decode_meta.block_tables,
@@ -739,6 +735,7 @@ def unified_flash_attention(
735
cache_seqlens=decode_meta.seq_lens_tensor,
736
softmax_scale=softmax_scale,
737
causal=True,
738
+ window_size=window_size,
739
alibi_slopes=alibi_slopes,
740
softcap=logits_soft_cap,
741
).squeeze(1)
vllm/attention/layer.py
+3
-4
@@ -78,10 +78,9 @@ class Attention(nn.Module):
78
# During model initialization, the default dtype is set as the model
79
# weight and activation dtype.
80
dtype = torch.get_default_dtype()
81
- attn_backend = get_attn_backend(head_size, sliding_window, dtype,
82
- kv_cache_dtype, block_size,
83
- is_attention_free, blocksparse_params
84
- is not None)
81
+ attn_backend = get_attn_backend(head_size, dtype, kv_cache_dtype,
82
+ block_size, is_attention_free,
83
+ blocksparse_params is not None)
84
impl_cls = attn_backend.get_impl_cls()
85
self.impl = impl_cls(num_heads, head_size, scale, num_kv_heads,
86
alibi_slopes, sliding_window, kv_cache_dtype,
vllm/attention/selector.py
+2
-8
@@ -90,7 +90,6 @@ def get_global_forced_attn_backend() -> Optional[_Backend]:
90
@lru_cache(maxsize=None)
91
def get_attn_backend(
92
head_size: int,
93
- sliding_window: Optional[int],
93
dtype: torch.dtype,
94
kv_cache_dtype: Optional[str],
95
block_size: int,
@@ -105,8 +104,8 @@ def get_attn_backend(
104
BlocksparseFlashAttentionBackend)
105
return BlocksparseFlashAttentionBackend
106
108
- backend = which_attn_to_use(head_size, sliding_window, dtype,
109
- kv_cache_dtype, block_size, is_attention_free)
107
+ backend = which_attn_to_use(head_size, dtype, kv_cache_dtype, block_size,
108
+ is_attention_free)
109
if backend == _Backend.FLASH_ATTN:
110
from vllm.attention.backends.flash_attn import ( # noqa: F401
111
FlashAttentionBackend)
@@ -155,7 +154,6 @@ def get_attn_backend(
154
155
def which_attn_to_use(
156
head_size: int,
158
- sliding_window: Optional[int],
157
dtype: torch.dtype,
158
kv_cache_dtype: Optional[str],
159
block_size: int,
@@ -243,10 +241,6 @@ def which_attn_to_use(
241
"Cannot use FlashAttention-2 backend for block size not "
242
"divisible by 16.")
243
selected_backend = _Backend.XFORMERS
246
- elif sliding_window is not None:
247
- logger.info(
248
- "Cannot use FlashAttention-2 backend due to sliding window.")
249
- selected_backend = _Backend.XFORMERS
244
245
# FlashAttn is valid for the model, checking if the package is installed.
246
if selected_backend == _Backend.FLASH_ATTN:
vllm/worker/cache_engine.py
-1
@@ -53,7 +53,6 @@ class CacheEngine:
53
54
# Get attention backend.
55
self.attn_backend = get_attn_backend(self.head_size,
56
- model_config.get_sliding_window(),
56
model_config.dtype,
57
cache_config.cache_dtype,
58
self.block_size,
vllm/worker/cpu_model_runner.py
-1
@@ -420,7 +420,6 @@ class CPUModelRunner(ModelRunnerBase[ModelInputForCPU]):
420
self.block_size = cache_config.block_size
421
self.attn_backend = get_attn_backend(
422
self.model_config.get_head_size(),
423
- self.model_config.get_sliding_window(),
423
self.model_config.dtype,
424
self.kv_cache_dtype,
425
self.block_size,
vllm/worker/cpu_worker.py
-1
@@ -57,7 +57,6 @@ class CPUCacheEngine:
57
# Get attention backend.
58
self.attn_backend = get_attn_backend(
59
self.model_config.get_head_size(),
60
- self.model_config.get_sliding_window(),
60
self.model_config.dtype,
61
cache_config.cache_dtype,
62
self.block_size,
vllm/worker/model_runner.py
-1
@@ -1011,7 +1011,6 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1011
1012
self.attn_backend = get_attn_backend(
1013
self.model_config.get_head_size(),
1014
- self.model_config.get_sliding_window(),
1014
self.model_config.dtype,
1015
self.kv_cache_dtype,
1016
self.block_size,
vllm/worker/openvino_model_runner.py
-1
@@ -75,7 +75,6 @@ class OpenVINOModelRunner:
75
76
self.attn_backend = get_attn_backend(
77
self.model_config.get_head_size(),
78
- self.model_config.get_sliding_window(),
78
self.model_config.dtype,
79
self.kv_cache_dtype,
80
self.block_size,
vllm/worker/openvino_worker.py
-1
@@ -71,7 +71,6 @@ class OpenVINOCacheEngine:
71
# Get attention backend.
72
self.attn_backend = get_attn_backend(
73
self.head_size,
74
- self.model_config.get_sliding_window(),
74
self.model_config.dtype,
75
self.cache_config.cache_dtype,
76
self.block_size,
vllm/worker/tpu_model_runner.py
-1
@@ -114,7 +114,6 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
114
dtype=np.int32)
115
self.attn_backend = get_attn_backend(
116
self.model_config.get_head_size(),
117
- self.model_config.get_sliding_window(),
117
self.model_config.dtype,
118
self.cache_config.cache_dtype,
119
self.block_size,
vllm/worker/xpu_model_runner.py
-1
@@ -374,7 +374,6 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPUWithSamplingMetadata]):
374
375
self.attn_backend = get_attn_backend(
376
self.model_config.get_head_size(),
377
- self.model_config.get_sliding_window(),
377
self.model_config.dtype,
378
self.kv_cache_dtype,
379
self.block_size,