A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Misc] Support attention logits soft-capping with flash-attn (#7022)
Woosuk Kwon committed
Aug 1, 2024 at 13:14 UTC
805a8a75f2f17ee56c0882efcc34d35e1801cbee
14 files changed
+71
-47
requirements-cuda.txt
+1
-1
@@ -8,4 +8,4 @@ torch == 2.4.0
8
# These must be updated alongside torch
9
torchvision == 0.19 # Required for phi3v processor. See https://github.com/pytorch/vision?tab=readme-ov-file#installation for corresponding version
10
xformers == 0.0.27.post2 # Requires PyTorch 2.4.0
11
-vllm-flash-attn == 2.6.0 # Requires PyTorch 2.4.0
11
+vllm-flash-attn == 2.6.1 # Requires PyTorch 2.4.0
tests/kernels/test_flash_attn.py
+13
-6
@@ -20,6 +20,7 @@ def ref_paged_attn(
20
block_tables: torch.Tensor,
21
scale: float,
22
sliding_window: Optional[int] = None,
23
+ soft_cap: Optional[float] = None,
24
) -> torch.Tensor:
25
num_seqs = len(query_lens)
26
block_tables = block_tables.cpu().numpy()
@@ -53,6 +54,8 @@ def ref_paged_attn(
54
(query_len + sliding_window) +
55
1).bool().logical_not()
56
mask |= sliding_window_mask
57
+ if soft_cap is not None:
58
+ attn = soft_cap * torch.tanh(attn / soft_cap)
59
attn.masked_fill_(mask, float("-inf"))
60
attn = torch.softmax(attn, dim=-1).to(v.dtype)
61
out = torch.einsum("hqk,khd->qhd", attn, v)
@@ -68,13 +71,15 @@ def ref_paged_attn(
71
@pytest.mark.parametrize("head_size", HEAD_SIZES)
72
@pytest.mark.parametrize("block_size", BLOCK_SIZES)
73
@pytest.mark.parametrize("dtype", DTYPES)
71
-@torch.inference_mode
74
+@pytest.mark.parametrize("soft_cap", [None, 10.0, 50.0])
75
+@torch.inference_mode()
76
def test_flash_attn_with_paged_kv(
77
kv_lens: List[int],
78
num_heads: Tuple[int, int],
79
head_size: int,
80
dtype: torch.dtype,
81
block_size: int,
82
+ soft_cap: Optional[float],
83
) -> None:
84
torch.set_default_device("cuda")
85
torch.cuda.manual_seed_all(0)
@@ -108,6 +113,7 @@ def test_flash_attn_with_paged_kv(
113
causal=True,
114
block_table=block_tables,
115
cache_seqlens=kv_lens_tensor,
116
+ softcap=soft_cap if soft_cap is not None else 0,
117
).squeeze(1)
118
119
ref_output = ref_paged_attn(
@@ -118,6 +124,7 @@ def test_flash_attn_with_paged_kv(
124
kv_lens=kv_lens,
125
block_tables=block_tables,
126
scale=scale,
127
+ soft_cap=soft_cap,
128
)
129
assert torch.allclose(output, ref_output, atol=1e-2, rtol=1e-2), \
130
f"{torch.max(torch.abs(output - ref_output))}"
@@ -129,7 +136,8 @@ def test_flash_attn_with_paged_kv(
136
@pytest.mark.parametrize("block_size", BLOCK_SIZES)
137
@pytest.mark.parametrize("sliding_window", [None])
138
@pytest.mark.parametrize("dtype", DTYPES)
132
-@torch.inference_mode
139
+@pytest.mark.parametrize("soft_cap", [None, 10.0, 50.0])
140
+@torch.inference_mode()
141
def test_varlen_with_paged_kv(
142
seq_lens: List[Tuple[int, int]],
143
num_heads: Tuple[int, int],
@@ -137,6 +145,7 @@ def test_varlen_with_paged_kv(
145
sliding_window: Optional[int],
146
dtype: torch.dtype,
147
block_size: int,
148
+ soft_cap: Optional[float],
149
) -> None:
150
torch.set_default_device("cuda")
151
torch.cuda.manual_seed_all(0)
@@ -163,10 +172,6 @@ def test_varlen_with_paged_kv(
172
head_size,
173
dtype=dtype)
174
value_cache = torch.randn_like(key_cache)
166
- # Normalize the scale of the key and value caches to mitigate
167
- # numerical instability.
168
- key_cache /= head_size**0.5
169
- value_cache /= head_size**0.5
175
cu_query_lens = torch.tensor([0] + query_lens,
176
dtype=torch.int32).cumsum(dim=0,
177
dtype=torch.int32)
@@ -192,6 +197,7 @@ def test_varlen_with_paged_kv(
197
causal=True,
198
window_size=window_size,
199
block_table=block_tables,
200
+ softcap=soft_cap if soft_cap is not None else 0,
201
)
202
203
ref_output = ref_paged_attn(
@@ -203,6 +209,7 @@ def test_varlen_with_paged_kv(
209
block_tables=block_tables,
210
scale=scale,
211
sliding_window=sliding_window,
212
+ soft_cap=soft_cap,
213
)
214
assert torch.allclose(output, ref_output, atol=1e-2, rtol=1e-2), \
215
f"{torch.max(torch.abs(output - ref_output))}"
vllm/attention/backends/abstract.py
+1
@@ -150,6 +150,7 @@ class AttentionImpl(ABC, Generic[T]):
150
sliding_window: Optional[int] = None,
151
kv_cache_dtype: str = "auto",
152
blocksparse_params: Optional[Dict[str, Any]] = None,
153
+ logits_soft_cap: Optional[float] = None,
154
) -> None:
155
raise NotImplementedError
156
vllm/attention/backends/blocksparse_attn.py
+3
@@ -283,12 +283,15 @@ class BlocksparseFlashAttentionImpl(AttentionImpl):
283
sliding_window: Optional[int],
284
kv_cache_dtype: str,
285
blocksparse_params: Optional[Dict[str, Any]] = None,
286
+ logits_soft_cap: Optional[float] = None,
287
) -> None:
288
assert blocksparse_params is not None
289
assert alibi_slopes is None, ValueError(
290
"Alibi not support for blocksparse flash attention.")
291
assert sliding_window is None, ValueError(
292
"sliding_window is invalid for blocksparse attention.")
293
+ assert logits_soft_cap is None, ValueError(
294
+ "logits_soft_cap is invalid for blocksparse attention.")
295
296
if "num_heads" not in blocksparse_params:
297
blocksparse_params["num_heads"] = num_heads
vllm/attention/backends/flash_attn.py
+10
-11
@@ -288,15 +288,6 @@ class FlashAttentionMetadataBuilder(
288
device = self.runner.device
289
use_captured_graph = cuda_graph_pad_size != -1
290
291
- logits_soft_cap = getattr(self.runner.model_config.hf_config,
292
- "attn_logit_softcapping", None)
293
- if logits_soft_cap is not None:
294
- raise ValueError(
295
- "Please use Flashinfer backend for models with logits_soft_cap"
296
- " (i.e., Gemma-2). Otherwise, the output might be wrong."
297
- " Set Flashinfer backend by "
298
- "export VLLM_ATTENTION_BACKEND=FLASHINFER.")
299
-
291
max_query_len = max(query_lens)
292
max_prefill_seq_len = max(self.prefill_seq_lens, default=0)
293
max_decode_seq_len = max(self.curr_seq_lens, default=0)
@@ -405,9 +396,11 @@ class FlashAttentionImpl(AttentionImpl):
396
sliding_window: Optional[int],
397
kv_cache_dtype: str,
398
blocksparse_params: Optional[Dict[str, Any]] = None,
399
+ logits_soft_cap: Optional[float] = None,
400
) -> None:
409
- assert blocksparse_params is None, ValueError(
410
- "FlashAttention does not support block-sparse attention.")
401
+ if blocksparse_params is not None:
402
+ raise ValueError(
403
+ "FlashAttention does not support block-sparse attention.")
404
self.num_heads = num_heads
405
self.head_size = head_size
406
self.scale = float(scale)
@@ -418,6 +411,10 @@ class FlashAttentionImpl(AttentionImpl):
411
self.sliding_window = ((sliding_window, sliding_window)
412
if sliding_window is not None else (-1, -1))
413
self.kv_cache_dtype = kv_cache_dtype
414
+ if logits_soft_cap is None:
415
+ # In flash-attn, setting logits_soft_cap as 0 means no soft cap.
416
+ logits_soft_cap = 0
417
+ self.logits_soft_cap = logits_soft_cap
418
419
assert self.num_heads % self.num_kv_heads == 0
420
self.num_queries_per_kv = self.num_heads // self.num_kv_heads
@@ -525,6 +522,7 @@ class FlashAttentionImpl(AttentionImpl):
522
causal=True,
523
window_size=self.sliding_window,
524
alibi_slopes=self.alibi_slopes,
525
+ softcap=self.logits_soft_cap,
526
)
527
assert output[:num_prefill_tokens].shape == out.shape
528
output[:num_prefill_tokens] = out
@@ -544,6 +542,7 @@ class FlashAttentionImpl(AttentionImpl):
542
causal=True,
543
alibi_slopes=self.alibi_slopes,
544
block_table=prefill_meta.block_tables,
545
+ softcap=self.logits_soft_cap,
546
)
547
548
if decode_meta := attn_metadata.decode_metadata:
vllm/attention/backends/flashinfer.py
+5
-9
@@ -116,8 +116,6 @@ class FlashInferMetadata(AttentionMetadata):
116
# The data type of the paged kv cache
117
data_type: torch.dtype = None
118
device: torch.device = torch.device("cuda")
119
- # Only used by gemma2 model
120
- logits_soft_cap: Optional[float] = None
119
120
def __post_init__(self):
121
# Refer to
@@ -391,9 +389,6 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
389
dtype=torch.long,
390
device=device)
391
394
- logits_soft_cap = getattr(self.runner.model_config.hf_config,
395
- "attn_logit_softcapping", None)
396
-
392
if len(self.paged_kv_indptr) > 0:
393
paged_kv_indices_tensor = torch.tensor(self.paged_kv_indices,
394
device="cpu",
@@ -430,8 +425,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
425
query_start_loc=query_start_loc,
426
device=device,
427
data_type=kv_cache_dtype,
433
- use_cuda_graph=use_captured_graph,
434
- logits_soft_cap=logits_soft_cap)
428
+ use_cuda_graph=use_captured_graph)
429
430
431
class FlashInferImpl(AttentionImpl):
@@ -446,6 +440,7 @@ class FlashInferImpl(AttentionImpl):
440
sliding_window: Optional[int],
441
kv_cache_dtype: str,
442
blocksparse_params: Optional[Dict[str, Any]] = None,
443
+ logits_soft_cap: Optional[float] = None,
444
) -> None:
445
self.num_heads = num_heads
446
self.head_size = head_size
@@ -458,6 +453,7 @@ class FlashInferImpl(AttentionImpl):
453
raise ValueError("Sliding window is not supported in FlashInfer.")
454
self.sliding_window = (-1, -1)
455
self.kv_cache_dtype = kv_cache_dtype
456
+ self.logits_soft_cap = logits_soft_cap
457
458
assert self.num_heads % self.num_kv_heads == 0
459
self.num_queries_per_kv = self.num_heads // self.num_kv_heads
@@ -532,7 +528,7 @@ class FlashInferImpl(AttentionImpl):
528
output = prefill_meta.prefill_wrapper.forward(
529
query,
530
kv_cache,
535
- logits_soft_cap=attn_metadata.logits_soft_cap,
531
+ logits_soft_cap=self.logits_soft_cap,
532
causal=True)
533
else:
534
assert attn_metadata.decode_metadata is not None
@@ -541,5 +537,5 @@ class FlashInferImpl(AttentionImpl):
537
query,
538
kv_cache,
539
sm_scale=self.scale,
544
- logits_soft_cap=attn_metadata.logits_soft_cap)
540
+ logits_soft_cap=self.logits_soft_cap)
541
return output.view(num_tokens, hidden_size)
vllm/attention/backends/ipex_attn.py
+6
-2
@@ -105,9 +105,13 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
105
sliding_window: Optional[int],
106
kv_cache_dtype: str,
107
blocksparse_params: Optional[Dict[str, Any]] = None,
108
+ logits_soft_cap: Optional[float] = None,
109
) -> None:
109
- assert blocksparse_params is None, ValueError(
110
- "Torch SPDA does not support block-sparse attention.")
110
+ if blocksparse_params is not None:
111
+ raise ValueError(
112
+ "IPEX backend does not support block-sparse attention.")
113
+ if logits_soft_cap is not None:
114
+ raise ValueError("IPEX backend does not support logits_soft_cap.")
115
self.num_heads = num_heads
116
self.head_size = head_size
117
self.scale = float(scale)
vllm/attention/backends/pallas.py
+4
@@ -91,6 +91,7 @@ class PallasAttentionBackendImpl(AttentionImpl):
91
sliding_window: Optional[int],
92
kv_cache_dtype: str,
93
blocksparse_params: Optional[Dict[str, Any]] = None,
94
+ logits_soft_cap: Optional[float] = None,
95
) -> None:
96
self.num_heads = num_heads
97
self.head_size = head_size
@@ -109,6 +110,9 @@ class PallasAttentionBackendImpl(AttentionImpl):
110
raise NotImplementedError("FP8 KV cache dtype is not supported.")
111
if blocksparse_params is not None:
112
raise NotImplementedError("Blocksparse is not supported.")
113
+ if logits_soft_cap is not None:
114
+ raise NotImplementedError(
115
+ "Attention logits soft-capping is not supported.")
116
117
if torch_xla.tpu.version() < 4:
118
raise NotImplementedError("TPU version must be 4 or higher.")
vllm/attention/backends/rocm_flash_attn.py
+8
-2
@@ -244,9 +244,15 @@ class ROCmFlashAttentionImpl(AttentionImpl):
244
sliding_window: Optional[int],
245
kv_cache_dtype: str,
246
blocksparse_params: Optional[Dict[str, Any]] = None,
247
+ logits_soft_cap: Optional[float] = None,
248
) -> None:
248
- assert blocksparse_params is None, ValueError(
249
- "ROCFlashAttention does not support blocksparse attention.")
249
+ if blocksparse_params is not None:
250
+ raise ValueError(
251
+ "ROCmFlashAttention does not support blocksparse attention.")
252
+ if logits_soft_cap is not None:
253
+ raise ValueError(
254
+ "ROCmFlashAttention does not support attention logits soft "
255
+ "capping.")
256
self.num_heads = num_heads
257
self.head_size = head_size
258
self.scale = float(scale)
vllm/attention/backends/torch_sdpa.py
+6
-2
@@ -109,9 +109,13 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
109
sliding_window: Optional[int],
110
kv_cache_dtype: str,
111
blocksparse_params: Optional[Dict[str, Any]] = None,
112
+ logits_soft_cap: Optional[float] = None,
113
) -> None:
113
- assert blocksparse_params is None, ValueError(
114
- "Torch SPDA does not support block-sparse attention.")
114
+ if blocksparse_params is not None:
115
+ raise ValueError(
116
+ "Torch SPDA does not support block-sparse attention.")
117
+ if logits_soft_cap is not None:
118
+ raise ValueError("Torch SPDA does not support logits soft cap.")
119
self.num_heads = num_heads
120
self.head_size = head_size
121
self.scale = float(scale)
vllm/attention/backends/utils.py
-9
@@ -165,15 +165,6 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
165
device = self.runner.device
166
use_captured_graph = cuda_graph_pad_size != -1
167
168
- logits_soft_cap = getattr(self.runner.model_config.hf_config,
169
- "attn_logit_softcapping", None)
170
- if logits_soft_cap is not None:
171
- raise ValueError(
172
- "Please use Flashinfer backend for models with logits_soft_cap "
173
- "(i.e., Gemma-2). Otherwise, the output might be wrong. "
174
- "Set Flashinfer backend by "
175
- "export VLLM_ATTENTION_BACKEND=FLASHINFER.")
176
-
168
max_query_len = max(query_lens)
169
max_prefill_seq_len = max(self.prefill_seq_lens, default=0)
170
max_decode_seq_len = max(self.curr_seq_lens, default=0)
vllm/attention/backends/xformers.py
+7
-2
@@ -408,9 +408,14 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
408
sliding_window: Optional[int],
409
kv_cache_dtype: str,
410
blocksparse_params: Optional[Dict[str, Any]] = None,
411
+ logits_soft_cap: Optional[float] = None,
412
) -> None:
412
- assert blocksparse_params is None, ValueError(
413
- "XFormer does not support block-sparse attention.")
413
+ if blocksparse_params is not None:
414
+ raise ValueError(
415
+ "XFormers does not support block-sparse attention.")
416
+ if logits_soft_cap is not None:
417
+ raise ValueError(
418
+ "XFormers does not support attention logits soft capping.")
419
self.num_heads = num_heads
420
self.head_size = head_size
421
self.scale = float(scale)
vllm/attention/layer.py
+2
-1
@@ -34,6 +34,7 @@ class Attention(nn.Module):
34
cache_config: Optional[CacheConfig] = None,
35
quant_config: Optional[QuantizationConfig] = None,
36
blocksparse_params: Optional[Dict[str, Any]] = None,
37
+ logits_soft_cap: Optional[float] = None,
38
prefix: str = "",
39
) -> None:
40
super().__init__()
@@ -82,7 +83,7 @@ class Attention(nn.Module):
83
impl_cls = attn_backend.get_impl_cls()
84
self.impl = impl_cls(num_heads, head_size, scale, num_kv_heads,
85
alibi_slopes, sliding_window, kv_cache_dtype,
85
- blocksparse_params)
86
+ blocksparse_params, logits_soft_cap)
87
88
def forward(
89
self,
vllm/model_executor/models/gemma2.py
+5
-2
@@ -90,7 +90,8 @@ class Gemma2Attention(nn.Module):
90
max_position_embeddings: int,
91
rope_theta: float,
92
cache_config: Optional[CacheConfig] = None,
93
- quant_config: Optional[QuantizationConfig] = None) -> None:
93
+ quant_config: Optional[QuantizationConfig] = None,
94
+ attn_logits_soft_cap: Optional[float] = None) -> None:
95
super().__init__()
96
self.layer_idx = layer_idx
97
self.config = config
@@ -150,7 +151,8 @@ class Gemma2Attention(nn.Module):
151
self.scaling,
152
num_kv_heads=self.num_kv_heads,
153
cache_config=cache_config,
153
- quant_config=quant_config)
154
+ quant_config=quant_config,
155
+ logits_soft_cap=attn_logits_soft_cap)
156
157
def forward(
158
self,
@@ -189,6 +191,7 @@ class Gemma2DecoderLayer(nn.Module):
191
rope_theta=config.rope_theta,
192
cache_config=cache_config,
193
quant_config=quant_config,
194
+ attn_logits_soft_cap=config.attn_logit_softcapping,
195
)
196
self.hidden_size = config.hidden_size
197
self.mlp = Gemma2MLP(