@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 805a8a75f2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Misc] Support attention logits soft-capping with flash-attn (#7022)

Woosuk Kwon committed Aug 1, 2024 at 13:14 UTC 805a8a75f2f17ee56c0882efcc34d35e1801cbee
14 files changed +71 -47
requirements-cuda.txt
+1 -1
@@ -8,4 +8,4 @@ torch == 2.4.0
8 # These must be updated alongside torch
9 torchvision == 0.19 # Required for phi3v processor. See https://github.com/pytorch/vision?tab=readme-ov-file#installation for corresponding version
10 xformers == 0.0.27.post2 # Requires PyTorch 2.4.0
11 -vllm-flash-attn == 2.6.0 # Requires PyTorch 2.4.0
11 +vllm-flash-attn == 2.6.1 # Requires PyTorch 2.4.0
tests/kernels/test_flash_attn.py
+13 -6
@@ -20,6 +20,7 @@ def ref_paged_attn(
20 block_tables: torch.Tensor,
21 scale: float,
22 sliding_window: Optional[int] = None,
23 + soft_cap: Optional[float] = None,
24 ) -> torch.Tensor:
25 num_seqs = len(query_lens)
26 block_tables = block_tables.cpu().numpy()
@@ -53,6 +54,8 @@ def ref_paged_attn(
54 (query_len + sliding_window) +
55 1).bool().logical_not()
56 mask |= sliding_window_mask
57 + if soft_cap is not None:
58 + attn = soft_cap * torch.tanh(attn / soft_cap)
59 attn.masked_fill_(mask, float("-inf"))
60 attn = torch.softmax(attn, dim=-1).to(v.dtype)
61 out = torch.einsum("hqk,khd->qhd", attn, v)
@@ -68,13 +71,15 @@ def ref_paged_attn(
71 @pytest.mark.parametrize("head_size", HEAD_SIZES)
72 @pytest.mark.parametrize("block_size", BLOCK_SIZES)
73 @pytest.mark.parametrize("dtype", DTYPES)
71 -@torch.inference_mode
74 +@pytest.mark.parametrize("soft_cap", [None, 10.0, 50.0])
75 +@torch.inference_mode()
76 def test_flash_attn_with_paged_kv(
77 kv_lens: List[int],
78 num_heads: Tuple[int, int],
79 head_size: int,
80 dtype: torch.dtype,
81 block_size: int,
82 + soft_cap: Optional[float],
83 ) -> None:
84 torch.set_default_device("cuda")
85 torch.cuda.manual_seed_all(0)
@@ -108,6 +113,7 @@ def test_flash_attn_with_paged_kv(
113 causal=True,
114 block_table=block_tables,
115 cache_seqlens=kv_lens_tensor,
116 + softcap=soft_cap if soft_cap is not None else 0,
117 ).squeeze(1)
118
119 ref_output = ref_paged_attn(
@@ -118,6 +124,7 @@ def test_flash_attn_with_paged_kv(
124 kv_lens=kv_lens,
125 block_tables=block_tables,
126 scale=scale,
127 + soft_cap=soft_cap,
128 )
129 assert torch.allclose(output, ref_output, atol=1e-2, rtol=1e-2), \
130 f"{torch.max(torch.abs(output - ref_output))}"
@@ -129,7 +136,8 @@ def test_flash_attn_with_paged_kv(
136 @pytest.mark.parametrize("block_size", BLOCK_SIZES)
137 @pytest.mark.parametrize("sliding_window", [None])
138 @pytest.mark.parametrize("dtype", DTYPES)
132 -@torch.inference_mode
139 +@pytest.mark.parametrize("soft_cap", [None, 10.0, 50.0])
140 +@torch.inference_mode()
141 def test_varlen_with_paged_kv(
142 seq_lens: List[Tuple[int, int]],
143 num_heads: Tuple[int, int],
@@ -137,6 +145,7 @@ def test_varlen_with_paged_kv(
145 sliding_window: Optional[int],
146 dtype: torch.dtype,
147 block_size: int,
148 + soft_cap: Optional[float],
149 ) -> None:
150 torch.set_default_device("cuda")
151 torch.cuda.manual_seed_all(0)
@@ -163,10 +172,6 @@ def test_varlen_with_paged_kv(
172 head_size,
173 dtype=dtype)
174 value_cache = torch.randn_like(key_cache)
166 - # Normalize the scale of the key and value caches to mitigate
167 - # numerical instability.
168 - key_cache /= head_size**0.5
169 - value_cache /= head_size**0.5
175 cu_query_lens = torch.tensor([0] + query_lens,
176 dtype=torch.int32).cumsum(dim=0,
177 dtype=torch.int32)
@@ -192,6 +197,7 @@ def test_varlen_with_paged_kv(
197 causal=True,
198 window_size=window_size,
199 block_table=block_tables,
200 + softcap=soft_cap if soft_cap is not None else 0,
201 )
202
203 ref_output = ref_paged_attn(
@@ -203,6 +209,7 @@ def test_varlen_with_paged_kv(
209 block_tables=block_tables,
210 scale=scale,
211 sliding_window=sliding_window,
212 + soft_cap=soft_cap,
213 )
214 assert torch.allclose(output, ref_output, atol=1e-2, rtol=1e-2), \
215 f"{torch.max(torch.abs(output - ref_output))}"
vllm/attention/backends/abstract.py
+1
@@ -150,6 +150,7 @@ class AttentionImpl(ABC, Generic[T]):
150 sliding_window: Optional[int] = None,
151 kv_cache_dtype: str = "auto",
152 blocksparse_params: Optional[Dict[str, Any]] = None,
153 + logits_soft_cap: Optional[float] = None,
154 ) -> None:
155 raise NotImplementedError
156
vllm/attention/backends/blocksparse_attn.py
+3
@@ -283,12 +283,15 @@ class BlocksparseFlashAttentionImpl(AttentionImpl):
283 sliding_window: Optional[int],
284 kv_cache_dtype: str,
285 blocksparse_params: Optional[Dict[str, Any]] = None,
286 + logits_soft_cap: Optional[float] = None,
287 ) -> None:
288 assert blocksparse_params is not None
289 assert alibi_slopes is None, ValueError(
290 "Alibi not support for blocksparse flash attention.")
291 assert sliding_window is None, ValueError(
292 "sliding_window is invalid for blocksparse attention.")
293 + assert logits_soft_cap is None, ValueError(
294 + "logits_soft_cap is invalid for blocksparse attention.")
295
296 if "num_heads" not in blocksparse_params:
297 blocksparse_params["num_heads"] = num_heads
vllm/attention/backends/flash_attn.py
+10 -11
@@ -288,15 +288,6 @@ class FlashAttentionMetadataBuilder(
288 device = self.runner.device
289 use_captured_graph = cuda_graph_pad_size != -1
290
291 - logits_soft_cap = getattr(self.runner.model_config.hf_config,
292 - "attn_logit_softcapping", None)
293 - if logits_soft_cap is not None:
294 - raise ValueError(
295 - "Please use Flashinfer backend for models with logits_soft_cap"
296 - " (i.e., Gemma-2). Otherwise, the output might be wrong."
297 - " Set Flashinfer backend by "
298 - "export VLLM_ATTENTION_BACKEND=FLASHINFER.")
299 -
291 max_query_len = max(query_lens)
292 max_prefill_seq_len = max(self.prefill_seq_lens, default=0)
293 max_decode_seq_len = max(self.curr_seq_lens, default=0)
@@ -405,9 +396,11 @@ class FlashAttentionImpl(AttentionImpl):
396 sliding_window: Optional[int],
397 kv_cache_dtype: str,
398 blocksparse_params: Optional[Dict[str, Any]] = None,
399 + logits_soft_cap: Optional[float] = None,
400 ) -> None:
409 - assert blocksparse_params is None, ValueError(
410 - "FlashAttention does not support block-sparse attention.")
401 + if blocksparse_params is not None:
402 + raise ValueError(
403 + "FlashAttention does not support block-sparse attention.")
404 self.num_heads = num_heads
405 self.head_size = head_size
406 self.scale = float(scale)
@@ -418,6 +411,10 @@ class FlashAttentionImpl(AttentionImpl):
411 self.sliding_window = ((sliding_window, sliding_window)
412 if sliding_window is not None else (-1, -1))
413 self.kv_cache_dtype = kv_cache_dtype
414 + if logits_soft_cap is None:
415 + # In flash-attn, setting logits_soft_cap as 0 means no soft cap.
416 + logits_soft_cap = 0
417 + self.logits_soft_cap = logits_soft_cap
418
419 assert self.num_heads % self.num_kv_heads == 0
420 self.num_queries_per_kv = self.num_heads // self.num_kv_heads
@@ -525,6 +522,7 @@ class FlashAttentionImpl(AttentionImpl):
522 causal=True,
523 window_size=self.sliding_window,
524 alibi_slopes=self.alibi_slopes,
525 + softcap=self.logits_soft_cap,
526 )
527 assert output[:num_prefill_tokens].shape == out.shape
528 output[:num_prefill_tokens] = out
@@ -544,6 +542,7 @@ class FlashAttentionImpl(AttentionImpl):
542 causal=True,
543 alibi_slopes=self.alibi_slopes,
544 block_table=prefill_meta.block_tables,
545 + softcap=self.logits_soft_cap,
546 )
547
548 if decode_meta := attn_metadata.decode_metadata:
vllm/attention/backends/flashinfer.py
+5 -9
@@ -116,8 +116,6 @@ class FlashInferMetadata(AttentionMetadata):
116 # The data type of the paged kv cache
117 data_type: torch.dtype = None
118 device: torch.device = torch.device("cuda")
119 - # Only used by gemma2 model
120 - logits_soft_cap: Optional[float] = None
119
120 def __post_init__(self):
121 # Refer to
@@ -391,9 +389,6 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
389 dtype=torch.long,
390 device=device)
391
394 - logits_soft_cap = getattr(self.runner.model_config.hf_config,
395 - "attn_logit_softcapping", None)
396 -
392 if len(self.paged_kv_indptr) > 0:
393 paged_kv_indices_tensor = torch.tensor(self.paged_kv_indices,
394 device="cpu",
@@ -430,8 +425,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
425 query_start_loc=query_start_loc,
426 device=device,
427 data_type=kv_cache_dtype,
433 - use_cuda_graph=use_captured_graph,
434 - logits_soft_cap=logits_soft_cap)
428 + use_cuda_graph=use_captured_graph)
429
430
431 class FlashInferImpl(AttentionImpl):
@@ -446,6 +440,7 @@ class FlashInferImpl(AttentionImpl):
440 sliding_window: Optional[int],
441 kv_cache_dtype: str,
442 blocksparse_params: Optional[Dict[str, Any]] = None,
443 + logits_soft_cap: Optional[float] = None,
444 ) -> None:
445 self.num_heads = num_heads
446 self.head_size = head_size
@@ -458,6 +453,7 @@ class FlashInferImpl(AttentionImpl):
453 raise ValueError("Sliding window is not supported in FlashInfer.")
454 self.sliding_window = (-1, -1)
455 self.kv_cache_dtype = kv_cache_dtype
456 + self.logits_soft_cap = logits_soft_cap
457
458 assert self.num_heads % self.num_kv_heads == 0
459 self.num_queries_per_kv = self.num_heads // self.num_kv_heads
@@ -532,7 +528,7 @@ class FlashInferImpl(AttentionImpl):
528 output = prefill_meta.prefill_wrapper.forward(
529 query,
530 kv_cache,
535 - logits_soft_cap=attn_metadata.logits_soft_cap,
531 + logits_soft_cap=self.logits_soft_cap,
532 causal=True)
533 else:
534 assert attn_metadata.decode_metadata is not None
@@ -541,5 +537,5 @@ class FlashInferImpl(AttentionImpl):
537 query,
538 kv_cache,
539 sm_scale=self.scale,
544 - logits_soft_cap=attn_metadata.logits_soft_cap)
540 + logits_soft_cap=self.logits_soft_cap)
541 return output.view(num_tokens, hidden_size)
vllm/attention/backends/ipex_attn.py
+6 -2
@@ -105,9 +105,13 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
105 sliding_window: Optional[int],
106 kv_cache_dtype: str,
107 blocksparse_params: Optional[Dict[str, Any]] = None,
108 + logits_soft_cap: Optional[float] = None,
109 ) -> None:
109 - assert blocksparse_params is None, ValueError(
110 - "Torch SPDA does not support block-sparse attention.")
110 + if blocksparse_params is not None:
111 + raise ValueError(
112 + "IPEX backend does not support block-sparse attention.")
113 + if logits_soft_cap is not None:
114 + raise ValueError("IPEX backend does not support logits_soft_cap.")
115 self.num_heads = num_heads
116 self.head_size = head_size
117 self.scale = float(scale)
vllm/attention/backends/pallas.py
+4
@@ -91,6 +91,7 @@ class PallasAttentionBackendImpl(AttentionImpl):
91 sliding_window: Optional[int],
92 kv_cache_dtype: str,
93 blocksparse_params: Optional[Dict[str, Any]] = None,
94 + logits_soft_cap: Optional[float] = None,
95 ) -> None:
96 self.num_heads = num_heads
97 self.head_size = head_size
@@ -109,6 +110,9 @@ class PallasAttentionBackendImpl(AttentionImpl):
110 raise NotImplementedError("FP8 KV cache dtype is not supported.")
111 if blocksparse_params is not None:
112 raise NotImplementedError("Blocksparse is not supported.")
113 + if logits_soft_cap is not None:
114 + raise NotImplementedError(
115 + "Attention logits soft-capping is not supported.")
116
117 if torch_xla.tpu.version() < 4:
118 raise NotImplementedError("TPU version must be 4 or higher.")
vllm/attention/backends/rocm_flash_attn.py
+8 -2
@@ -244,9 +244,15 @@ class ROCmFlashAttentionImpl(AttentionImpl):
244 sliding_window: Optional[int],
245 kv_cache_dtype: str,
246 blocksparse_params: Optional[Dict[str, Any]] = None,
247 + logits_soft_cap: Optional[float] = None,
248 ) -> None:
248 - assert blocksparse_params is None, ValueError(
249 - "ROCFlashAttention does not support blocksparse attention.")
249 + if blocksparse_params is not None:
250 + raise ValueError(
251 + "ROCmFlashAttention does not support blocksparse attention.")
252 + if logits_soft_cap is not None:
253 + raise ValueError(
254 + "ROCmFlashAttention does not support attention logits soft "
255 + "capping.")
256 self.num_heads = num_heads
257 self.head_size = head_size
258 self.scale = float(scale)
vllm/attention/backends/torch_sdpa.py
+6 -2
@@ -109,9 +109,13 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
109 sliding_window: Optional[int],
110 kv_cache_dtype: str,
111 blocksparse_params: Optional[Dict[str, Any]] = None,
112 + logits_soft_cap: Optional[float] = None,
113 ) -> None:
113 - assert blocksparse_params is None, ValueError(
114 - "Torch SPDA does not support block-sparse attention.")
114 + if blocksparse_params is not None:
115 + raise ValueError(
116 + "Torch SPDA does not support block-sparse attention.")
117 + if logits_soft_cap is not None:
118 + raise ValueError("Torch SPDA does not support logits soft cap.")
119 self.num_heads = num_heads
120 self.head_size = head_size
121 self.scale = float(scale)
vllm/attention/backends/utils.py
-9
@@ -165,15 +165,6 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
165 device = self.runner.device
166 use_captured_graph = cuda_graph_pad_size != -1
167
168 - logits_soft_cap = getattr(self.runner.model_config.hf_config,
169 - "attn_logit_softcapping", None)
170 - if logits_soft_cap is not None:
171 - raise ValueError(
172 - "Please use Flashinfer backend for models with logits_soft_cap "
173 - "(i.e., Gemma-2). Otherwise, the output might be wrong. "
174 - "Set Flashinfer backend by "
175 - "export VLLM_ATTENTION_BACKEND=FLASHINFER.")
176 -
168 max_query_len = max(query_lens)
169 max_prefill_seq_len = max(self.prefill_seq_lens, default=0)
170 max_decode_seq_len = max(self.curr_seq_lens, default=0)
vllm/attention/backends/xformers.py
+7 -2
@@ -408,9 +408,14 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
408 sliding_window: Optional[int],
409 kv_cache_dtype: str,
410 blocksparse_params: Optional[Dict[str, Any]] = None,
411 + logits_soft_cap: Optional[float] = None,
412 ) -> None:
412 - assert blocksparse_params is None, ValueError(
413 - "XFormer does not support block-sparse attention.")
413 + if blocksparse_params is not None:
414 + raise ValueError(
415 + "XFormers does not support block-sparse attention.")
416 + if logits_soft_cap is not None:
417 + raise ValueError(
418 + "XFormers does not support attention logits soft capping.")
419 self.num_heads = num_heads
420 self.head_size = head_size
421 self.scale = float(scale)
vllm/attention/layer.py
+2 -1
@@ -34,6 +34,7 @@ class Attention(nn.Module):
34 cache_config: Optional[CacheConfig] = None,
35 quant_config: Optional[QuantizationConfig] = None,
36 blocksparse_params: Optional[Dict[str, Any]] = None,
37 + logits_soft_cap: Optional[float] = None,
38 prefix: str = "",
39 ) -> None:
40 super().__init__()
@@ -82,7 +83,7 @@ class Attention(nn.Module):
83 impl_cls = attn_backend.get_impl_cls()
84 self.impl = impl_cls(num_heads, head_size, scale, num_kv_heads,
85 alibi_slopes, sliding_window, kv_cache_dtype,
85 - blocksparse_params)
86 + blocksparse_params, logits_soft_cap)
87
88 def forward(
89 self,
vllm/model_executor/models/gemma2.py
+5 -2
@@ -90,7 +90,8 @@ class Gemma2Attention(nn.Module):
90 max_position_embeddings: int,
91 rope_theta: float,
92 cache_config: Optional[CacheConfig] = None,
93 - quant_config: Optional[QuantizationConfig] = None) -> None:
93 + quant_config: Optional[QuantizationConfig] = None,
94 + attn_logits_soft_cap: Optional[float] = None) -> None:
95 super().__init__()
96 self.layer_idx = layer_idx
97 self.config = config
@@ -150,7 +151,8 @@ class Gemma2Attention(nn.Module):
151 self.scaling,
152 num_kv_heads=self.num_kv_heads,
153 cache_config=cache_config,
153 - quant_config=quant_config)
154 + quant_config=quant_config,
155 + logits_soft_cap=attn_logits_soft_cap)
156
157 def forward(
158 self,
@@ -189,6 +191,7 @@ class Gemma2DecoderLayer(nn.Module):
191 rope_theta=config.rope_theta,
192 cache_config=cache_config,
193 quant_config=quant_config,
194 + attn_logits_soft_cap=config.attn_logit_softcapping,
195 )
196 self.hidden_size = config.hidden_size
197 self.mlp = Gemma2MLP(