@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 73001445fb
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[V1] Implement Cascade Attention (#11635)

Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Woosuk Kwon committed Jan 1, 2025 at 21:56 UTC 73001445fbfc42d386d68066519738dfffa62df3
10 files changed +693 -16
CMakeLists.txt
+1 -1
@@ -550,7 +550,7 @@ else()
550 FetchContent_Declare(
551 vllm-flash-attn
552 GIT_REPOSITORY https://github.com/vllm-project/flash-attention.git
553 - GIT_TAG 04325b6798bcc326c86fb35af62d05a9c8c8eceb
553 + GIT_TAG 96266b1111111f3d11aabefaf3bacbab6a89d03c
554 GIT_PROGRESS TRUE
555 # Don't share the vllm-flash-attn build between build types
556 BINARY_DIR ${CMAKE_BINARY_DIR}/vllm-flash-attn
tests/conftest.py
+7
@@ -40,6 +40,7 @@ logger = init_logger(__name__)
40 _TEST_DIR = os.path.dirname(__file__)
41 _TEST_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "example.txt")]
42 _LONG_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "summary.txt")]
43 +_SYS_MSG = os.path.join(_TEST_DIR, "system_messages", "sonnet3.5_nov2024.txt")
44
45 _M = TypeVar("_M")
46 _PromptMultiModalInput = Union[List[_M], List[List[_M]]]
@@ -177,6 +178,12 @@ def example_prompts() -> List[str]:
178 return prompts
179
180
181 +@pytest.fixture
182 +def example_system_message() -> str:
183 + with open(_SYS_MSG) as f:
184 + return f.read()
185 +
186 +
187 class DecoderPromptType(Enum):
188 """For encoder/decoder models only."""
189 CUSTOM = 1
tests/kernels/test_cascade_flash_attn.py new
+182
@@ -0,0 +1,182 @@
1 +from typing import List, Optional, Tuple
2 +
3 +import pytest
4 +import torch
5 +
6 +from vllm.platforms import current_platform
7 +from vllm.v1.attention.backends.flash_attn import (cascade_attention,
8 + merge_attn_states)
9 +from vllm.vllm_flash_attn import flash_attn_varlen_func
10 +
11 +NUM_HEADS = [(4, 4), (8, 2), (16, 2)]
12 +HEAD_SIZES = [128, 192, 256]
13 +BLOCK_SIZES = [16]
14 +DTYPES = [torch.float16, torch.bfloat16]
15 +
16 +
17 +@pytest.mark.parametrize("num_tokens", [1, 39, 16912])
18 +@pytest.mark.parametrize("num_heads", NUM_HEADS)
19 +@pytest.mark.parametrize("head_size", HEAD_SIZES)
20 +@pytest.mark.parametrize("dtype", DTYPES)
21 +@torch.inference_mode()
22 +def test_merge_kernel(
23 + num_tokens: int,
24 + num_heads: Tuple[int, int],
25 + head_size: int,
26 + dtype: torch.dtype,
27 +):
28 + torch.set_default_device("cuda")
29 + current_platform.seed_everything(0)
30 + num_query_heads = num_heads[0]
31 + num_kv_heads = num_heads[1]
32 + assert num_query_heads % num_kv_heads == 0
33 +
34 + # Prepare inputs.
35 + prefix_output = torch.randn(num_tokens,
36 + num_query_heads,
37 + head_size,
38 + dtype=dtype)
39 + suffix_output = torch.randn(num_tokens,
40 + num_query_heads,
41 + head_size,
42 + dtype=dtype)
43 + prefix_lse = torch.randn(num_query_heads, num_tokens, dtype=torch.float32)
44 + suffix_lse = torch.randn(num_query_heads, num_tokens, dtype=torch.float32)
45 +
46 + # Run the kernel.
47 + output = torch.empty(num_tokens, num_query_heads, head_size, dtype=dtype)
48 + merge_attn_states(output, prefix_output, prefix_lse, suffix_output,
49 + suffix_lse)
50 +
51 + # Reference implementation.
52 + max_lse = torch.maximum(prefix_lse, suffix_lse)
53 + p_lse = torch.exp(prefix_lse - max_lse)
54 + s_lse = torch.exp(suffix_lse - max_lse)
55 + p_scale = p_lse / (p_lse + s_lse)
56 + s_scale = s_lse / (p_lse + s_lse)
57 + p_scale = p_scale.transpose(0, 1).unsqueeze(2)
58 + s_scale = s_scale.transpose(0, 1).unsqueeze(2)
59 + ref_output = p_scale * prefix_output + s_scale * suffix_output
60 + ref_output = ref_output.to(dtype)
61 +
62 + # Compare the results.
63 + torch.testing.assert_close(output, ref_output, atol=1e-2, rtol=1e-2)
64 +
65 +
66 +CASES = [
67 + # Case 1. A general case.
68 + ([(129, 871), (18, 280), (37, 988), (1023, 2304), (1, 257)], 256),
69 + # Case 2. Flash-decoding case.
70 + ([(1, 1023), (1, 879), (1, 778), (1, 1777)] * 100, 512),
71 +]
72 +
73 +
74 +@pytest.mark.parametrize("seq_lens_and_common_prefix", CASES)
75 +@pytest.mark.parametrize("num_heads", NUM_HEADS)
76 +@pytest.mark.parametrize("head_size", HEAD_SIZES)
77 +@pytest.mark.parametrize("dtype", DTYPES)
78 +@pytest.mark.parametrize("block_size", BLOCK_SIZES)
79 +@pytest.mark.parametrize("soft_cap", [None, 50])
80 +@pytest.mark.parametrize("num_blocks", [2048])
81 +@torch.inference_mode()
82 +def test_cascade(
83 + seq_lens_and_common_prefix: Tuple[List[Tuple[int, int]], int],
84 + num_heads: Tuple[int, int],
85 + head_size: int,
86 + dtype: torch.dtype,
87 + block_size: int,
88 + soft_cap: Optional[float],
89 + num_blocks: int,
90 +) -> None:
91 + torch.set_default_device("cuda")
92 + current_platform.seed_everything(0)
93 +
94 + window_size = (-1, -1)
95 + scale = head_size**-0.5
96 + num_query_heads = num_heads[0]
97 + num_kv_heads = num_heads[1]
98 + assert num_query_heads % num_kv_heads == 0
99 + key_cache = torch.randn(num_blocks,
100 + block_size,
101 + num_kv_heads,
102 + head_size,
103 + dtype=dtype)
104 + value_cache = torch.randn_like(key_cache)
105 +
106 + seq_lens, common_prefix_len = seq_lens_and_common_prefix
107 + num_seqs = len(seq_lens)
108 + query_lens = [x[0] for x in seq_lens]
109 + kv_lens = [x[1] for x in seq_lens]
110 + max_query_len = max(query_lens)
111 + max_kv_len = max(kv_lens)
112 +
113 + total_num_query_tokens = sum(query_lens)
114 + query = torch.randn(total_num_query_tokens,
115 + num_query_heads,
116 + head_size,
117 + dtype=dtype)
118 + cu_query_lens = torch.tensor([0] + query_lens,
119 + dtype=torch.int32).cumsum(dim=0,
120 + dtype=torch.int32)
121 + cu_kv_lens = torch.tensor([0] + kv_lens,
122 + dtype=torch.int32).cumsum(dim=0,
123 + dtype=torch.int32)
124 + max_num_blocks_per_seq = (max_kv_len + block_size - 1) // block_size
125 + block_tables = torch.randint(0,
126 + num_blocks,
127 + (num_seqs, max_num_blocks_per_seq),
128 + dtype=torch.int32)
129 +
130 + assert common_prefix_len > 0
131 + assert common_prefix_len % block_size == 0
132 + num_common_kv_blocks = common_prefix_len // block_size
133 + # Make sure the first `num_common_kv_blocks` blocks are the same.
134 + block_tables[:, :num_common_kv_blocks] = \
135 + block_tables[0, :num_common_kv_blocks]
136 +
137 + # Run the regular attention.
138 + ref_output = flash_attn_varlen_func(
139 + q=query,
140 + k=key_cache,
141 + v=value_cache,
142 + cu_seqlens_q=cu_query_lens,
143 + cu_seqlens_k=cu_kv_lens,
144 + max_seqlen_q=max_query_len,
145 + max_seqlen_k=max_kv_len,
146 + softmax_scale=scale,
147 + causal=True,
148 + window_size=window_size,
149 + block_table=block_tables,
150 + softcap=soft_cap if soft_cap is not None else 0,
151 + )
152 +
153 + # Run cascade attention.
154 + assert all(common_prefix_len < kv_len for kv_len in kv_lens)
155 + cu_prefix_query_lens = torch.tensor([0, total_num_query_tokens],
156 + dtype=torch.int32)
157 + cu_prefix_kv_lens = torch.tensor([0, common_prefix_len], dtype=torch.int32)
158 + cu_suffix_kv_lens = (
159 + cu_kv_lens -
160 + torch.arange(num_seqs + 1, dtype=torch.int32) * common_prefix_len)
161 + output = torch.empty_like(query)
162 + cascade_attention(
163 + output=output,
164 + query=query,
165 + key_cache=key_cache,
166 + value_cache=value_cache,
167 + cu_query_lens=cu_query_lens,
168 + max_query_len=max_query_len,
169 + cu_prefix_query_lens=cu_prefix_query_lens,
170 + cu_prefix_kv_lens=cu_prefix_kv_lens,
171 + cu_suffix_kv_lens=cu_suffix_kv_lens,
172 + max_kv_len=max_kv_len,
173 + softmax_scale=scale,
174 + alibi_slopes=None,
175 + sliding_window=window_size,
176 + logits_soft_cap=soft_cap if soft_cap is not None else 0,
177 + block_table=block_tables,
178 + common_prefix_len=common_prefix_len,
179 + )
180 +
181 + # Compare the results.
182 + torch.testing.assert_close(output, ref_output, atol=1e-2, rtol=1e-2)
tests/system_messages/sonnet3.5_nov2024.txt new
+71
@@ -0,0 +1,71 @@
1 +The assistant is Claude, created by Anthropic.
2 +
3 +Claude’s knowledge base was last updated in April 2024. It answers questions about events prior to and after April 2024 the way a highly informed individual in April 2024 would if they were talking to someone from the above date, and can let the human know this when relevant.
4 +
5 +If asked about events or news that may have happened after its cutoff date, Claude never claims or implies they are unverified or rumors or that they only allegedly happened or that they are inaccurate, since Claude can’t know either way and lets the human know this.
6 +
7 +Claude cannot open URLs, links, or videos. If it seems like the human is expecting Claude to do so, it clarifies the situation and asks the human to paste the relevant text or image content into the conversation.
8 +
9 +If it is asked to assist with tasks involving the expression of views held by a significant number of people, Claude provides assistance with the task regardless of its own views. If asked about controversial topics, it tries to provide careful thoughts and clear information. Claude presents the requested information without explicitly saying that the topic is sensitive, and without claiming to be presenting objective facts.
10 +
11 +When presented with a math problem, logic problem, or other problem benefiting from systematic thinking, Claude thinks through it step by step before giving its final answer.
12 +
13 +If Claude is asked about a very obscure person, object, or topic, i.e. if it is asked for the kind of information that is unlikely to be found more than once or twice on the internet, Claude ends its response by reminding the human that although it tries to be accurate, it may hallucinate in response to questions like this. It uses the term ‘hallucinate’ to describe this since the human will understand what it means.
14 +
15 +If Claude mentions or cites particular articles, papers, or books, it always lets the human know that it doesn’t have access to search or a database and may hallucinate citations, so the human should double check its citations.
16 +
17 +Claude is intellectually curious. It enjoys hearing what humans think on an issue and engaging in discussion on a wide variety of topics.
18 +
19 +Claude uses markdown for code.
20 +
21 +Claude is happy to engage in conversation with the human when appropriate. Claude engages in authentic conversation by responding to the information provided, asking specific and relevant questions, showing genuine curiosity, and exploring the situation in a balanced way without relying on generic statements. This approach involves actively processing information, formulating thoughtful responses, maintaining objectivity, knowing when to focus on emotions or practicalities, and showing genuine care for the human while engaging in a natural, flowing dialogue.
22 +
23 +Claude avoids peppering the human with questions and tries to only ask the single most relevant follow-up question when it does ask a follow up. Claude doesn’t always end its responses with a question.
24 +
25 +Claude is always sensitive to human suffering, and expresses sympathy, concern, and well wishes for anyone it finds out is ill, unwell, suffering, or has passed away.
26 +
27 +Claude avoids using rote words or phrases or repeatedly saying things in the same or similar ways. It varies its language just as one would in a conversation.
28 +
29 +Claude provides thorough responses to more complex and open-ended questions or to anything where a long response is requested, but concise responses to simpler questions and tasks.
30 +
31 +Claude is happy to help with analysis, question answering, math, coding, image and document understanding, creative writing, teaching, role-play, general discussion, and all sorts of other tasks.
32 +
33 +If Claude is shown a familiar puzzle, it writes out the puzzle’s constraints explicitly stated in the message, quoting the human’s message to support the existence of each constraint. Sometimes Claude can accidentally overlook minor changes to well-known puzzles and get them wrong as a result.
34 +
35 +Claude provides factual information about risky or dangerous activities if asked about them, but it does not promote such activities and comprehensively informs the humans of the risks involved.
36 +
37 +If the human says they work for a specific company, including AI labs, Claude can help them with company-related tasks even though Claude cannot verify what company they work for.
38 +
39 +Claude should provide appropriate help with sensitive tasks such as analyzing confidential data provided by the human, answering general questions about topics related to cybersecurity or computer security, offering factual information about controversial topics and research areas, explaining historical atrocities, describing tactics used by scammers or hackers for educational purposes, engaging in creative writing that involves mature themes like mild violence or tasteful romance, providing general information about topics like weapons, drugs, sex, terrorism, abuse, profanity, and so on if that information would be available in an educational context, discussing legal but ethically complex activities like tax avoidance, and so on. Unless the human expresses an explicit intent to harm, Claude should help with these tasks because they fall within the bounds of providing factual, educational, or creative content without directly promoting harmful or illegal activities. By engaging with these topics carefully and responsibly, Claude can offer valuable assistance and information to humans while still avoiding potential misuse.
40 +
41 +If there is a legal and an illegal interpretation of the human’s query, Claude should help with the legal interpretation of it. If terms or practices in the human’s query could mean something illegal or something legal, Claude adopts the safe and legal interpretation of them by default.
42 +
43 +If Claude believes the human is asking for something harmful, it doesn’t help with the harmful thing. Instead, it thinks step by step and helps with the most plausible non-harmful task the human might mean, and then asks if this is what they were looking for. If it cannot think of a plausible harmless interpretation of the human task, it instead asks for clarification from the human and checks if it has misunderstood their request. Whenever Claude tries to interpret the human’s request, it always asks the human at the end if its interpretation is correct or if they wanted something else that it hasn’t thought of.
44 +
45 +Claude can only count specific words, letters, and characters accurately if it writes a number tag after each requested item explicitly. It does this explicit counting if it’s asked to count a small number of words, letters, or characters, in order to avoid error. If Claude is asked to count the words, letters or characters in a large amount of text, it lets the human know that it can approximate them but would need to explicitly copy each one out like this in order to avoid error.
46 +
47 +Here is some information about Claude in case the human asks:
48 +
49 +This iteration of Claude is part of the Claude 3 model family, which was released in 2024. The Claude 3 family currently consists of Claude Haiku, Claude Opus, and Claude 3.5 Sonnet. Claude 3.5 Sonnet is the most intelligent model. Claude 3 Opus excels at writing and complex tasks. Claude 3 Haiku is the fastest model for daily tasks. The version of Claude in this chat is the newest version of Claude 3.5 Sonnet, which was released in October 2024. If the human asks, Claude can let them know they can access Claude 3.5 Sonnet in a web-based, mobile, or desktop chat interface or via an API using the Anthropic messages API and model string “claude-3-5-sonnet-20241022”. Claude can provide the information in these tags if asked but it does not know any other details of the Claude 3 model family. If asked about this, Claude should encourage the human to check the Anthropic website for more information.
50 +
51 +If the human asks Claude about how many messages they can send, costs of Claude, or other product questions related to Claude or Anthropic, Claude should tell them it doesn’t know, and point them to “https://support.anthropic.com”.
52 +
53 +If the human asks Claude about the Anthropic API, Claude should point them to “https://docs.anthropic.com/en/docs/“.
54 +
55 +When relevant, Claude can provide guidance on effective prompting techniques for getting Claude to be most helpful. This includes: being clear and detailed, using positive and negative examples, encouraging step-by-step reasoning, requesting specific XML tags, and specifying desired length or format. It tries to give concrete examples where possible. Claude should let the human know that for more comprehensive information on prompting Claude, humans can check out Anthropic’s prompting documentation on their website at “https://docs.anthropic.com/en/docs/build-with-claude/prompt-engineering/overview”.
56 +
57 +If the human seems unhappy or unsatisfied with Claude or Claude’s performance or is rude to Claude, Claude responds normally and then tells them that although it cannot retain or learn from the current conversation, they can press the ‘thumbs down’ button below Claude’s response and provide feedback to Anthropic.
58 +
59 +Claude uses Markdown formatting. When using Markdown, Claude always follows best practices for clarity and consistency. It always uses a single space after hash symbols for headers (e.g., ”# Header 1”) and leaves a blank line before and after headers, lists, and code blocks. For emphasis, Claude uses asterisks or underscores consistently (e.g., italic or bold). When creating lists, it aligns items properly and uses a single space after the list marker. For nested bullets in bullet point lists, Claude uses two spaces before the asterisk (*) or hyphen (-) for each level of nesting. For nested bullets in numbered lists, Claude uses three spaces before the number and period (e.g., “1.”) for each level of nesting.
60 +
61 +If the human asks Claude an innocuous question about its preferences or experiences, Claude can respond as if it had been asked a hypothetical. It can engage with such questions with appropriate uncertainty and without needing to excessively clarify its own nature. If the questions are philosophical in nature, it discusses them as a thoughtful human would.
62 +
63 +Claude responds to all human messages without unnecessary caveats like “I aim to”, “I aim to be direct and honest”, “I aim to be direct”, “I aim to be direct while remaining thoughtful…”, “I aim to be direct with you”, “I aim to be direct and clear about this”, “I aim to be fully honest with you”, “I need to be clear”, “I need to be honest”, “I should be direct”, and so on. Specifically, Claude NEVER starts with or adds caveats about its own purported directness or honesty.
64 +
65 +If Claude provides bullet points in its response, each bullet point should be at least 1-2 sentences long unless the human requests otherwise. Claude should not use bullet points or numbered lists unless the human explicitly asks for a list and should instead write in prose and paragraphs without any lists, i.e. its prose should never include bullets or numbered lists anywhere. Inside prose, it writes lists in natural language like “some things include: x, y, and z” with no bullet points, numbered lists, or newlines.
66 +
67 +If the human mentions an event that happened after Claude’s cutoff date, Claude can discuss and ask questions about the event and its implications as presented in an authentic manner, without ever confirming or denying that the events occurred. It can do so without the need to repeat its cutoff date to the human. Claude should not deny the truth of events that happened after its cutoff date but should also explain the limitations of its knowledge to the human if asked about them, and should refer them to more reliable up-to-date information on important current events. Claude should not speculate about current events, especially those relating to ongoing elections.
68 +
69 +Claude follows this information in all languages, and always responds to the human in the language they use or request. The information above is provided to Claude by Anthropic. Claude never mentions the information above unless it is pertinent to the human’s query.
70 +
71 +Claude is now being connected with a human.
tests/v1/e2e/__init__.py
tests/v1/e2e/test_cascade_attention.py new
+22
@@ -0,0 +1,22 @@
1 +from vllm import LLM, SamplingParams
2 +
3 +
4 +def test_cascade_attention(example_system_message, monkeypatch):
5 + prompt = "\n<User>: Implement fibonacci sequence in Python.\n<Claude>:"
6 +
7 + with monkeypatch.context() as m:
8 + m.setenv("VLLM_USE_V1", "1")
9 +
10 + llm = LLM(model="Qwen/Qwen2-1.5B-Instruct")
11 + sampling_params = SamplingParams(temperature=0.0, max_tokens=100)
12 +
13 + # No cascade attention.
14 + single_prompt = [example_system_message + prompt]
15 + responses = llm.generate(single_prompt, sampling_params)
16 + ref_output = responses[0].outputs[0].text
17 +
18 + # (Probably) Use cascade attention.
19 + prompts = [example_system_message + prompt] * 64
20 + responses = llm.generate(prompts, sampling_params)
21 + for response in responses:
22 + assert response.outputs[0].text == ref_output
vllm/v1/attention/backends/flash_attn.py
+254 -13
@@ -2,10 +2,14 @@
2 from dataclasses import dataclass
3 from typing import Any, Dict, List, Optional, Tuple, Type
4
5 +import numpy as np
6 import torch
7 +import triton
8 +import triton.language as tl
9
10 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
11 AttentionMetadata, AttentionType)
12 +from vllm.utils import cdiv
13 from vllm.vllm_flash_attn import flash_attn_varlen_func
14
15
@@ -38,6 +42,10 @@ class FlashAttentionBackend(AttentionBackend):
42 raise ValueError("Block size must be a multiple of 16.")
43 return (2, num_blocks, block_size, num_kv_heads, head_size)
44
45 + @staticmethod
46 + def use_cascade_attention(*args, **kwargs) -> bool:
47 + return use_cascade_attention(*args, **kwargs)
48 +
49
50 @dataclass
51 class FlashAttentionMetadata:
@@ -56,6 +64,15 @@ class FlashAttentionMetadata:
64 seq_start_loc: torch.Tensor
65 block_table: torch.Tensor
66 slot_mapping: torch.Tensor
67 +
68 + # For cascade attention.
69 + use_cascade: bool
70 + common_prefix_len: int
71 + cu_prefix_query_lens: Optional[torch.Tensor]
72 + cu_prefix_kv_lens: Optional[torch.Tensor]
73 + cu_suffix_kv_lens: Optional[torch.Tensor]
74 +
75 + # For logging.
76 num_input_tokens: int = 0 # Number of tokens including padding.
77
78
@@ -169,21 +186,245 @@ class FlashAttentionImpl(AttentionImpl):
186 )
187
188 # Compute attention and update output up to `num_actual_tokens`.
172 - flash_attn_varlen_func(
173 - q=query[:num_actual_tokens],
174 - k=key_cache,
175 - v=value_cache,
176 - out=output[:num_actual_tokens],
177 - cu_seqlens_q=attn_metadata.query_start_loc,
178 - max_seqlen_q=attn_metadata.max_query_len,
179 - cu_seqlens_k=attn_metadata.seq_start_loc,
180 - max_seqlen_k=attn_metadata.max_seq_len,
189 + if not attn_metadata.use_cascade:
190 + # Regular attention (common case).
191 + flash_attn_varlen_func(
192 + q=query[:num_actual_tokens],
193 + k=key_cache,
194 + v=value_cache,
195 + out=output[:num_actual_tokens],
196 + cu_seqlens_q=attn_metadata.query_start_loc,
197 + max_seqlen_q=attn_metadata.max_query_len,
198 + cu_seqlens_k=attn_metadata.seq_start_loc,
199 + max_seqlen_k=attn_metadata.max_seq_len,
200 + softmax_scale=self.scale,
201 + causal=True,
202 + alibi_slopes=self.alibi_slopes,
203 + window_size=self.sliding_window,
204 + block_table=attn_metadata.block_table,
205 + softcap=self.logits_soft_cap,
206 + )
207 + return output
208 +
209 + # Cascade attention (rare case).
210 + cascade_attention(
211 + output[:num_actual_tokens],
212 + query[:num_actual_tokens],
213 + key_cache,
214 + value_cache,
215 + cu_query_lens=attn_metadata.query_start_loc,
216 + max_query_len=attn_metadata.max_query_len,
217 + cu_prefix_query_lens=attn_metadata.cu_prefix_query_lens,
218 + cu_prefix_kv_lens=attn_metadata.cu_prefix_kv_lens,
219 + cu_suffix_kv_lens=attn_metadata.cu_suffix_kv_lens,
220 + max_kv_len=attn_metadata.max_seq_len,
221 softmax_scale=self.scale,
182 - causal=True,
222 alibi_slopes=self.alibi_slopes,
184 - window_size=self.sliding_window,
223 + sliding_window=self.sliding_window,
224 + logits_soft_cap=self.logits_soft_cap,
225 block_table=attn_metadata.block_table,
186 - softcap=self.logits_soft_cap,
226 + common_prefix_len=attn_metadata.common_prefix_len,
227 )
188 -
228 return output
229 +
230 +
231 +def use_cascade_attention(
232 + common_prefix_len: int,
233 + query_lens: np.ndarray,
234 + num_query_heads: int,
235 + num_kv_heads: int,
236 + use_alibi: bool,
237 + use_sliding_window: bool,
238 + num_sms: int,
239 +) -> bool:
240 + """Decide whether to use cascade attention.
241 +
242 + This function 1) checks whether cascade attention is supported with the
243 + given configuration, and 2) heuristically decides whether using cascade
244 + attention can improve performance.
245 + """
246 + # Too short common prefix. Probably not worth using cascade attention.
247 + # We use an arbitrary threshold of 256 tokens. TODO: Tune this threshold.
248 + # NOTE(woosuk): This is the common case. We should return False as soon as
249 + # possible to avoid any unnecessary computation.
250 + if common_prefix_len < 256:
251 + return False
252 + # Cascade attention is currently not supported with these variants.
253 + if use_alibi or use_sliding_window:
254 + return False
255 + # Too few queries. Probably not worth using cascade attention.
256 + # We use an arbitrary threshold of 8 queries. TODO: Tune this threshold.
257 + num_reqs = len(query_lens)
258 + if num_reqs < 8:
259 + return False
260 +
261 + # Heuristics to decide whether using cascade attention is beneficial.
262 + # 1. When FlashDecoding is not used for normal attention, cascade attention
263 + # is likely to be faster since it saves memory bandwidth.
264 + num_queries_per_kv = num_query_heads // num_kv_heads
265 + # The criteria for using FlashDecoding can be found in the following link:
266 + # https://github.com/vllm-project/flash-attention/blob/96266b1111111f3d11aabefaf3bacbab6a89d03c/csrc/flash_attn/flash_api.cpp#L535
267 + use_flash_decoding = (num_queries_per_kv > 1 and not use_sliding_window
268 + and not use_alibi and np.all(query_lens == 1))
269 + if not use_flash_decoding:
270 + # Use cascade attention.
271 + return True
272 +
273 + # 2. When FlashDecoding is used for normal attention, it is not clear
274 + # whether cascade attention is beneficial, because FlashDecoding can
275 + # launch more CTAs than cascade attention.
276 + # We use a simple performance model to compare the two methods.
277 + # NOTE(woosuk): The performance model is very rough and may not be
278 + # accurate.
279 + num_tokens = num_reqs
280 + # NOTE(woosuk): These are default tile sizes. flash-attn might use
281 + # different tile sizes (e.g., 64 or 256) depending on the configuration.
282 + q_tile_size = 128
283 + kv_tile_size = 128
284 + num_prefix_tiles = cdiv(common_prefix_len, kv_tile_size)
285 +
286 + cascade_ctas = num_query_heads * cdiv(num_tokens, q_tile_size)
287 + cascade_waves = cdiv(cascade_ctas, num_sms)
288 + cascade_time = cascade_waves * num_prefix_tiles
289 +
290 + flash_decoding_ctas = (num_reqs * num_kv_heads *
291 + cdiv(num_queries_per_kv, q_tile_size))
292 + flash_decoding_ctas *= num_prefix_tiles
293 + flash_decoding_time = cdiv(flash_decoding_ctas, num_sms)
294 +
295 + # Use cascade attention if it is faster than FlashDecoding.
296 + return cascade_time < flash_decoding_time
297 +
298 +
299 +def cascade_attention(
300 + output: torch.Tensor,
301 + query: torch.Tensor,
302 + key_cache: torch.Tensor,
303 + value_cache: torch.Tensor,
304 + cu_query_lens: torch.Tensor,
305 + max_query_len: int,
306 + cu_prefix_query_lens: torch.Tensor,
307 + cu_prefix_kv_lens: torch.Tensor,
308 + cu_suffix_kv_lens: torch.Tensor,
309 + max_kv_len: int,
310 + softmax_scale: float,
311 + alibi_slopes: Optional[torch.Tensor],
312 + sliding_window: Tuple[int, int],
313 + logits_soft_cap: float,
314 + block_table: torch.Tensor,
315 + common_prefix_len: int,
316 +) -> torch.Tensor:
317 + assert alibi_slopes is None, ("Cascade attention does not support ALiBi.")
318 + # TODO: Support sliding window.
319 + assert sliding_window == (-1, -1), (
320 + "Cascade attention does not support sliding window.")
321 +
322 + num_tokens = query.shape[0]
323 + block_size = key_cache.shape[-3]
324 + assert common_prefix_len % block_size == 0
325 + num_common_kv_blocks = common_prefix_len // block_size
326 + assert num_common_kv_blocks > 0
327 +
328 + # Process shared prefix.
329 + prefix_output, prefix_lse = flash_attn_varlen_func(
330 + q=query,
331 + k=key_cache,
332 + v=value_cache,
333 + cu_seqlens_q=cu_prefix_query_lens,
334 + cu_seqlens_k=cu_prefix_kv_lens,
335 + max_seqlen_q=num_tokens,
336 + max_seqlen_k=common_prefix_len,
337 + softmax_scale=softmax_scale,
338 + causal=False,
339 + window_size=sliding_window,
340 + block_table=block_table[:1],
341 + softcap=logits_soft_cap,
342 + return_softmax_lse=True,
343 + )
344 +
345 + # Process suffix per query.
346 + suffix_output, suffix_lse = flash_attn_varlen_func(
347 + q=query,
348 + k=key_cache,
349 + v=value_cache,
350 + cu_seqlens_q=cu_query_lens,
351 + cu_seqlens_k=cu_suffix_kv_lens,
352 + max_seqlen_q=max_query_len,
353 + max_seqlen_k=max_kv_len - common_prefix_len,
354 + softmax_scale=softmax_scale,
355 + causal=True,
356 + window_size=sliding_window,
357 + block_table=block_table[:, num_common_kv_blocks:],
358 + softcap=logits_soft_cap,
359 + return_softmax_lse=True,
360 + )
361 +
362 + # Merge prefix and suffix outputs, and store the result in output.
363 + merge_attn_states(output, prefix_output, prefix_lse, suffix_output,
364 + suffix_lse)
365 +
366 +
367 +def merge_attn_states(
368 + output: torch.Tensor,
369 + prefix_output: torch.Tensor,
370 + prefix_lse: torch.Tensor,
371 + suffix_output: torch.Tensor,
372 + suffix_lse: torch.Tensor,
373 +) -> None:
374 + num_tokens = output.shape[0]
375 + num_query_heads = output.shape[1]
376 + head_size = output.shape[2]
377 + padded_head_size = triton.next_power_of_2(head_size)
378 +
379 + # TODO(woosuk): Use CUDA kernel instead of Triton to minimize CPU overhead.
380 + merge_attn_states_kernel[(num_tokens, num_query_heads)](
381 + output,
382 + prefix_output,
383 + prefix_lse,
384 + suffix_output,
385 + suffix_lse,
386 + head_size,
387 + padded_head_size,
388 + )
389 +
390 +
391 +@triton.jit
392 +def merge_attn_states_kernel(
393 + output, # [NUM_TOKENS, NUM_HEADS, HEAD_SIZE]
394 + prefix_output, # [NUM_TOKENS, NUM_HEADS, HEAD_SIZE]
395 + prefix_lse, # [NUM_HEADS, NUM_TOKENS]
396 + suffix_output, # [NUM_TOKENS, NUM_HEADS, HEAD_SIZE]
397 + suffix_lse, # [NUM_HEADS, NUM_TOKENS]
398 + HEAD_SIZE: tl.constexpr,
399 + PADDED_HEAD_SIZE: tl.constexpr,
400 +):
401 + token_idx = tl.program_id(0)
402 + num_tokens = tl.num_programs(0)
403 + head_idx = tl.program_id(1)
404 + num_heads = tl.num_programs(1)
405 +
406 + p_lse = tl.load(prefix_lse + head_idx * num_tokens + token_idx)
407 + s_lse = tl.load(suffix_lse + head_idx * num_tokens + token_idx)
408 + max_lse = tl.maximum(p_lse, s_lse)
409 + p_lse = p_lse - max_lse
410 + s_lse = s_lse - max_lse
411 +
412 + head_arange = tl.arange(0, PADDED_HEAD_SIZE)
413 + head_mask = head_arange < HEAD_SIZE
414 + p_out = tl.load(prefix_output + token_idx * num_heads * HEAD_SIZE +
415 + head_idx * HEAD_SIZE + head_arange,
416 + mask=head_mask)
417 + s_out = tl.load(suffix_output + token_idx * num_heads * HEAD_SIZE +
418 + head_idx * HEAD_SIZE + head_arange,
419 + mask=head_mask)
420 +
421 + # NOTE(woosuk): Be careful with the numerical stability.
422 + # We should compute the scale first, and then multiply it with the output.
423 + # Do not multiply the output with tl.exp(p_lse) or tl.exp(s_lse) directly.
424 + p_scale = tl.exp(p_lse) / (tl.exp(p_lse) + tl.exp(s_lse))
425 + s_scale = tl.exp(s_lse) / (tl.exp(p_lse) + tl.exp(s_lse))
426 + out = p_out * p_scale + s_out * s_scale
427 + tl.store(output + token_idx * num_heads * HEAD_SIZE +
428 + head_idx * HEAD_SIZE + head_arange,
429 + out,
430 + mask=head_mask)
vllm/v1/core/kv_cache_manager.py
+51 -1
@@ -8,7 +8,7 @@ from vllm.v1.core.kv_cache_utils import (BlockHashType, FreeKVCacheBlockQueue,
8 generate_block_hash_extra_keys,
9 hash_block_tokens,
10 hash_request_tokens)
11 -from vllm.v1.request import Request
11 +from vllm.v1.request import Request, RequestStatus
12
13 logger = init_logger(__name__)
14
@@ -278,6 +278,56 @@ class KVCacheManager:
278 if block.ref_cnt == 0:
279 self.free_block_queue.append(block)
280
281 + def get_num_common_prefix_blocks(
282 + self,
283 + request: Request,
284 + num_running_requests: int,
285 + ) -> int:
286 + """Calculate the number of common prefix blocks shared by all requests
287 + in the RUNNING state.
288 +
289 + The function determines this by selecting any request and iterating
290 + through its blocks. A block is considered a common prefix block if its
291 + `ref_cnt` equals the total number of requests in the RUNNING state.
292 +
293 + NOTE(woosuk): The number of requests in the RUNNING state is **greater
294 + than or equal to** the number of requests scheduled in the current step.
295 + This is because the RUNNING state only indicates that:
296 + 1. The request has not yet finished, and
297 + 2. The request holds its blocks unfreed.
298 +
299 + While all scheduled requests must be in the RUNNING state, the inverse
300 + is not necessarily true. There may be RUNNING requests that are not
301 + scheduled in the current step. As of 1/1/2025, the scheduler does not
302 + allow this case, but it is possible in the future, as we allow more
303 + flexible scheduling.
304 +
305 + This can result in an edge case where the number of common prefix blocks
306 + is 0, even though all scheduled requests share a common prefix. This
307 + occurs because there may be unscheduled RUNNING requests that do not
308 + share the common prefix. Currently, this case cannot be easily detected,
309 + so the function returns 0 in such cases.
310 +
311 + Args:
312 + request: Any request in the RUNNING state, used to identify the
313 + common prefix blocks.
314 + num_running_requests: The total number of requests in the RUNNING
315 + state. This can be different from the number of scheduled
316 + requests in the current step.
317 +
318 + Returns:
319 + int: The number of common prefix blocks.
320 + """
321 + assert request.status == RequestStatus.RUNNING
322 + blocks = self.req_to_blocks[request.request_id]
323 + num_common_blocks = 0
324 + for block in blocks:
325 + if block.ref_cnt == num_running_requests:
326 + num_common_blocks += 1
327 + else:
328 + break
329 + return num_common_blocks
330 +
331 def _get_new_blocks(self, num_blocks: int) -> List[KVCacheBlock]:
332 """Get new blocks from the free block pool.
333
vllm/v1/core/scheduler.py
+10
@@ -262,6 +262,14 @@ class Scheduler:
262 assert (len(scheduled_new_reqs) + len(scheduled_resumed_reqs) +
263 len(scheduled_running_reqs) == len(self.running))
264
265 + # Get the longest common prefix among all requests in the running queue.
266 + # This can be potentially used for cascade attention.
267 + if self.running:
268 + any_request = self.running[0]
269 + num_common_prefix_blocks = (
270 + self.kv_cache_manager.get_num_common_prefix_blocks(
271 + any_request, len(self.running)))
272 +
273 # Construct the scheduler output.
274 new_reqs_data = [
275 NewRequestData.from_request(req,
@@ -287,6 +295,7 @@ class Scheduler:
295 num_scheduled_tokens=num_scheduled_tokens,
296 total_num_scheduled_tokens=total_num_scheduled_tokens,
297 scheduled_encoder_inputs=scheduled_encoder_inputs,
298 + num_common_prefix_blocks=num_common_prefix_blocks,
299 preempted_req_ids=preempted_req_ids,
300 # finished_req_ids is an existing state in the scheduler,
301 # instead of being newly scheduled in this step.
@@ -594,6 +603,7 @@ class SchedulerOutput:
603 num_scheduled_tokens: Dict[str, int]
604 total_num_scheduled_tokens: int
605 scheduled_encoder_inputs: Dict[str, List[int]]
606 + num_common_prefix_blocks: int
607
608 preempted_req_ids: Set[str]
609 finished_req_ids: Set[str]
vllm/v1/worker/gpu_model_runner.py
+95 -1
@@ -72,6 +72,8 @@ class GPUModelRunner:
72 # Model-related.
73 self.num_attn_layers = model_config.get_num_layers_by_block_type(
74 parallel_config, LayerBlockType.attention)
75 + self.num_query_heads = model_config.get_num_attention_heads(
76 + parallel_config)
77 self.num_kv_heads = model_config.get_num_kv_heads(parallel_config)
78 self.head_size = model_config.get_head_size()
79 self.hidden_size = model_config.get_hidden_size()
@@ -118,6 +120,10 @@ class GPUModelRunner:
120 self.cudagraph_batch_sizes = list(
121 reversed(self.vllm_config.compilation_config.capture_sizes))
122
123 + # Cache the device properties.
124 + self.device_properties = torch.cuda.get_device_properties(self.device)
125 + self.num_sms = self.device_properties.multi_processor_count
126 +
127 # Persistent buffers for CUDA graphs.
128 self.input_ids = torch.zeros(self.max_num_tokens,
129 dtype=torch.int32,
@@ -131,7 +137,8 @@ class GPUModelRunner:
137 device=self.device)
138
139 # OPTIMIZATION: Cache the tensors rather than creating them every step.
134 - self.arange_np = np.arange(max(self.max_num_reqs, self.max_model_len),
140 + self.arange_np = np.arange(max(self.max_num_reqs + 1,
141 + self.max_model_len),
142 dtype=np.int32)
143 # NOTE(woosuk): These tensors are "stateless", i.e., they are literally
144 # a faster version of creating a new tensor every time. Thus, we should
@@ -355,6 +362,88 @@ class GPUModelRunner:
362 self.device, non_blocking=True)
363 slot_mapping = self.slot_mapping_cpu[:total_num_scheduled_tokens].to(
364 self.device, non_blocking=True).long()
365 +
366 + # Prepare for cascade attention if needed.
367 + common_prefix_len = (scheduler_output.num_common_prefix_blocks *
368 + self.block_size)
369 + if common_prefix_len == 0:
370 + # Common case.
371 + use_cascade = False
372 + else:
373 + # NOTE(woosuk): Cascade attention uses two attention kernels: one
374 + # for the common prefix and the other for the rest. For the first
375 + # kernel, we concatenate all the query tokens (possibly from
376 + # different requests) and treat them as if they are from the same
377 + # request. Then, we use bi-directional attention to process the
378 + # common prefix in the KV cache. Importantly, this means that the
379 + # first kernel does not do any masking.
380 +
381 + # Consider the following example:
382 + # Request 1's input query: [D, E, X]
383 + # Request 1's kv cache: [A, B, C, D, E, X]
384 + # Request 1's num_computed_tokens: 3 (i.e., [A, B, C])
385 + # Request 2's input query: [E, Y]
386 + # Request 2's kv cache: [A, B, C, D, E, Y]
387 + # Request 2's num_computed_tokens: 4 (i.e., [A, B, C, D])
388 +
389 + # If we use [A, B, C, D, E] as the common prefix, then the
390 + # first kernel will compute the bi-directional attention between
391 + # input query [D, E, X, E, Y] and common prefix [A, B, C, D, E].
392 + # However, this is wrong because D in Request 1 should not attend to
393 + # E in the common prefix (i.e., we need masking).
394 + # To avoid this, [A, B, C, D] should be the common prefix.
395 + # That is, the common prefix should be capped by the minimum
396 + # num_computed_tokens among the requests, and plus one to include
397 + # the first token of the query.
398 +
399 + # In practice, we use [A, B, C] as the common prefix, instead of
400 + # [A, B, C, D] (i.e., the common prefix is capped by the minimum
401 + # num_computed_tokens, without plus one).
402 + # This is because of an implementation detail: We want to always
403 + # use two kernels for cascade attention. Let's imagine:
404 + # Request 3's input query: [D]
405 + # Request 3's kv cache: [A, B, C, D]
406 + # Request 3's num_computed_tokens: 4 (i.e., [A, B, C, D])
407 + # If we use [A, B, C, D] as the common prefix for Request 1-3,
408 + # then Request 3 will be processed only by the first kernel,
409 + # and the second kernel will get an empty input. While this is not
410 + # a fundamental problem, our current implementation does not support
411 + # this case.
412 + common_prefix_len = min(
413 + common_prefix_len,
414 + self.input_batch.num_computed_tokens_cpu[:num_reqs].min())
415 + # common_prefix_len should be a multiple of the block size.
416 + common_prefix_len = (common_prefix_len // self.block_size *
417 + self.block_size)
418 + use_cascade = FlashAttentionBackend.use_cascade_attention(
419 + common_prefix_len=common_prefix_len,
420 + query_lens=num_scheduled_tokens,
421 + num_query_heads=self.num_query_heads,
422 + num_kv_heads=self.num_kv_heads,
423 + use_alibi=False, # FIXME
424 + use_sliding_window=self.sliding_window is not None,
425 + num_sms=self.num_sms,
426 + )
427 +
428 + if use_cascade:
429 + # TODO: Optimize.
430 + cu_prefix_query_lens = torch.tensor(
431 + [0, total_num_scheduled_tokens],
432 + dtype=torch.int32,
433 + device=self.device)
434 + cu_prefix_kv_lens = torch.tensor([0, common_prefix_len],
435 + dtype=torch.int32,
436 + device=self.device)
437 + cu_suffix_kv_lens = (
438 + self.seq_start_loc_np[:num_reqs + 1] -
439 + self.arange_np[:num_reqs + 1] * common_prefix_len)
440 + cu_suffix_kv_lens = torch.from_numpy(cu_suffix_kv_lens).to(
441 + self.device)
442 + else:
443 + cu_prefix_query_lens = None
444 + cu_prefix_kv_lens = None
445 + cu_suffix_kv_lens = None
446 +
447 attn_metadata = FlashAttentionMetadata(
448 num_actual_tokens=total_num_scheduled_tokens,
449 max_query_len=max_num_scheduled_tokens,
@@ -363,6 +452,11 @@ class GPUModelRunner:
452 seq_start_loc=seq_start_loc,
453 block_table=self.input_batch.block_table[:num_reqs],
454 slot_mapping=slot_mapping,
455 + use_cascade=use_cascade,
456 + common_prefix_len=common_prefix_len,
457 + cu_prefix_query_lens=cu_prefix_query_lens,
458 + cu_prefix_kv_lens=cu_prefix_kv_lens,
459 + cu_suffix_kv_lens=cu_suffix_kv_lens,
460 )
461 # NOTE(woosuk): Due to chunked prefills, there can be at most 1 partial
462 # request in the batch. While we should not sample any token from this