@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 98f47f2a40
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[V1] Optimize the CPU overheads in FlashAttention custom op (#10733)

Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Woosuk Kwon committed Nov 28, 2024 at 09:01 UTC 98f47f2a4032f8c395268de80858c64ffcfc60fa
1 file changed +9 -8
vllm/v1/attention/backends/flash_attn.py
+9 -8
@@ -135,6 +135,13 @@ class FlashAttentionImpl(AttentionImpl):
135 assert k_scale == 1.0 and v_scale == 1.0, (
136 "key/v_scale is not supported in FlashAttention.")
137
138 + # Reshape the query, key, and value tensors.
139 + # NOTE(woosuk): We do this outside the custom op to minimize the CPU
140 + # overheads from the non-CUDA-graph regions.
141 + query = query.view(-1, self.num_heads, self.head_size)
142 + key = key.view(-1, self.num_kv_heads, self.head_size)
143 + value = value.view(-1, self.num_kv_heads, self.head_size)
144 +
145 output = torch.empty_like(query)
146 torch.ops.vllm.unified_v1_flash_attention(
147 output,
@@ -153,7 +160,7 @@ class FlashAttentionImpl(AttentionImpl):
160 self.alibi_slopes,
161 self.logits_soft_cap,
162 )
156 - return output
163 + return output.view(-1, self.num_heads * self.head_size)
164
165
166 def unified_v1_flash_attention(
@@ -184,11 +191,6 @@ def unified_v1_flash_attention(
191 attn_metadata: FlashAttentionMetadata = current_metadata
192 num_actual_tokens = attn_metadata.num_actual_tokens
193
187 - # Reshape the query, key, and value tensors.
188 - query = query.view(-1, num_heads, head_size)
189 - key = key.view(-1, num_kv_heads, head_size)
190 - value = value.view(-1, num_kv_heads, head_size)
191 -
194 # Reshape the input keys and values and store them in the cache.
195 key_cache = kv_cache[0]
196 value_cache = kv_cache[1]
@@ -218,8 +220,7 @@ def unified_v1_flash_attention(
220 block_table=attn_metadata.block_table,
221 softcap=logits_soft_cap,
222 )
221 - attn_output = attn_output.view(num_actual_tokens, -1)
222 - # TODO(woosuk): Optimize this.
223 + # TODO(woosuk): Remove this unnecessary copy.
224 output[:num_actual_tokens].copy_(attn_output)
225
226