A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[V1] Further reduce CPU overheads in flash-attn (#10989)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Woosuk Kwon committed
Dec 9, 2024 at 12:38 UTC
3b61cb450d899dc423feb264c297d4d18d701678
2 files changed
+28
-7
csrc/cache_kernels.cu
+12
-2
@@ -307,10 +307,20 @@ void reshape_and_cache_flash(
307
torch::Tensor& key_cache, // [num_blocks, block_size, num_heads, head_size]
308
torch::Tensor&
309
value_cache, // [num_blocks, block_size, num_heads, head_size]
310
- torch::Tensor& slot_mapping, // [num_tokens]
310
+ torch::Tensor& slot_mapping, // [num_tokens] or [num_actual_tokens]
311
const std::string& kv_cache_dtype, const double k_scale,
312
const double v_scale) {
313
- int num_tokens = key.size(0);
313
+ // NOTE(woosuk): In vLLM V1, key.size(0) can be different from
314
+ // slot_mapping.size(0) because of padding for CUDA graphs.
315
+ // In vLLM V0, key.size(0) is always equal to slot_mapping.size(0) because
316
+ // both include padding.
317
+ // In vLLM V1, however, key.size(0) can be larger than slot_mapping.size(0)
318
+ // since key includes padding for CUDA graphs, while slot_mapping does not.
319
+ // In this case, slot_mapping.size(0) represents the actual number of tokens
320
+ // before padding.
321
+ // For compatibility with both cases, we use slot_mapping.size(0) as the
322
+ // number of tokens.
323
+ int num_tokens = slot_mapping.size(0);
324
int num_heads = key.size(1);
325
int head_size = key.size(2);
326
int block_size = key_cache.size(1);
vllm/v1/attention/backends/flash_attn.py
+16
-5
@@ -138,14 +138,25 @@ class FlashAttentionImpl(AttentionImpl):
138
# Profiling run.
139
return output
140
141
- num_actual_tokens = attn_metadata.num_actual_tokens
141
+ # IMPORTANT!
142
+ # NOTE(woosuk): With piece-wise CUDA graphs, this method is executed in
143
+ # eager-mode PyTorch. Thus, we need to be careful about any CPU overhead
144
+ # in this method. For example, `view` and `slice` (or `[:n]`) operations
145
+ # are surprisingly slow even in the case they do not invoke any GPU ops.
146
+ # Minimize the PyTorch ops in this method as much as possible.
147
+ # Whenever making a change in this method, please benchmark the
148
+ # performance to make sure it does not introduce any overhead.
149
150
+ num_actual_tokens = attn_metadata.num_actual_tokens
151
# Reshape the input keys and values and store them in the cache.
144
- key_cache = kv_cache[0]
145
- value_cache = kv_cache[1]
152
+ # NOTE(woosuk): Here, key and value are padded while slot_mapping is
153
+ # not padded. However, we don't need to do key[:num_actual_tokens] and
154
+ # value[:num_actual_tokens] because the reshape_and_cache_flash op uses
155
+ # the slot_mapping's shape to determine the number of actual tokens.
156
+ key_cache, value_cache = kv_cache.unbind(0)
157
torch.ops._C_cache_ops.reshape_and_cache_flash(
147
- key[:num_actual_tokens],
148
- value[:num_actual_tokens],
158
+ key,
159
+ value,
160
key_cache,
161
value_cache,
162
attn_metadata.slot_mapping,