@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3b61cb450d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[V1] Further reduce CPU overheads in flash-attn (#10989)

Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Woosuk Kwon committed Dec 9, 2024 at 12:38 UTC 3b61cb450d899dc423feb264c297d4d18d701678
2 files changed +28 -7
csrc/cache_kernels.cu
+12 -2
@@ -307,10 +307,20 @@ void reshape_and_cache_flash(
307 torch::Tensor& key_cache, // [num_blocks, block_size, num_heads, head_size]
308 torch::Tensor&
309 value_cache, // [num_blocks, block_size, num_heads, head_size]
310 - torch::Tensor& slot_mapping, // [num_tokens]
310 + torch::Tensor& slot_mapping, // [num_tokens] or [num_actual_tokens]
311 const std::string& kv_cache_dtype, const double k_scale,
312 const double v_scale) {
313 - int num_tokens = key.size(0);
313 + // NOTE(woosuk): In vLLM V1, key.size(0) can be different from
314 + // slot_mapping.size(0) because of padding for CUDA graphs.
315 + // In vLLM V0, key.size(0) is always equal to slot_mapping.size(0) because
316 + // both include padding.
317 + // In vLLM V1, however, key.size(0) can be larger than slot_mapping.size(0)
318 + // since key includes padding for CUDA graphs, while slot_mapping does not.
319 + // In this case, slot_mapping.size(0) represents the actual number of tokens
320 + // before padding.
321 + // For compatibility with both cases, we use slot_mapping.size(0) as the
322 + // number of tokens.
323 + int num_tokens = slot_mapping.size(0);
324 int num_heads = key.size(1);
325 int head_size = key.size(2);
326 int block_size = key_cache.size(1);
vllm/v1/attention/backends/flash_attn.py
+16 -5
@@ -138,14 +138,25 @@ class FlashAttentionImpl(AttentionImpl):
138 # Profiling run.
139 return output
140
141 - num_actual_tokens = attn_metadata.num_actual_tokens
141 + # IMPORTANT!
142 + # NOTE(woosuk): With piece-wise CUDA graphs, this method is executed in
143 + # eager-mode PyTorch. Thus, we need to be careful about any CPU overhead
144 + # in this method. For example, `view` and `slice` (or `[:n]`) operations
145 + # are surprisingly slow even in the case they do not invoke any GPU ops.
146 + # Minimize the PyTorch ops in this method as much as possible.
147 + # Whenever making a change in this method, please benchmark the
148 + # performance to make sure it does not introduce any overhead.
149
150 + num_actual_tokens = attn_metadata.num_actual_tokens
151 # Reshape the input keys and values and store them in the cache.
144 - key_cache = kv_cache[0]
145 - value_cache = kv_cache[1]
152 + # NOTE(woosuk): Here, key and value are padded while slot_mapping is
153 + # not padded. However, we don't need to do key[:num_actual_tokens] and
154 + # value[:num_actual_tokens] because the reshape_and_cache_flash op uses
155 + # the slot_mapping's shape to determine the number of actual tokens.
156 + key_cache, value_cache = kv_cache.unbind(0)
157 torch.ops._C_cache_ops.reshape_and_cache_flash(
147 - key[:num_actual_tokens],
148 - value[:num_actual_tokens],
158 + key,
159 + value,
160 key_cache,
161 value_cache,
162 attn_metadata.slot_mapping,