@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 897cb2ae28
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Optimize data movement (#20)

Woosuk Kwon committed Apr 2, 2023 at 00:30 UTC 897cb2ae28e93de1b22ecfbffcccfb9493f8f4d9
17 files changed +275 -135
cacheflow/models/activation.py new
+20
@@ -0,0 +1,20 @@
1 +import torch
2 +import torch.nn as nn
3 +
4 +from cacheflow import activation_ops
5 +
6 +
7 +class SiluAndMul(nn.Module):
8 +
9 + def __init__(self):
10 + super().__init__()
11 +
12 + def forward(
13 + self,
14 + x: torch.Tensor, # (num_tokens, 2 * d)
15 + ) -> torch.Tensor: # (num_tokens, d)
16 + num_tokens = x.shape[0]
17 + d = x.shape[1] // 2
18 + out = torch.empty(num_tokens, d, dtype=x.dtype, device=x.device)
19 + activation_ops.silu_and_mul(out, x)
20 + return out
cacheflow/models/attention.py
+46 -46
@@ -1,6 +1,6 @@
1 -from typing import List, Optional
1 +from typing import Optional
2
3 -from flash_attn.flash_attention import FlashAttention
3 +from flash_attn.flash_attn_interface import _flash_attn_forward
4 import torch
5 import torch.nn as nn
6
@@ -16,40 +16,38 @@ class GPTCacheFlowAttention(nn.Module):
16 super().__init__()
17 self.scale = float(scale)
18
19 - self.flash_attn = FlashAttention(softmax_scale=self.scale)
20 -
19 def multi_query_kv_attention(
20 self,
23 - output: torch.Tensor, # [num_prompt_tokens, num_heads, head_size]
24 - query: torch.Tensor, # [num_prompt_tokens, num_heads, head_size]
25 - key: torch.Tensor, # [num_prompt_tokens, num_heads, head_size]
26 - value: torch.Tensor, # [num_prompt_tokens, num_heads, head_size]
27 - prompt_lens: List[int],
21 + output: torch.Tensor, # [num_prompt_tokens, num_heads, head_size]
22 + query: torch.Tensor, # [num_prompt_tokens, num_heads, head_size]
23 + key: torch.Tensor, # [num_prompt_tokens, num_heads, head_size]
24 + value: torch.Tensor, # [num_prompt_tokens, num_heads, head_size]
25 + cumulative_prompt_lens: torch.Tensor, # [num_prompts + 1]
26 + max_prompt_len: int,
27 ) -> None:
28 if query.dtype == torch.float:
29 raise ValueError('The float data type is not supported by '
30 'FlashAttention. Use the half data type instead.')
32 - head_size = query.shape[2]
31 + head_size = query.shape[-1]
32 if head_size > 128:
33 raise ValueError('FlashAttention does not support head_size > 128.')
34
36 - device = query.device
37 - prefix_sum = [0]
38 - for prompt_len in prompt_lens:
39 - prefix_sum.append(prefix_sum[-1] + prompt_len)
40 - prefix_sum = torch.tensor(prefix_sum, dtype=torch.int, device=device)
41 - max_prompt_len = max(prompt_lens)
42 -
43 - # FIXME(woosuk): Unnecessary copy. Optimize this.
44 - qkv = torch.stack([query, key, value], dim=1)
45 - out = self.flash_attn(
46 - qkv,
47 - cu_seqlens=prefix_sum,
48 - max_s=max_prompt_len,
35 + # Directly call FlashAttention's internal function to avoid allocating
36 + # a new tensor for the output.
37 + _flash_attn_forward(
38 + query,
39 + key,
40 + value,
41 + output,
42 + cumulative_prompt_lens,
43 + cumulative_prompt_lens,
44 + max_prompt_len,
45 + max_prompt_len,
46 + dropout_p=0.0,
47 + softmax_scale=self.scale,
48 causal=True,
50 - )[0]
51 - # FIXME(woosuk): Unnecessary copy. Optimize this.
52 - output.copy_(out, non_blocking=True)
49 + return_softmax=False,
50 + )
51
52 def single_query_cached_kv_attention(
53 self,
@@ -90,21 +88,18 @@ class GPTCacheFlowAttention(nn.Module):
88 input_metadata: InputMetadata,
89 cache_event: Optional[torch.cuda.Event],
90 ) -> torch.Tensor: # [num_tokens, num_heads * head_size]
93 - # Pre-allocate the output tensor.
94 - output = torch.empty_like(query)
95 -
96 - # Prune out paddings if any.
97 - query = query[:input_metadata.num_valid_tokens]
98 - key = key[:input_metadata.num_valid_tokens]
99 - value = value[:input_metadata.num_valid_tokens]
91 + # NOTE: The query, key, and value tensors must be sliced from a qkv
92 + # tensor of shape [num_tokens, 3 * num_heads * head_size].
93
101 - # Reshape the input tensors.
94 + # Reshape the query, key, and value tensors.
95 num_heads = value_cache.shape[1]
96 head_size = value_cache.shape[2]
97 query = query.view(-1, num_heads, head_size)
98 key = key.view(-1, num_heads, head_size)
99 value = value.view(-1, num_heads, head_size)
107 - output = output.view(-1, num_heads, head_size)
100 +
101 + # Pre-allocate the output tensor.
102 + output = torch.empty_like(query)
103
104 # Compute the attention op for prompts.
105 num_prompt_tokens = input_metadata.num_prompt_tokens
@@ -114,7 +109,8 @@ class GPTCacheFlowAttention(nn.Module):
109 query[:num_prompt_tokens],
110 key[:num_prompt_tokens],
111 value[:num_prompt_tokens],
117 - input_metadata.prompt_lens,
112 + input_metadata.cumulative_prompt_lens,
113 + input_metadata.max_prompt_len,
114 )
115
116 # Wait until the cache op is done.
@@ -122,14 +118,22 @@ class GPTCacheFlowAttention(nn.Module):
118 cache_event.wait()
119
120 # Reshape the keys and values and store them in the cache.
125 - cache_ops.reshape_and_cache(
126 - key, value, key_cache, value_cache, input_metadata.slot_mapping)
121 + num_valid_tokens = input_metadata.num_valid_tokens
122 + if num_valid_tokens > 0:
123 + # The stride is 3 because the key and value are sliced from qkv.
124 + cache_ops.reshape_and_cache(
125 + key[:num_valid_tokens],
126 + value[:num_valid_tokens],
127 + key_cache,
128 + value_cache,
129 + input_metadata.slot_mapping,
130 + )
131
132 if input_metadata.num_generation_tokens > 0:
133 # Compute the attention op for generation tokens.
134 self.single_query_cached_kv_attention(
131 - output[num_prompt_tokens:],
132 - query[num_prompt_tokens:],
135 + output[num_prompt_tokens:num_valid_tokens],
136 + query[num_prompt_tokens:num_valid_tokens],
137 key_cache,
138 value_cache,
139 input_metadata)
@@ -186,19 +190,15 @@ class LlamaCacheFlowAttention(GPTCacheFlowAttention):
190 ) -> torch.Tensor: # [num_tokens, num_heads * head_size]
191 # Apply rotary embedding to the query and key before passing them
192 # to the attention op.
189 - out_query = torch.empty_like(query)
190 - out_key = torch.empty_like(key)
193 pos_encoding_ops.rotary_embedding_neox(
192 - out_query,
193 - out_key,
194 positions,
195 query,
196 key,
197 self.cos_sin_cache,
198 )
199 return super().forward(
200 - out_query,
201 - out_key,
200 + query,
201 + key,
202 value,
203 key_cache,
204 value_cache,
cacheflow/models/input_metadata.py
+5
@@ -12,6 +12,7 @@ class InputMetadata:
12 seq_groups: List[Tuple[List[int], SamplingParams]],
13 seq_logprobs: Dict[int, float], # Seq id -> cumulative logprobs.
14 prompt_lens: List[int],
15 + cumulative_prompt_lens: torch.Tensor,
16 slot_mapping: torch.Tensor,
17 context_lens: torch.Tensor,
18 max_context_len: int,
@@ -20,6 +21,7 @@ class InputMetadata:
21 self.seq_groups = seq_groups
22 self.seq_logprobs = seq_logprobs
23 self.prompt_lens = prompt_lens
24 + self.cumulative_prompt_lens = cumulative_prompt_lens
25 self.slot_mapping = slot_mapping
26 self.context_lens = context_lens
27 self.max_context_len = max_context_len
@@ -27,6 +29,7 @@ class InputMetadata:
29
30 self.num_prompts = len(prompt_lens)
31 self.num_prompt_tokens = sum(prompt_lens)
32 + self.max_prompt_len = max(prompt_lens) if prompt_lens else 0
33 self.num_generation_tokens = context_lens.shape[0]
34 self.num_valid_tokens = slot_mapping.shape[0]
35 if block_tables.numel() > 0:
@@ -40,11 +43,13 @@ class InputMetadata:
43 return (f'InputMetadata('
44 f'num_prompts={self.num_prompts}, '
45 f'num_prompt_tokens={self.num_prompt_tokens}, '
46 + f'max_prompt_len={self.max_prompt_len}, '
47 f'num_generation_tokens={self.num_generation_tokens}, '
48 f'num_valid_tokens={self.num_valid_tokens}, '
49 f'max_num_blocks_per_seq={self.max_num_blocks_per_seq}, '
50 f'max_context_len={self.max_context_len}), '
51 f'prompt_lens={self.prompt_lens}, '
52 + f'cumulative_prompt_lens={self.cumulative_prompt_lens}, '
53 f'slot_mapping={self.slot_mapping}, '
54 f'context_lens={self.context_lens}, '
55 f'block_tables={self.block_tables})')
cacheflow/models/llama.py
+7 -12
@@ -11,6 +11,7 @@ from torch import nn
11 from transformers import LlamaConfig
12
13 from cacheflow.models import InputMetadata
14 +from cacheflow.models.activation import SiluAndMul
15 from cacheflow.models.attention import LlamaCacheFlowAttention
16 from cacheflow.models.layernorm import RMSNorm
17 from cacheflow.models.sample import Sampler
@@ -39,16 +40,14 @@ class LlamaMLP(nn.Module):
40 self.down_proj = RowParallelLinear(intermediate_size, hidden_size,
41 bias=False, input_is_parallel=True,
42 perform_initialization=False)
42 - assert hidden_act == 'silu'
43 - self.act_fn = nn.SiLU()
43 + if hidden_act != 'silu':
44 + raise ValueError(f'Unsupported activation: {hidden_act}. '
45 + 'Only silu is supported for now.')
46 + self.act_fn = SiluAndMul()
47
48 def forward(self, x):
49 gate_up, _ = self.gate_up_proj(x)
47 - gate_up = gate_up.reshape(gate_up.shape[:-1] + (2, -1))
48 - gate, up = torch.split(gate_up, 1, dim=-2)
49 - gate = gate.squeeze(dim=-2).contiguous()
50 - up = up.squeeze(dim=-2).contiguous()
51 - x = self.act_fn(gate) * up
50 + x = self.act_fn(gate_up)
51 x, _ = self.down_proj(x)
52 return x
53
@@ -94,11 +93,7 @@ class LlamaAttention(nn.Module):
93 cache_event: Optional[torch.cuda.Event],
94 ) -> torch.Tensor:
95 qkv, _ = self.qkv_proj(hidden_states)
97 - qkv = qkv.reshape(qkv.shape[:-1] + (3, -1))
98 - q, k, v = torch.split(qkv, 1, dim=-2)
99 - q = q.squeeze(dim=-2).contiguous()
100 - k = k.squeeze(dim=-2).contiguous()
101 - v = v.squeeze(dim=-2).contiguous()
96 + q, k, v = qkv.chunk(chunks=3, dim=-1)
97 k_cache, v_cache = kv_cache
98 attn_output = self.attn(
99 positions, q, k, v, k_cache, v_cache, input_metadata, cache_event)
cacheflow/models/opt.py
+2 -5
@@ -69,17 +69,14 @@ class OPTAttention(nn.Module):
69 cache_event: Optional[torch.cuda.Event],
70 ) -> torch.Tensor:
71 qkv, _ = self.qkv_proj(hidden_states)
72 - qkv = qkv.reshape(qkv.shape[:-1] + (3, -1))
73 - q, k, v = torch.split(qkv, 1, dim=-2)
74 - q = q.squeeze(dim=-2).contiguous()
75 - k = k.squeeze(dim=-2).contiguous()
76 - v = v.squeeze(dim=-2).contiguous()
72 + q, k, v = qkv.chunk(chunks=3, dim=-1)
73 key_cache, value_cache = kv_cache
74 attn_output = self.attn(
75 q, k, v, key_cache, value_cache, input_metadata, cache_event)
76 output, _ = self.out_proj(attn_output)
77 return output
78
79 +
80 class OPTDecoderLayer(nn.Module):
81
82 def __init__(self, config: OPTConfig):
cacheflow/worker/worker.py
+8
@@ -128,6 +128,11 @@ class Worker:
128 slot = block_number * self.block_size + block_offset
129 slot_mapping.append(slot)
130
131 + cumulative_prompt_lens: List[int] = [0]
132 + for prompt_len in prompt_lens:
133 + cumulative_prompt_lens.append(
134 + cumulative_prompt_lens[-1] + prompt_len)
135 +
136 # Add generation tokens.
137 max_context_len = 0
138 max_num_blocks_per_seq = 0
@@ -183,11 +188,14 @@ class Worker:
188 for block_table in generation_block_tables]
189 block_tables_tensor = torch.tensor(
190 padded_block_tables, dtype=torch.int, device='cuda')
191 + cumulative_prompt_lens_tensor = torch.tensor(
192 + cumulative_prompt_lens, dtype=torch.int, device='cuda')
193
194 input_metadata = InputMetadata(
195 seq_groups=seq_groups,
196 seq_logprobs=seq_logprobs,
197 prompt_lens=prompt_lens,
198 + cumulative_prompt_lens=cumulative_prompt_lens_tensor,
199 slot_mapping=slot_mapping_tensor,
200 context_lens=context_lens_tensor,
201 max_context_len=max_context_len,
csrc/activation.cpp new
+12
@@ -0,0 +1,12 @@
1 +#include <torch/extension.h>
2 +
3 +void silu_and_mul(
4 + torch::Tensor& out,
5 + torch::Tensor& input);
6 +
7 +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
8 + m.def(
9 + "silu_and_mul",
10 + &silu_and_mul,
11 + "Activation function used in SwiGLU.");
12 +}
csrc/activation_kernels.cu new
+46
@@ -0,0 +1,46 @@
1 +#include <torch/extension.h>
2 +#include <ATen/cuda/CUDAContext.h>
3 +
4 +namespace cacheflow {
5 +
6 +template<typename T>
7 +__device__ __forceinline__ T silu(const T& x) {
8 + // x * sigmoid(x)
9 + return (T) (((float) x) / (1.0f + expf((float) -x)));
10 +}
11 +
12 +template<typename scalar_t>
13 +__global__ void silu_and_mul_kernel(
14 + scalar_t* __restrict__ out, // [num_tokens, d]
15 + const scalar_t* __restrict__ input, // [num_tokens, 2, d]
16 + const int d) {
17 + const int token_idx = blockIdx.x;
18 + for (int idx = threadIdx.x; idx < d; idx += blockDim.x) {
19 + const scalar_t x = __ldg(&input[token_idx * 2 * d + idx]);
20 + const scalar_t y = __ldg(&input[token_idx * 2 * d + d + idx]);
21 + out[token_idx * d + idx] = silu(x) * y;
22 + }
23 +}
24 +
25 +} // namespace cacheflow
26 +
27 +void silu_and_mul(
28 + torch::Tensor& out, // [num_tokens, d]
29 + torch::Tensor& input) // [num_tokens, 2 * d]
30 +{
31 + int num_tokens = input.size(0);
32 + int d = input.size(1) / 2;
33 +
34 + dim3 grid(num_tokens);
35 + dim3 block(std::min(d, 1024));
36 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
37 + AT_DISPATCH_FLOATING_TYPES_AND_HALF(
38 + input.scalar_type(),
39 + "silu_and_mul_kernel",
40 + [&] {
41 + cacheflow::silu_and_mul_kernel<scalar_t><<<grid, block, 0, stream>>>(
42 + out.data_ptr<scalar_t>(),
43 + input.data_ptr<scalar_t>(),
44 + d);
45 + });
46 +}
csrc/attention_kernels.cu
+13 -9
@@ -25,7 +25,8 @@ __global__ void single_query_cached_kv_attention_kernel(
25 const float scale,
26 const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
27 const int* __restrict__ context_lens, // [num_seqs]
28 - const int max_num_blocks_per_seq) {
28 + const int max_num_blocks_per_seq,
29 + const int q_stride) {
30 constexpr int THREAD_GROUP_SIZE = WARP_SIZE / BLOCK_SIZE;
31 constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
32 const int thread_idx = threadIdx.x;
@@ -56,7 +57,8 @@ __global__ void single_query_cached_kv_attention_kernel(
57 // For example, if the the thread group size is 4, then the first thread in the group
58 // has 0, 4, 8, ... th vectors of the query, and the second thread has 1, 5, 9, ...
59 // th vectors of the query, and so on.
59 - const scalar_t* q_ptr = q + seq_idx * num_heads * HEAD_SIZE + head_idx * HEAD_SIZE;
60 + // NOTE(woosuk): Because q is split from a qkv tensor, it may not be contiguous.
61 + const scalar_t* q_ptr = q + seq_idx * q_stride + head_idx * HEAD_SIZE;
62 Q_vec q_vecs[NUM_VECS_PER_THREAD];
63 #pragma unroll
64 for (int i = 0; i < NUM_VECS_PER_THREAD; i++) {
@@ -264,7 +266,8 @@ __global__ void single_query_cached_kv_attention_kernel(
266 scale, \
267 block_tables_ptr, \
268 context_lens_ptr, \
267 - max_num_blocks_per_seq);
269 + max_num_blocks_per_seq, \
270 + query_stride);
271
272 // TODO(woosuk): Tune NUM_THREADS.
273 template<
@@ -284,6 +287,7 @@ void single_query_cached_kv_attention_launcher(
287 int num_heads = query.size(1);
288 int head_size = query.size(2);
289 int max_num_blocks_per_seq = block_tables.size(1);
290 + int query_stride = query.stride(0);
291
292 T* out_ptr = reinterpret_cast<T*>(out.data_ptr());
293 T* query_ptr = reinterpret_cast<T*>(query.data_ptr());
@@ -333,13 +337,13 @@ void single_query_cached_kv_attention_launcher(
337 }
338
339 void single_query_cached_kv_attention(
336 - torch::Tensor& out,
337 - torch::Tensor& query,
338 - torch::Tensor& key_cache,
339 - torch::Tensor& value_cache,
340 + torch::Tensor& out, // [num_seqs, num_heads, head_size]
341 + torch::Tensor& query, // [num_seqs, num_heads, head_size]
342 + torch::Tensor& key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
343 + torch::Tensor& value_cache, // [num_blocks, num_heads, head_size, block_size]
344 float scale,
341 - torch::Tensor& block_tables,
342 - torch::Tensor& context_lens,
345 + torch::Tensor& block_tables, // [num_seqs, max_num_blocks_per_seq]
346 + torch::Tensor& context_lens, // [num_seqs]
347 int block_size,
348 int max_context_len) {
349 // TODO(woosuk): Support BF16.
csrc/cache_kernels.cu
+17 -8
@@ -81,6 +81,8 @@ __global__ void reshape_and_cache_kernel(
81 scalar_t* __restrict__ key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
82 scalar_t* __restrict__ value_cache, // [num_blocks, num_heads, head_size, block_size]
83 const int* __restrict__ slot_mapping, // [num_tokens]
84 + const int key_stride,
85 + const int value_stride,
86 const int num_heads,
87 const int head_size,
88 const int block_size,
@@ -92,7 +94,8 @@ __global__ void reshape_and_cache_kernel(
94
95 const int n = num_heads * head_size;
96 for (int i = threadIdx.x; i < n; i += blockDim.x) {
95 - const int src_idx = token_idx * n + i;
97 + const int src_key_idx = token_idx * key_stride + i;
98 + const int src_value_idx = token_idx * value_stride + i;
99
100 const int head_idx = i / head_size;
101 const int head_offset = i % head_size;
@@ -108,25 +111,29 @@ __global__ void reshape_and_cache_kernel(
111 + head_idx * head_size * block_size
112 + head_offset * block_size
113 + block_offset;
111 - key_cache[tgt_key_idx] = __ldg(&key[src_idx]);
112 - value_cache[tgt_value_idx] = __ldg(&value[src_idx]);
114 + key_cache[tgt_key_idx] = __ldg(&key[src_key_idx]);
115 + value_cache[tgt_value_idx] = __ldg(&value[src_value_idx]);
116 }
117 }
118
119 } // namespace cacheflow
120
121 void reshape_and_cache(
119 - torch::Tensor& key,
120 - torch::Tensor& value,
121 - torch::Tensor& key_cache,
122 - torch::Tensor& value_cache,
123 - torch::Tensor& slot_mapping) {
122 + torch::Tensor& key, // [num_tokens, num_heads, head_size]
123 + torch::Tensor& value, // [num_tokens, num_heads, head_size]
124 + torch::Tensor& key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
125 + torch::Tensor& value_cache, // [num_blocks, num_heads, head_size, block_size]
126 + torch::Tensor& slot_mapping) // [num_tokens]
127 +{
128 int num_tokens = key.size(0);
129 int num_heads = key.size(1);
130 int head_size = key.size(2);
131 int block_size = key_cache.size(3);
132 int x = key_cache.size(4);
133
134 + int key_stride = key.stride(0);
135 + int value_stride = value.stride(0);
136 +
137 dim3 grid(num_tokens);
138 dim3 block(std::min(num_heads * head_size, 512));
139 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
@@ -140,6 +147,8 @@ void reshape_and_cache(
147 key_cache.data_ptr<scalar_t>(),
148 value_cache.data_ptr<scalar_t>(),
149 slot_mapping.data_ptr<int>(),
150 + key_stride,
151 + value_stride,
152 num_heads,
153 head_size,
154 block_size,
csrc/pos_encoding.cpp
-2
@@ -1,8 +1,6 @@
1 #include <torch/extension.h>
2
3 void rotary_embedding_neox(
4 - torch::Tensor& out_query,
5 - torch::Tensor& out_key,
4 torch::Tensor& positions,
5 torch::Tensor& query,
6 torch::Tensor& key,
csrc/pos_encoding_kernels.cu
+22 -27
@@ -5,12 +5,11 @@ namespace cacheflow {
5
6 template<typename scalar_t>
7 __global__ void rotary_embedding_neox_kernel(
8 - scalar_t* __restrict__ out_query, // [num_tokens, num_heads, head_size]
9 - scalar_t* __restrict__ out_key, // [num_tokens, num_heads, head_size]
8 const int64_t* __restrict__ positions, // [num_tokens]
11 - const scalar_t* __restrict__ query, // [num_tokens, num_heads, head_size]
12 - const scalar_t* __restrict__ key, // [num_tokens, num_heads, head_size]
9 + scalar_t* __restrict__ query, // [num_tokens, num_heads, head_size]
10 + scalar_t* __restrict__ key, // [num_tokens, num_heads, head_size]
11 const scalar_t* __restrict__ cos_sin_cache, // [max_position, 2, head_size // 2]
12 + const int stride,
13 const int num_heads,
14 const int head_size) {
15 // Each thread block is responsible for one token.
@@ -19,41 +18,36 @@ __global__ void rotary_embedding_neox_kernel(
18 const scalar_t* cache_ptr = cos_sin_cache + pos * head_size;
19
20 const int embed_dim = head_size / 2;
22 - const int n = num_heads * head_size;
21 + const int n = num_heads * embed_dim;
22 for (int i = threadIdx.x; i < n; i += blockDim.x) {
24 - const int idx = token_idx * n + i;
23 + const int head_idx = i / embed_dim;
24 + const int token_head = token_idx * stride + head_idx * head_size;
25
26 - const int head_idx = i / head_size;
27 - const int head_offset = i % head_size;
28 - const int token_head = token_idx * n + head_idx * head_size;
29 -
30 - const bool is_first_half = head_offset < embed_dim;
31 - const int rot_offset = head_offset % embed_dim;
26 + const int rot_offset = i % embed_dim;
27 const int x_index = rot_offset;
28 const int y_index = embed_dim + rot_offset;
29
30 + const int out_x = token_idx * stride + head_idx * head_size + x_index;
31 + const int out_y = token_idx * stride + head_idx * head_size + y_index;
32 +
33 const scalar_t cos = __ldg(cache_ptr + x_index);
34 const scalar_t sin = __ldg(cache_ptr + y_index);
35
38 - const scalar_t q_x = __ldg(query + token_head + x_index);
39 - const scalar_t q_y = __ldg(query + token_head + y_index);
40 - const scalar_t q_cos = is_first_half ? q_x : q_y;
41 - const scalar_t q_sin = is_first_half ? -q_y : q_x;
42 - out_query[idx] = q_cos * cos + q_sin * sin;
36 + const scalar_t q_x = query[token_head + x_index];
37 + const scalar_t q_y = query[token_head + y_index];
38 + query[out_x] = q_x * cos - q_y * sin;
39 + query[out_y] = q_y * cos + q_x * sin;
40
44 - const scalar_t k_x = __ldg(key + token_head + x_index);
45 - const scalar_t k_y = __ldg(key + token_head + y_index);
46 - const scalar_t k_cos = is_first_half ? k_x : k_y;
47 - const scalar_t k_sin = is_first_half ? -k_y : k_x;
48 - out_key[idx] = k_cos * cos + k_sin * sin;
41 + const scalar_t k_x = key[token_head + x_index];
42 + const scalar_t k_y = key[token_head + y_index];
43 + key[out_x] = k_x * cos - k_y * sin;
44 + key[out_y] = k_y * cos + k_x * sin;
45 }
46 }
47
48 } // namespace cacheflow
49
50 void rotary_embedding_neox(
55 - torch::Tensor& out_query, // [num_tokens, num_heads * head_size]
56 - torch::Tensor& out_key, // [num_tokens, num_heads * head_size]
51 torch::Tensor& positions, // [num_tokens]
52 torch::Tensor& query, // [num_tokens, num_heads * head_size]
53 torch::Tensor& key, // [num_tokens, num_heads * head_size]
@@ -62,21 +56,22 @@ void rotary_embedding_neox(
56 int num_tokens = query.size(0);
57 int head_size = cos_sin_cache.size(1);
58 int num_heads = query.size(1) / head_size;
59 + int stride = query.stride(0);
60 + TORCH_CHECK(stride == key.stride(0));
61
62 dim3 grid(num_tokens);
67 - dim3 block(std::min(num_heads * head_size, 512));
63 + dim3 block(std::min(num_heads * head_size / 2, 512));
64 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
65 AT_DISPATCH_FLOATING_TYPES_AND_HALF(
66 query.scalar_type(),
67 "rotary_embedding_neox",
68 [&] {
69 cacheflow::rotary_embedding_neox_kernel<scalar_t><<<grid, block, 0, stream>>>(
74 - out_query.data_ptr<scalar_t>(),
75 - out_key.data_ptr<scalar_t>(),
70 positions.data_ptr<int64_t>(),
71 query.data_ptr<scalar_t>(),
72 key.data_ptr<scalar_t>(),
73 cos_sin_cache.data_ptr<scalar_t>(),
74 + stride,
75 num_heads,
76 head_size);
77 });
setup.py
+7
@@ -39,6 +39,13 @@ layernorm_extension = cpp_extension.CUDAExtension(
39 )
40 ext_modules.append(layernorm_extension)
41
42 +activation_extension = cpp_extension.CUDAExtension(
43 + name='cacheflow.activation_ops',
44 + sources=['csrc/activation.cpp', 'csrc/activation_kernels.cu'],
45 + extra_compile_args={'cxx': CXX_FLAGS, 'nvcc': NVCC_FLAGS},
46 +)
47 +ext_modules.append(activation_extension)
48 +
49 setuptools.setup(
50 name='cacheflow',
51 ext_modules=ext_modules,
tests/kernels/activation.py new
+30
@@ -0,0 +1,30 @@
1 +import torch
2 +import torch.nn.functional as F
3 +
4 +from cacheflow import activation_ops
5 +
6 +
7 +def ref_silu_and_mul(x: torch.Tensor) -> torch.Tensor:
8 + x1, x2 = x.chunk(chunks=2, dim=1)
9 + return F.silu(x1) * x2
10 +
11 +
12 +@torch.inference_mode()
13 +def test_silu_and_mul(
14 + num_tokens: int,
15 + d: int,
16 + dtype: torch.dtype,
17 +) -> None:
18 + x = torch.randn(num_tokens, 2 * d, dtype=dtype, device='cuda')
19 + out = torch.empty(num_tokens, d, dtype=dtype, device='cuda')
20 + activation_ops.silu_and_mul(out, x)
21 + ref_out = ref_silu_and_mul(x)
22 + assert torch.allclose(out, ref_out, atol=1e-5, rtol=1e-5)
23 +
24 +
25 +if __name__ == '__main__':
26 + for dtype in [torch.half, torch.float]:
27 + for num_tokens in [7, 83, 2048]:
28 + for d in [512, 4096, 13824]:
29 + print(f'Testing dtype={dtype}, num_tokens={num_tokens}, d={d}')
30 + test_silu_and_mul(num_tokens, d, dtype)
tests/kernels/attention.py
+31 -15
@@ -1,7 +1,7 @@
1 import random
2 from typing import List, Optional
3
4 -from flash_attn.flash_attention import FlashAttention
4 +from flash_attn.flash_attn_interface import _flash_attn_forward
5 import torch
6
7 from cacheflow import attention_ops
@@ -105,8 +105,9 @@ def test_single_query_cached_kv_attention(
105 num_blocks: int,
106 dtype: torch.dtype,
107 ) -> None:
108 - query = torch.randn(
109 - num_tokens, num_heads, head_size, dtype=dtype, device='cuda')
108 + qkv = torch.randn(
109 + num_tokens, 3, num_heads, head_size, dtype=dtype, device='cuda')
110 + query, _, _ = qkv.unbind(dim=1)
111 x = 16 // torch.tensor([], dtype=dtype).element_size()
112 key_block_shape = (num_heads, head_size // x, block_size, x)
113 key_cache = torch.randn(
@@ -115,6 +116,11 @@ def test_single_query_cached_kv_attention(
116 value_cache = torch.randn(
117 size=(num_blocks, *value_block_shape), dtype=dtype, device='cuda')
118
119 + # Adjust the range of the values to reduce precision errors.
120 + query = query / (head_size ** 0.5)
121 + key_cache = key_cache / (head_size ** 0.5)
122 + value_cache = value_cache / (head_size ** 0.5)
123 +
124 context_lens = [random.randint(1, MAX_SEQ_LEN) for _ in range(num_tokens)]
125 max_context_len = max(context_lens)
126 context_lens = torch.tensor(context_lens, dtype=torch.int, device='cuda')
@@ -130,7 +136,8 @@ def test_single_query_cached_kv_attention(
136 block_tables = torch.tensor(block_tables, dtype=torch.int, device='cuda')
137
138 scale = float(1.0 / (head_size ** 0.5))
133 - output = torch.empty_like(query)
139 + output = torch.empty(
140 + num_tokens, num_heads, head_size, dtype=dtype, device='cuda')
141 attention_ops.single_query_cached_kv_attention(
142 output,
143 query,
@@ -175,19 +182,28 @@ def test_multi_query_kv_attention(
182 cu_seq_lens = torch.tensor(cu_seq_lens, dtype=torch.int, device='cuda')
183
184 scale = float(1.0 / (head_size ** 0.5))
178 - query = torch.randn(
185 + qkv = torch.randn(
186 + num_tokens, 3, num_heads, head_size, dtype=dtype, device='cuda')
187 + # Adjust the range of the values to reduce precision errors.
188 + qkv = qkv / (head_size ** 0.5)
189 +
190 + query, key, value = qkv.unbind(dim=1)
191 + output = torch.empty(
192 num_tokens, num_heads, head_size, dtype=dtype, device='cuda')
180 - key = torch.rand_like(query)
181 - value = torch.rand_like(query)
182 -
183 - qkv = torch.stack([query, key, value], dim=1)
184 - flash_attn = FlashAttention(softmax_scale=scale)
185 - output = flash_attn(
186 - qkv,
187 - cu_seqlens=cu_seq_lens,
188 - max_s=max_seq_len,
193 + _flash_attn_forward(
194 + query,
195 + key,
196 + value,
197 + output,
198 + cu_seq_lens,
199 + cu_seq_lens,
200 + max_seq_len,
201 + max_seq_len,
202 + dropout_p=0.0,
203 + softmax_scale=scale,
204 causal=True,
190 - )[0]
205 + return_softmax=False,
206 + )
207
208 cu_seq_lens = cu_seq_lens.cpu().tolist()
209 ref_output = ref_multi_query_kv_attention(
tests/kernels/cache.py
+5 -5
@@ -17,10 +17,10 @@ def test_reshape_and_cache(
17 slot_mapping = random.sample(range(num_slots), num_tokens)
18 slot_mapping = torch.tensor(slot_mapping, dtype=torch.int, device='cuda')
19
20 - kv_shape = (num_tokens, num_heads, head_size)
21 - key = torch.randn(size=kv_shape, dtype=dtype, device='cuda')
22 - value = torch.randn(size=kv_shape, dtype=dtype, device='cuda')
23 -
20 + qkv = torch.randn(
21 + num_tokens, 3, num_heads, head_size, dtype=dtype, device='cuda')
22 + _, key, value = qkv.unbind(dim=1)
23 +
24 x = 16 // torch.tensor([], dtype=dtype).element_size()
25 key_cache_shape = (num_blocks, num_heads, head_size // x, block_size, x)
26 key_cache = torch.randn(size=key_cache_shape, dtype=dtype, device='cuda')
@@ -35,7 +35,7 @@ def test_reshape_and_cache(
35
36 for i in range(num_tokens):
37 reshaped_key = key.reshape(num_tokens, num_heads, head_size // x, x)
38 - block_idx = slot_mapping[i] // block_size
38 + block_idx = torch.div(slot_mapping[i], block_size, rounding_mode='floor')
39 block_offset = slot_mapping[i] % block_size
40 cloned_key_cache[block_idx, :, :, block_offset, :] = reshaped_key[i]
41 cloned_value_cache[block_idx, :, :, block_offset] = value[i]
tests/kernels/pos_encoding.py
+4 -6
@@ -85,15 +85,13 @@ def test_rotary_embedding_neox(
85 cos_sin_cache = torch.cat((cos, sin), dim=-1)
86 cos_sin_cache = cos_sin_cache.to(dtype=dtype, device='cuda')
87
88 - # Run the kernel.
89 - out_query = torch.empty_like(query)
90 - out_key = torch.empty_like(key)
88 + # Run the kernel. The kernel is in-place, so we need to clone the inputs.
89 + out_query = query.clone()
90 + out_key = key.clone()
91 pos_encoding_ops.rotary_embedding_neox(
92 + positions,
93 out_query,
94 out_key,
94 - positions,
95 - query,
96 - key,
95 cos_sin_cache,
96 )
97