@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 88c0268a18
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Implement custom kernel for LLaMA rotary embedding (#14)

Woosuk Kwon committed Mar 30, 2023 at 11:04 UTC 88c0268a18f1c724d59a624364635d5c7ac39408
10 files changed +318 -69
cacheflow/models/attention.py
+71 -2
@@ -6,13 +6,14 @@ import torch.nn as nn
6
7 from cacheflow import attention_ops
8 from cacheflow import cache_ops
9 +from cacheflow import pos_encoding_ops
10 from cacheflow.models import InputMetadata
11
12
12 -class OPTCacheFlowAttention(nn.Module):
13 +class GPTCacheFlowAttention(nn.Module):
14
15 def __init__(self, scale: float) -> None:
15 - super(OPTCacheFlowAttention, self).__init__()
16 + super().__init__()
17 self.scale = float(scale)
18
19 self.flash_attn = FlashAttention(softmax_scale=self.scale)
@@ -136,3 +137,71 @@ class OPTCacheFlowAttention(nn.Module):
137 # Reshape the output tensor.
138 # NOTE(woosuk): The output tensor may include paddings.
139 return output.view(-1, num_heads * head_size)
140 +
141 +
142 +class OPTCacheFlowAttention(GPTCacheFlowAttention):
143 + """OPT uses the same attention mechanism as GPT."""
144 +
145 + def __init__(self, scale: float) -> None:
146 + super().__init__(scale)
147 +
148 +
149 +class LlamaCacheFlowAttention(GPTCacheFlowAttention):
150 + """Llama uses GPT-NeoX style rotary embedding."""
151 +
152 + def __init__(
153 + self,
154 + scale: float,
155 + head_size: int,
156 + max_position: int = 8192,
157 + base: int = 10000,
158 + ) -> None:
159 + super().__init__(scale)
160 +
161 + # Create the cos and sin cache.
162 + inv_freq = 1.0 / (base ** (torch.arange(0, head_size, 2) / head_size))
163 + t = torch.arange(max_position).float()
164 + freqs = torch.einsum('i,j -> ij', t, inv_freq.float())
165 + cos = freqs.cos()
166 + sin = freqs.sin()
167 + cache = torch.cat((cos, sin), dim=-1)
168 +
169 + # FIXME(woosuk): This assumes that we configure the default dtype when
170 + # initializing the model. Make it more robust.
171 + torch_dtype = torch.get_default_dtype()
172 + cache = cache.to(torch_dtype)
173 + # Embedding size: [max_position, head_size]
174 + self.register_buffer('cos_sin_cache', cache, persistent=False)
175 +
176 + def forward(
177 + self,
178 + positions: torch.LongTensor, # [num_tokens]
179 + query: torch.Tensor, # [num_tokens, num_heads * head_size]
180 + key: torch.Tensor, # [num_tokens, num_heads * head_size]
181 + value: torch.Tensor, # [num_tokens, num_heads * head_size]
182 + key_cache: torch.Tensor, # [num_blocks, num_heads, head_size/x, block_size, x]
183 + value_cache: torch.Tensor, # [num_blocks, num_heads, head_size, block_size]
184 + input_metadata: InputMetadata,
185 + cache_event: Optional[torch.cuda.Event],
186 + ) -> torch.Tensor: # [num_tokens, num_heads * head_size]
187 + # Apply rotary embedding to the query and key before passing them
188 + # to the attention op.
189 + out_query = torch.empty_like(query)
190 + out_key = torch.empty_like(key)
191 + pos_encoding_ops.rotary_embedding_neox(
192 + out_query,
193 + out_key,
194 + positions,
195 + query,
196 + key,
197 + self.cos_sin_cache,
198 + )
199 + return super().forward(
200 + out_query,
201 + out_key,
202 + value,
203 + key_cache,
204 + value_cache,
205 + input_metadata,
206 + cache_event,
207 + )
cacheflow/models/llama.py
+5 -59
@@ -8,12 +8,10 @@ from typing import Dict, List, Optional, Tuple
8 import numpy as np
9 import torch
10 from torch import nn
11 -import torch.nn.functional as F
11 from transformers import LlamaConfig
13 -from transformers import PreTrainedModel
12
13 from cacheflow.models import InputMetadata
16 -from cacheflow.models.attention import OPTCacheFlowAttention
14 +from cacheflow.models.attention import LlamaCacheFlowAttention
15 from cacheflow.models.sample import Sampler
16 from cacheflow.parallel_utils.parallel_state import (
17 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
@@ -41,48 +39,8 @@ class LlamaRMSNorm(nn.Module):
39 return self.weight * hidden_states
40
41
44 -class LlamaRotaryEmbedding(torch.nn.Module):
45 -
46 - def __init__(self, dim, max_position_embeddings=2048, base=10000):
47 - super().__init__()
48 - self.max_position_embeddings = max_position_embeddings
49 -
50 - inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2) / dim))
51 - self.register_buffer("inv_freq", inv_freq)
52 -
53 - # Create cos and sin embeddings.
54 - t = torch.arange(max_position_embeddings).float()
55 - freqs = torch.einsum("i,j->ij", t, self.inv_freq.float())
56 - emb = torch.cat((freqs, freqs), dim=-1)
57 - cos = emb.cos().to(dtype=self.inv_freq.dtype)
58 - sin = emb.sin().to(dtype=self.inv_freq.dtype)
59 - self.register_buffer("cos_cached", cos, persistent=False)
60 - self.register_buffer("sin_cached", sin, persistent=False)
61 -
62 - def forward(
63 - self,
64 - positions: torch.LongTensor,
65 - ) -> Tuple[torch.Tensor, torch.Tensor]:
66 - cos = F.embedding(positions, self.cos_cached)
67 - sin = F.embedding(positions, self.sin_cached)
68 - return cos, sin
69 -
70 -
71 -def rotate_half(x):
72 - """Rotates half the hidden dims of the input."""
73 - x1 = x[..., : x.shape[-1] // 2]
74 - x2 = x[..., x.shape[-1] // 2 :]
75 - return torch.cat((-x2, x1), dim=-1)
76 -
77 -
78 -def apply_rotary_pos_emb(q, k, cos, sin):
79 - # TODO: Optimize.
80 - q_embed = (q * cos) + (rotate_half(q) * sin)
81 - k_embed = (k * cos) + (rotate_half(k) * sin)
82 - return q_embed, k_embed
83 -
84 -
42 class LlamaMLP(nn.Module):
43 +
44 def __init__(
45 self,
46 hidden_size: int,
@@ -156,9 +114,7 @@ class LlamaAttention(nn.Module):
114 input_is_parallel=True,
115 perform_initialization=False,
116 )
159 - self.rotary_emb = LlamaRotaryEmbedding(self.head_dim)
160 - # FIXME(woosuk): Rename this.
161 - self.attn = OPTCacheFlowAttention(scale=self.scaling)
117 + self.attn = LlamaCacheFlowAttention(self.scaling, self.head_dim)
118
119 def forward(
120 self,
@@ -171,19 +127,9 @@ class LlamaAttention(nn.Module):
127 q, _ = self.q_proj(hidden_states)
128 k, _ = self.k_proj(hidden_states)
129 v, _ = self.v_proj(hidden_states)
174 -
175 - # Apply rotrary embedding.
176 - # TODO: Optimize.
177 - q = q.view(-1, self.num_heads, self.head_dim).transpose(0, 1)
178 - k = k.view(-1, self.num_heads, self.head_dim).transpose(0, 1)
179 - cos, sin = self.rotary_emb(positions)
180 - q, k = apply_rotary_pos_emb(q, k, cos, sin)
181 - q = q.transpose(0, 1).contiguous().view(-1, self.num_heads * self.head_dim)
182 - k = k.transpose(0, 1).contiguous().view(-1, self.num_heads * self.head_dim)
183 -
184 - key_cache, value_cache = kv_cache
130 + k_cache, v_cache = kv_cache
131 attn_output = self.attn(
186 - q, k, v, key_cache, value_cache, input_metadata, cache_event)
132 + positions, q, k, v, k_cache, v_cache, input_metadata, cache_event)
133 output, _ = self.o_proj(attn_output)
134 return output
135
cacheflow/models/memory_analyzer.py
+1 -2
@@ -165,8 +165,7 @@ class LlamaMemoryAnalyzer(CacheFlowMemoryAnalyzer):
165 self.head_size = config.hidden_size // self.num_heads
166 self.ffn_size = config.intermediate_size
167 self.vocab_size = config.vocab_size
168 - # FIXME
169 - self.max_position = 2048
168 + self.max_position = 8192
169
170 def _get_param_size(self) -> int:
171 word_embedding = self.vocab_size * self.hidden_size // self.tensor_parallel_size
cacheflow/models/opt.py
+1 -2
@@ -51,7 +51,7 @@ class OPTAttention(nn.Module):
51 assert num_heads % tensor_model_parallel_world_size == 0
52 self.num_heads = total_num_heads // tensor_model_parallel_world_size
53 self.head_dim = embed_dim // total_num_heads
54 - self.scaling = self.head_dim**-0.5
54 + self.scaling = self.head_dim ** -0.5
55
56 # TODO(woosuk): Fuse the three linear layers into one QKV linear layer.
57 self.k_proj = ColumnParallelLinear(embed_dim, embed_dim, bias=bias,
@@ -66,7 +66,6 @@ class OPTAttention(nn.Module):
66 self.out_proj = RowParallelLinear(embed_dim, embed_dim, bias=bias,
67 input_is_parallel=True,
68 perform_initialization=False)
69 -
69 self.attn = OPTCacheFlowAttention(scale=self.scaling)
70
71 def forward(
cacheflow/models/sample.py
+1 -1
@@ -12,7 +12,7 @@ from cacheflow.parallel_utils.tensor_parallel import gather_from_tensor_model_pa
12 class Sampler(nn.Module):
13
14 def __init__(self) -> None:
15 - super(Sampler, self).__init__()
15 + super().__init__()
16
17 def forward(
18 self,
csrc/cache_kernels.cu
+3 -3
@@ -122,13 +122,13 @@ void reshape_and_cache(
122 torch::Tensor& value_cache,
123 torch::Tensor& slot_mapping) {
124 int num_tokens = key.size(0);
125 - int head_num = key.size(1);
125 + int num_heads = key.size(1);
126 int head_size = key.size(2);
127 int block_size = key_cache.size(3);
128 int x = key_cache.size(4);
129
130 dim3 grid(num_tokens);
131 - dim3 block(std::min(head_num * head_size, 512));
131 + dim3 block(std::min(num_heads * head_size, 512));
132 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
133 AT_DISPATCH_FLOATING_TYPES_AND_HALF(
134 key.scalar_type(),
@@ -140,7 +140,7 @@ void reshape_and_cache(
140 key_cache.data_ptr<scalar_t>(),
141 value_cache.data_ptr<scalar_t>(),
142 slot_mapping.data_ptr<int>(),
143 - head_num,
143 + num_heads,
144 head_size,
145 block_size,
146 x);
csrc/pos_encoding.cpp new
+16
@@ -0,0 +1,16 @@
1 +#include <torch/extension.h>
2 +
3 +void rotary_embedding_neox(
4 + torch::Tensor& out_query,
5 + torch::Tensor& out_key,
6 + torch::Tensor& positions,
7 + torch::Tensor& query,
8 + torch::Tensor& key,
9 + torch::Tensor& cos_sin_cache);
10 +
11 +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
12 + m.def(
13 + "rotary_embedding_neox",
14 + &rotary_embedding_neox,
15 + "Apply GPT-NeoX style rotary embedding to query and key");
16 +}
csrc/pos_encoding_kernels.cu new
+83
@@ -0,0 +1,83 @@
1 +#include <torch/extension.h>
2 +#include <ATen/cuda/CUDAContext.h>
3 +
4 +namespace cacheflow {
5 +
6 +template<typename scalar_t>
7 +__global__ void rotary_embedding_neox_kernel(
8 + scalar_t* __restrict__ out_query, // [num_tokens, num_heads, head_size]
9 + scalar_t* __restrict__ out_key, // [num_tokens, num_heads, head_size]
10 + const int64_t* __restrict__ positions, // [num_tokens]
11 + const scalar_t* __restrict__ query, // [num_tokens, num_heads, head_size]
12 + const scalar_t* __restrict__ key, // [num_tokens, num_heads, head_size]
13 + const scalar_t* __restrict__ cos_sin_cache, // [max_position, 2, head_size // 2]
14 + const int num_heads,
15 + const int head_size) {
16 + // Each thread block is responsible for one token.
17 + const int token_idx = blockIdx.x;
18 + int64_t pos = positions[token_idx];
19 + const scalar_t* cache_ptr = cos_sin_cache + pos * head_size;
20 +
21 + const int embed_dim = head_size / 2;
22 + const int n = num_heads * head_size;
23 + for (int i = threadIdx.x; i < n; i += blockDim.x) {
24 + const int idx = token_idx * n + i;
25 +
26 + const int head_idx = i / head_size;
27 + const int head_offset = i % head_size;
28 + const int token_head = token_idx * n + head_idx * head_size;
29 +
30 + const bool is_first_half = head_offset < embed_dim;
31 + const int rot_offset = head_offset % embed_dim;
32 + const int x_index = rot_offset;
33 + const int y_index = embed_dim + rot_offset;
34 +
35 + const scalar_t cos = __ldg(cache_ptr + x_index);
36 + const scalar_t sin = __ldg(cache_ptr + y_index);
37 +
38 + const scalar_t q_x = __ldg(query + token_head + x_index);
39 + const scalar_t q_y = __ldg(query + token_head + y_index);
40 + const scalar_t q_cos = is_first_half ? q_x : q_y;
41 + const scalar_t q_sin = is_first_half ? -q_y : q_x;
42 + out_query[idx] = q_cos * cos + q_sin * sin;
43 +
44 + const scalar_t k_x = __ldg(key + token_head + x_index);
45 + const scalar_t k_y = __ldg(key + token_head + y_index);
46 + const scalar_t k_cos = is_first_half ? k_x : k_y;
47 + const scalar_t k_sin = is_first_half ? -k_y : k_x;
48 + out_key[idx] = k_cos * cos + k_sin * sin;
49 + }
50 +}
51 +
52 +} // namespace cacheflow
53 +
54 +void rotary_embedding_neox(
55 + torch::Tensor& out_query, // [num_tokens, num_heads * head_size]
56 + torch::Tensor& out_key, // [num_tokens, num_heads * head_size]
57 + torch::Tensor& positions, // [num_tokens]
58 + torch::Tensor& query, // [num_tokens, num_heads * head_size]
59 + torch::Tensor& key, // [num_tokens, num_heads * head_size]
60 + torch::Tensor& cos_sin_cache) // [max_position, head_size]
61 +{
62 + int num_tokens = query.size(0);
63 + int head_size = cos_sin_cache.size(1);
64 + int num_heads = query.size(1) / head_size;
65 +
66 + dim3 grid(num_tokens);
67 + dim3 block(std::min(num_heads * head_size, 512));
68 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
69 + AT_DISPATCH_FLOATING_TYPES_AND_HALF(
70 + query.scalar_type(),
71 + "rotary_embedding_neox",
72 + [&] {
73 + cacheflow::rotary_embedding_neox_kernel<scalar_t><<<grid, block, 0, stream>>>(
74 + out_query.data_ptr<scalar_t>(),
75 + out_key.data_ptr<scalar_t>(),
76 + positions.data_ptr<int64_t>(),
77 + query.data_ptr<scalar_t>(),
78 + key.data_ptr<scalar_t>(),
79 + cos_sin_cache.data_ptr<scalar_t>(),
80 + num_heads,
81 + head_size);
82 + });
83 +}
setup.py
+8
@@ -23,6 +23,14 @@ attention_extension = cpp_extension.CUDAExtension(
23 )
24 ext_modules.append(attention_extension)
25
26 +# Positional encodings.
27 +positional_encoding_extension = cpp_extension.CUDAExtension(
28 + name='cacheflow.pos_encoding_ops',
29 + sources=['csrc/pos_encoding.cpp', 'csrc/pos_encoding_kernels.cu'],
30 + extra_compile_args={'cxx': CXX_FLAGS, 'nvcc': NVCC_FLAGS},
31 +)
32 +ext_modules.append(positional_encoding_extension)
33 +
34 setuptools.setup(
35 name='cacheflow',
36 ext_modules=ext_modules,
tests/kernels/pos_encoding.py new
+129
@@ -0,0 +1,129 @@
1 +from typing import Tuple
2 +
3 +import torch
4 +import torch.nn as nn
5 +import torch.nn.functional as F
6 +
7 +from cacheflow import pos_encoding_ops
8 +
9 +
10 +def rotate_half(x: torch.Tensor) -> torch.Tensor:
11 + x1 = x[..., : x.shape[-1] // 2]
12 + x2 = x[..., x.shape[-1] // 2 :]
13 + return torch.cat((-x2, x1), dim=-1)
14 +
15 +
16 +def apply_rotary_pos_emb(
17 + q: torch.Tensor,
18 + k: torch.Tensor,
19 + cos: torch.Tensor,
20 + sin: torch.Tensor,
21 +) -> Tuple[torch.Tensor, torch.Tensor]:
22 + q_embed = (q * cos) + (rotate_half(q) * sin)
23 + k_embed = (k * cos) + (rotate_half(k) * sin)
24 + return q_embed, k_embed
25 +
26 +
27 +class RefRotaryEmbeddingNeox(nn.Module):
28 + """Reference implementation of the GPT-NeoX style rotary embedding."""
29 +
30 + def __init__(
31 + self,
32 + dim: int,
33 + max_position_embeddings: int = 2048,
34 + base: int = 10000,
35 + ) -> None:
36 + super().__init__()
37 + self.max_position_embeddings = max_position_embeddings
38 +
39 + # Create cos and sin embeddings.
40 + inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2) / dim))
41 + t = torch.arange(max_position_embeddings).float()
42 + freqs = torch.einsum("i,j->ij", t, inv_freq.float())
43 + emb = torch.cat((freqs, freqs), dim=-1)
44 + cos = emb.cos().to(dtype=inv_freq.dtype)
45 + sin = emb.sin().to(dtype=inv_freq.dtype)
46 + self.register_buffer("cos_cached", cos, persistent=False)
47 + self.register_buffer("sin_cached", sin, persistent=False)
48 +
49 + def forward(
50 + self,
51 + positions: torch.LongTensor, # [num_tokens]
52 + query: torch.Tensor, # [num_tokens, num_heads, head_size]
53 + key: torch.Tensor, # [num_tokens, num_heads, head_size]
54 + ) -> Tuple[torch.Tensor, torch.Tensor]:
55 + cos = F.embedding(positions, self.cos_cached)
56 + sin = F.embedding(positions, self.sin_cached)
57 + query = query.transpose(0, 1)
58 + key = key.transpose(0, 1)
59 + query, key = apply_rotary_pos_emb(query, key, cos, sin)
60 + query = query.transpose(0, 1).contiguous()
61 + key = key.transpose(0, 1).contiguous()
62 + # Output query/key shape: [num_tokens, num_tokens, head_size]
63 + return query, key
64 +
65 +
66 +@torch.inference_mode()
67 +def test_rotary_embedding_neox(
68 + num_tokens: int,
69 + num_heads: int,
70 + head_size: int,
71 + max_position: int,
72 + dtype: torch.dtype,
73 + base: int = 10000,
74 +) -> None:
75 + positions = torch.randint(0, max_position, (num_tokens,), device='cuda')
76 + query = torch.randn(num_tokens, num_heads * head_size, dtype=dtype, device='cuda')
77 + key = torch.randn(num_tokens, num_heads * head_size, dtype=dtype, device='cuda')
78 +
79 + # Create the rotary embedding.
80 + inv_freq = 1.0 / (base ** (torch.arange(0, head_size, 2) / head_size))
81 + t = torch.arange(max_position).float()
82 + freqs = torch.einsum('i,j -> ij', t, inv_freq.float())
83 + cos = freqs.cos()
84 + sin = freqs.sin()
85 + cos_sin_cache = torch.cat((cos, sin), dim=-1)
86 + cos_sin_cache = cos_sin_cache.to(dtype=dtype, device='cuda')
87 +
88 + # Run the kernel.
89 + out_query = torch.empty_like(query)
90 + out_key = torch.empty_like(key)
91 + pos_encoding_ops.rotary_embedding_neox(
92 + out_query,
93 + out_key,
94 + positions,
95 + query,
96 + key,
97 + cos_sin_cache,
98 + )
99 +
100 + # Run the reference implementation.
101 + ref_rotary_embedding = RefRotaryEmbeddingNeox(
102 + dim=head_size,
103 + max_position_embeddings=max_position,
104 + base=base,
105 + ).to(dtype=dtype, device='cuda')
106 + ref_query, ref_key = ref_rotary_embedding(
107 + positions,
108 + query.view(num_tokens, num_heads, head_size),
109 + key.view(num_tokens, num_heads, head_size),
110 + )
111 + ref_query = ref_query.view(num_tokens, num_heads * head_size)
112 + ref_key = ref_key.view(num_tokens, num_heads * head_size)
113 +
114 + # Compare the results.
115 + assert torch.allclose(out_query, ref_query, atol=1e-3, rtol=1e-5)
116 + assert torch.allclose(out_key, ref_key, atol=1e-3, rtol=1e-5)
117 +
118 +
119 +if __name__ == '__main__':
120 + for dtype in [torch.half, torch.float]:
121 + for head_size in [32, 64, 80, 96, 128, 160, 192, 256]:
122 + print(f'Running tests for head_size={head_size} and dtype={dtype}')
123 + test_rotary_embedding_neox(
124 + num_tokens=2145,
125 + num_heads=5,
126 + head_size=head_size,
127 + max_position=8192,
128 + dtype=dtype,
129 + )