@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e548c1488a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Add support for GPT-2 (#60)

Woosuk Kwon committed May 4, 2023 at 02:59 UTC e548c1488ae1e93b5ca60359f4b292d2bee0b60d
7 files changed +350 -8
cacheflow/models/gpt2.py new
+265
@@ -0,0 +1,265 @@
1 +"""1D GPT-2 model compatible with HuggingFace weights."""
2 +from typing import Dict, List, Optional, Tuple
3 +
4 +import torch
5 +from torch import nn
6 +from transformers import GPT2Config
7 +
8 +from cacheflow.models import InputMetadata
9 +from cacheflow.models.attention import GPTCacheFlowAttention
10 +from cacheflow.models.sample import Sampler
11 +from cacheflow.models.utils import (hf_model_weights_iterator,
12 + load_tensor_parallel_weights)
13 +from cacheflow.parallel_utils.parallel_state import (
14 + get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
15 +from cacheflow.parallel_utils.tensor_parallel import (VocabParallelEmbedding,
16 + ColumnParallelLinear,
17 + RowParallelLinear)
18 +from cacheflow.sequence import SequenceOutputs
19 +
20 +KVCache = Tuple[torch.Tensor, torch.Tensor]
21 +
22 +
23 +class GPT2Attention(nn.Module):
24 +
25 + def __init__(self, config: GPT2Config):
26 + super().__init__()
27 + self.hidden_size = config.hidden_size
28 + total_num_heads = config.num_attention_heads
29 + tensor_model_parallel_world_size = get_tensor_model_parallel_world_size()
30 + assert total_num_heads % tensor_model_parallel_world_size == 0
31 + self.num_heads = total_num_heads // tensor_model_parallel_world_size
32 + self.head_dim = self.hidden_size // total_num_heads
33 + self.scale = self.head_dim ** -0.5
34 +
35 + self.c_attn = ColumnParallelLinear(self.hidden_size, 3 * self.hidden_size, bias=True,
36 + gather_output=False,
37 + perform_initialization=False)
38 + self.c_proj = RowParallelLinear(self.hidden_size, self.hidden_size, bias=True,
39 + input_is_parallel=True,
40 + perform_initialization=False)
41 + self.attn = GPTCacheFlowAttention(scale=self.scale)
42 +
43 + def forward(
44 + self,
45 + hidden_states: torch.Tensor,
46 + kv_cache: KVCache,
47 + input_metadata: InputMetadata,
48 + cache_event: Optional[torch.cuda.Event],
49 + ) -> torch.Tensor:
50 + qkv, _ = self.c_attn(hidden_states)
51 + q, k, v = qkv.chunk(chunks=3, dim=-1)
52 + key_cache, value_cache = kv_cache
53 + attn_output = self.attn(
54 + q, k, v, key_cache, value_cache, input_metadata, cache_event)
55 + attn_output, _ = self.c_proj(attn_output)
56 + return attn_output
57 +
58 +
59 +class GPT2MLP(nn.Module):
60 +
61 + def __init__(
62 + self,
63 + intermediate_size: int,
64 + config: GPT2Config,
65 + ):
66 + super().__init__()
67 + hidden_size = config.hidden_size
68 + self.c_fc = ColumnParallelLinear(hidden_size, intermediate_size,
69 + bias=True, gather_output=False,
70 + perform_initialization=False)
71 + self.c_proj = RowParallelLinear(intermediate_size, hidden_size,
72 + bias=True, input_is_parallel=True,
73 + perform_initialization=False)
74 +
75 + act_fn = config.activation_function
76 + if act_fn != "gelu_new":
77 + raise ValueError(f"Unsupported activation: {act_fn}. "
78 + "GPT-2 only supports gelu_new for now.")
79 + self.act = torch.nn.GELU(approximate="tanh")
80 +
81 + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
82 + hidden_states, _ = self.c_fc(hidden_states)
83 + hidden_states = self.act(hidden_states)
84 + hidden_states, _ = self.c_proj(hidden_states)
85 + return hidden_states
86 +
87 +
88 +class GPT2Block(nn.Module):
89 +
90 + def __init__(self, config: GPT2Config):
91 + super().__init__()
92 + hidden_size = config.hidden_size
93 + inner_dim = config.n_inner if config.n_inner is not None else 4 * hidden_size
94 +
95 + self.ln_1 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
96 + self.attn = GPT2Attention(config)
97 + self.ln_2 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
98 + self.mlp = GPT2MLP(inner_dim, config)
99 +
100 + def forward(
101 + self,
102 + hidden_states: torch.Tensor,
103 + kv_cache: KVCache,
104 + input_metadata: InputMetadata,
105 + cache_event: Optional[torch.cuda.Event],
106 + ) -> torch.Tensor:
107 + residual = hidden_states
108 + hidden_states = self.ln_1(hidden_states)
109 + attn_output = self.attn(
110 + hidden_states=hidden_states,
111 + kv_cache=kv_cache,
112 + input_metadata=input_metadata,
113 + cache_event=cache_event,
114 + )
115 + # residual connection
116 + hidden_states = attn_output + residual
117 +
118 + residual = hidden_states
119 + hidden_states = self.ln_2(hidden_states)
120 + feed_forward_hidden_states = self.mlp(hidden_states)
121 + # residual connection
122 + hidden_states = residual + feed_forward_hidden_states
123 + return hidden_states
124 +
125 +
126 +class GPT2Model(nn.Module):
127 +
128 + def __init__(self, config: GPT2Config):
129 + super().__init__()
130 + self.config = config
131 + assert config.add_cross_attention == False
132 + assert config.scale_attn_by_inverse_layer_idx == False
133 + assert config.reorder_and_upcast_attn == False
134 + self.embed_dim = config.hidden_size
135 +
136 + # Optimization: While the vocab size of GPT-2 is 50257, we extend it
137 + # to 50304 in order to make it divisible by 64.
138 + # This improves performance since GPUs are faster if the dimension
139 + # is divisible by 64. In addition, it allows us to shard the embedding
140 + # layer across 2, 4, 8, or more GPUs.
141 + vocab_size = ((config.vocab_size + 63) // 64) * 64
142 + self.wte = VocabParallelEmbedding(vocab_size, self.embed_dim)
143 + self.wpe = nn.Embedding(config.max_position_embeddings, self.embed_dim)
144 + self.h = nn.ModuleList(
145 + [GPT2Block(config) for _ in range(config.num_hidden_layers)])
146 + self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
147 +
148 + def forward(
149 + self,
150 + input_ids: torch.LongTensor,
151 + position_ids: torch.LongTensor,
152 + kv_caches: List[KVCache],
153 + input_metadata: InputMetadata,
154 + cache_events: Optional[List[torch.cuda.Event]],
155 + ) -> torch.Tensor:
156 + inputs_embeds = self.wte(input_ids)
157 + position_embeds = self.wpe(position_ids)
158 + hidden_states = inputs_embeds + position_embeds
159 +
160 + for i in range(len(self.h)):
161 + if cache_events is None:
162 + cache_event = None
163 + else:
164 + cache_event = cache_events[i]
165 + layer = self.h[i]
166 + hidden_states = layer(
167 + hidden_states, kv_caches[i], input_metadata, cache_event)
168 +
169 + hidden_states = self.ln_f(hidden_states)
170 + return hidden_states
171 +
172 +
173 +class GPT2LMHeadModel(nn.Module):
174 +
175 + def __init__(self, config: GPT2Config):
176 + super().__init__()
177 + self.config = config
178 + self.transformer = GPT2Model(config)
179 + # TODO(zhuohan): create a new weight after implementing pipeline
180 + # parallelism
181 + self.lm_head_weight = self.transformer.wte.weight
182 + self.sampler = Sampler(config.vocab_size)
183 +
184 + def forward(
185 + self,
186 + input_ids: torch.LongTensor,
187 + positions: torch.LongTensor,
188 + kv_caches: List[KVCache],
189 + input_metadata: InputMetadata,
190 + cache_events: Optional[List[torch.cuda.Event]],
191 + ) -> Dict[int, SequenceOutputs]:
192 + hidden_states = self.transformer(
193 + input_ids, positions, kv_caches, input_metadata, cache_events)
194 + next_tokens = self.sampler(
195 + self.lm_head_weight, hidden_states, input_metadata)
196 + return next_tokens
197 +
198 + _column_parallel_weights = ["wte.weight", "c_fc.weight", "c_fc.bias"]
199 + _row_parallel_weights = ["c_proj.weight"]
200 +
201 + def load_weights(self, model_name_or_path: str,
202 + cache_dir: Optional[str] = None,
203 + use_np_cache: bool = False):
204 + tensor_model_parallel_world_size = get_tensor_model_parallel_world_size()
205 + tensor_model_parallel_rank = get_tensor_model_parallel_rank()
206 + state_dict = self.state_dict()
207 +
208 + for name, loaded_weight in hf_model_weights_iterator(
209 + model_name_or_path, cache_dir, use_np_cache):
210 + if "lm_head.weight" in name:
211 + # GPT-2 ties the weights of the embedding layer and the final
212 + # linear layer.
213 + continue
214 + if ".attn.bias" in name:
215 + # Skip attention mask.
216 + # NOTE: "c_attn.bias" should not be skipped.
217 + continue
218 + name = "transformer." + name
219 +
220 + # The HF's GPT-2 implementation uses Conv1D instead of Linear.
221 + # Because of this, we need to transpose the weights.
222 + for conv1d_weight_name in ["c_attn", "c_proj", "c_fc"]:
223 + if conv1d_weight_name not in name:
224 + continue
225 + if not name.endswith(".weight"):
226 + continue
227 + loaded_weight = loaded_weight.t()
228 + param = state_dict[name]
229 +
230 + if name == "transformer.wte.weight":
231 + # Consider padding in the vocab size.
232 + padded_vocab_size = param.shape[0] * tensor_model_parallel_world_size
233 + num_extra_rows = padded_vocab_size - self.config.vocab_size
234 + extra_rows = torch.empty(num_extra_rows, loaded_weight.shape[1])
235 + extra_rows = extra_rows.to(loaded_weight)
236 + loaded_weight = torch.cat([loaded_weight, extra_rows], dim=0)
237 +
238 + # For the fused QKV linear layer, manually shard the weights.
239 + if "c_attn" in name:
240 + # GPT-2's fused QKV has the shape of [3 * num_heads * head_size, hidden_size].
241 + # When tensor parallelism is used, we shard the weights along the head dimension.
242 + total_num_heads = self.config.num_attention_heads
243 + hidden_size = self.config.hidden_size
244 + head_size = hidden_size // total_num_heads
245 + num_heads = total_num_heads // tensor_model_parallel_world_size
246 + head_start = tensor_model_parallel_rank * num_heads
247 + head_end = (tensor_model_parallel_rank + 1) * num_heads
248 +
249 + if name.endswith(".weight"):
250 + loaded_weight = loaded_weight.view(3, total_num_heads, head_size, hidden_size)
251 + loaded_weight = loaded_weight[:, head_start:head_end, :, :]
252 + loaded_weight = loaded_weight.reshape(-1, hidden_size)
253 + elif name.endswith(".bias"):
254 + loaded_weight = loaded_weight.view(3, total_num_heads, head_size)
255 + loaded_weight = loaded_weight[:, head_start:head_end, :]
256 + loaded_weight = loaded_weight.reshape(-1)
257 + else:
258 + raise ValueError(f"Unexpected parameter name {name}")
259 + load_tensor_parallel_weights(param, loaded_weight, name,
260 + self._column_parallel_weights,
261 + self._row_parallel_weights)
262 +
263 + def initialize_dummy_weights(self) -> None:
264 + for param in self.state_dict().values():
265 + param.data.uniform_(-1e-3, 1e-3)
cacheflow/models/gpt_neox.py
+5 -5
@@ -173,7 +173,7 @@ class GPTNeoXForCausalLM(nn.Module):
173 self.embed_out = ColumnParallelLinear(config.hidden_size, config.vocab_size,
174 bias=False, gather_output=False,
175 perform_initialization=False)
176 - self.sampler = Sampler()
176 + self.sampler = Sampler(config.vocab_size)
177
178 def forward(
179 self,
@@ -205,8 +205,8 @@ class GPTNeoXForCausalLM(nn.Module):
205 param = state_dict[name]
206 if "query_key_value" in name:
207 # NOTE(woosuk): GPT-NeoX's fused QKV has the shape of
208 - # [num_heads * 3 * head_size, num_heads * head_size], while the
209 - # required shape is [3 * num_heads * head_size, num_heads * head_size].
208 + # [num_heads * 3 * head_size, hidden_size], while the
209 + # required shape is [3 * num_heads * head_size, hidden_size].
210 # Thus, we need weight conversion.
211 shard_size = param.shape[0]
212 loaded_weight = loaded_weight[shard_size * tensor_model_parallel_rank
@@ -218,11 +218,11 @@ class GPTNeoXForCausalLM(nn.Module):
218 if 'query_key_value.weight' in name:
219 loaded_weight = loaded_weight.view(-1, 3, head_size, hidden_size)
220 loaded_weight = loaded_weight.transpose(0, 1)
221 - loaded_weight = loaded_weight.reshape(-1, hidden_size).contiguous()
221 + loaded_weight = loaded_weight.reshape(-1, hidden_size)
222 elif 'query_key_value.bias' in name:
223 loaded_weight = loaded_weight.view(-1, 3, head_size)
224 loaded_weight = loaded_weight.transpose(0, 1)
225 - loaded_weight = loaded_weight.reshape(-1).contiguous()
225 + loaded_weight = loaded_weight.reshape(-1)
226 else:
227 raise ValueError(f"Unexpected weight name: {name}")
228 load_tensor_parallel_weights(param, loaded_weight, name,
cacheflow/models/llama.py
+1 -1
@@ -192,7 +192,7 @@ class LlamaForCausalLM(nn.Module):
192 bias=False,
193 gather_output=False,
194 perform_initialization=False)
195 - self.sampler = Sampler()
195 + self.sampler = Sampler(config.vocab_size)
196
197 def forward(
198 self,
cacheflow/models/memory_analyzer.py
+70
@@ -72,6 +72,76 @@ class CacheFlowMemoryAnalyzer:
72 return max_num_blocks
73
74
75 +class GPT2MemoryAnalyzer(CacheFlowMemoryAnalyzer):
76 +
77 + def __init__(
78 + self,
79 + model_name: str,
80 + block_size: int,
81 + dtype: torch.dtype,
82 + gpu_memory: int,
83 + cpu_memory: int,
84 + tensor_parallel_size: int,
85 + ) -> None:
86 + self.model_name = model_name
87 + self.block_size = block_size
88 + self.dtype = dtype
89 + self.gpu_memory = gpu_memory
90 + self.cpu_memory = cpu_memory
91 + self.tensor_parallel_size = tensor_parallel_size
92 +
93 + config = AutoConfig.from_pretrained(model_name)
94 + self.num_layers = config.num_hidden_layers
95 + self.hidden_size = config.hidden_size
96 + self.num_heads = config.num_attention_heads
97 + self.head_size = config.hidden_size // self.num_heads
98 + self.ffn_size = config.n_inner if config.n_inner is not None else 4 * self.hidden_size
99 + self.vocab_size = config.vocab_size
100 + self.max_position = config.max_position_embeddings
101 +
102 + def get_param_size(self) -> int:
103 + word_embedding = self.vocab_size * self.hidden_size // self.tensor_parallel_size
104 + position_embedding = self.max_position * self.hidden_size
105 +
106 + ln1 = 2 * self.hidden_size
107 + q = self.hidden_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
108 + k = self.hidden_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
109 + v = self.hidden_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
110 + out = self.hidden_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
111 + mha = ln1 + q + k + v + out
112 +
113 + ln2 = 2 * self.hidden_size
114 + ffn1 = self.hidden_size * self.ffn_size // self.tensor_parallel_size + self.ffn_size
115 + ffn2 = self.ffn_size * self.hidden_size // self.tensor_parallel_size + self.hidden_size
116 + ffn = ln2 + ffn1 + ffn2
117 +
118 + total = (word_embedding + position_embedding +
119 + self.num_layers * (mha + ffn))
120 + dtype_size = get_dtype_size(self.dtype)
121 + return dtype_size * total
122 +
123 + def get_max_act_size(
124 + self,
125 + max_num_batched_tokens: int,
126 + ) -> int:
127 + # NOTE: We approxmiately calculate the maximum activation size by
128 + # estimating
129 + # 1) the maximum activation tensor size during inference
130 + # 2) the residual tensor size during inference
131 + # Here, we assume that FlashAttention is used and
132 + # thus the attention maps are never materialized in GPU DRAM.
133 + residual = max_num_batched_tokens * self.hidden_size
134 + qkv = 3 * (max_num_batched_tokens * self.hidden_size) // self.tensor_parallel_size
135 + ffn = max_num_batched_tokens * self.ffn_size // self.tensor_parallel_size
136 + # Double the activation size for input and output.
137 + max_act = 2 * (max(qkv, ffn) + residual)
138 + # Size of output logits.
139 + output_logits = 2 * (max_num_batched_tokens * self.vocab_size)
140 + max_act = max(max_act, output_logits)
141 + dtype_size = get_dtype_size(self.dtype)
142 + return dtype_size * max_act
143 +
144 +
145 class OPTMemoryAnalyzer(CacheFlowMemoryAnalyzer):
146
147 def __init__(
cacheflow/models/model_utils.py
+4
@@ -5,9 +5,11 @@ import torch.nn as nn
5 from transformers import AutoConfig
6
7 from cacheflow.models.memory_analyzer import CacheFlowMemoryAnalyzer
8 +from cacheflow.models.memory_analyzer import GPT2MemoryAnalyzer
9 from cacheflow.models.memory_analyzer import GPTNeoXMemoryAnalyzer
10 from cacheflow.models.memory_analyzer import LlamaMemoryAnalyzer
11 from cacheflow.models.memory_analyzer import OPTMemoryAnalyzer
12 +from cacheflow.models.gpt2 import GPT2LMHeadModel
13 from cacheflow.models.gpt_neox import GPTNeoXForCausalLM
14 from cacheflow.models.llama import LlamaForCausalLM
15 from cacheflow.models.opt import OPTForCausalLM
@@ -15,6 +17,7 @@ from cacheflow.models.utils import get_torch_dtype
17
18
19 _MODELS = {
20 + 'gpt2': GPT2LMHeadModel,
21 'llama': LlamaForCausalLM,
22 'opt': OPTForCausalLM,
23 'stablelm': GPTNeoXForCausalLM,
@@ -22,6 +25,7 @@ _MODELS = {
25 }
26
27 _MEMORY_ANALYZERS = {
28 + 'gpt2': GPT2MemoryAnalyzer,
29 'llama': LlamaMemoryAnalyzer,
30 'opt': OPTMemoryAnalyzer,
31 'stablelm': GPTNeoXMemoryAnalyzer,
cacheflow/models/opt.py
+1 -1
@@ -234,7 +234,7 @@ class OPTForCausalLM(nn.Module):
234 # TODO(zhuohan): create a new weight after implementing pipeline
235 # parallelism
236 self.lm_head_weight = self.model.decoder.embed_tokens.weight
237 - self.sampler = Sampler()
237 + self.sampler = Sampler(config.vocab_size)
238
239 def forward(
240 self,
cacheflow/models/sample.py
+4 -1
@@ -11,8 +11,9 @@ from cacheflow.parallel_utils.tensor_parallel import gather_from_tensor_model_pa
11
12 class Sampler(nn.Module):
13
14 - def __init__(self) -> None:
14 + def __init__(self, vocab_size: int) -> None:
15 super().__init__()
16 + self.vocab_size = vocab_size
17
18 def forward(
19 self,
@@ -26,6 +27,8 @@ class Sampler(nn.Module):
27 # Get the logits for the next tokens.
28 logits = torch.matmul(hidden_states, embedding.t())
29 logits = gather_from_tensor_model_parallel_region(logits)
30 + # Remove paddings in vocab.
31 + logits = logits[:, :self.vocab_size]
32
33 # Apply temperature scaling.
34 temperatures = _get_temperatures(input_metadata)