A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
GPTBigCode (StarCoder, SantaCoder Support) (#209)
Michael Feil committed
Jun 22, 2023 at 19:49 UTC
298695b76691867ecd320ea6a2c6d0c6a843d5ae
4 files changed
+298
-2
vllm/model_executor/layers/activation.py
+1
@@ -8,6 +8,7 @@ _ACTIVATION_REGISTRY = {
8
"gelu": nn.GELU(),
9
"gelu_new": nn.GELU(approximate="tanh"), # NOTE: This may introduce small rounding errors.
10
"gelu_fast": nn.GELU(approximate="tanh"), # NOTE: This may introduce small rounding errors.
11
+ "gelu_pytorch_tanh": nn.GELU(approximate="tanh"), # NOTE: This may introduce small rounding errors.
12
"relu": nn.ReLU(),
13
}
14
vllm/model_executor/model_loader.py
+2
-1
@@ -6,13 +6,14 @@ import torch.nn as nn
6
from transformers import PretrainedConfig
7
8
from vllm.config import ModelConfig
9
-from vllm.model_executor.models import (GPT2LMHeadModel, GPTNeoXForCausalLM,
9
+from vllm.model_executor.models import (GPT2LMHeadModel, GPTBigCodeForCausalLM, GPTNeoXForCausalLM,
10
LlamaForCausalLM, OPTForCausalLM)
11
from vllm.model_executor.weight_utils import initialize_dummy_weights
12
13
# TODO(woosuk): Lazy-load the model classes.
14
_MODEL_REGISTRY = {
15
"GPT2LMHeadModel": GPT2LMHeadModel,
16
+ "GPTBigCodeForCausalLM": GPTBigCodeForCausalLM,
17
"GPTNeoXForCausalLM": GPTNeoXForCausalLM,
18
"LlamaForCausalLM": LlamaForCausalLM,
19
"OPTForCausalLM": OPTForCausalLM,
vllm/model_executor/models/__init__.py
+4
-1
@@ -1,11 +1,14 @@
1
-from vllm.model_executor.models.gpt_neox import GPTNeoXForCausalLM
1
from vllm.model_executor.models.gpt2 import GPT2LMHeadModel
2
+from vllm.model_executor.models.gpt_bigcode import GPTBigCodeForCausalLM
3
+from vllm.model_executor.models.gpt_neox import GPTNeoXForCausalLM
4
from vllm.model_executor.models.llama import LlamaForCausalLM
5
from vllm.model_executor.models.opt import OPTForCausalLM
6
7
8
+
9
__all__ = [
10
"GPT2LMHeadModel",
11
+ "GPTBigCodeForCausalLM",
12
"GPTNeoXForCausalLM",
13
"LlamaForCausalLM",
14
"OPTForCausalLM",
vllm/model_executor/models/gpt_bigcode.py
new
+291
@@ -0,0 +1,291 @@
1
+# coding=utf-8
2
+# Adapted from https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/gpt2/modeling_gpt2.py
3
+# Copyright 2023 The vLLM team.
4
+# Copyright 2023 CTranslate2, and Michael Feil
5
+# Copyright 2018 The OpenAI Team Authors and HuggingFace Inc. team.
6
+# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved.
7
+#
8
+# Licensed under the Apache License, Version 2.0 (the "License");
9
+# you may not use this file except in compliance with the License.
10
+# You may obtain a copy of the License at
11
+#
12
+# http://www.apache.org/licenses/LICENSE-2.0
13
+#
14
+# Unless required by applicable law or agreed to in writing, software
15
+# distributed under the License is distributed on an "AS IS" BASIS,
16
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17
+# See the License for the specific language governing permissions and
18
+# limitations under the License.
19
+"""Inference-only GPTBigCode model compatible with HuggingFace weights.
20
+
21
+The input of the model is flattened to a 1D tensor of tokens. The model uses
22
+InputMetadata to extract the original 2D shape of the input.
23
+"""
24
+from typing import Dict, List, Optional, Tuple
25
+
26
+import torch
27
+from torch import nn
28
+import numpy as np
29
+from transformers import GPTBigCodeConfig
30
+
31
+from vllm.model_executor.input_metadata import InputMetadata
32
+from vllm.model_executor.layers.activation import get_act_fn
33
+from vllm.model_executor.layers.attention import PagedAttention
34
+from vllm.model_executor.layers.sampler import Sampler
35
+from vllm.model_executor.weight_utils import (hf_model_weights_iterator,
36
+ load_tensor_parallel_weights)
37
+from vllm.model_executor.parallel_utils.parallel_state import (
38
+ get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
39
+from vllm.model_executor.parallel_utils.tensor_parallel import (
40
+ VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
41
+from vllm.sequence import SequenceOutputs
42
+
43
+KVCache = Tuple[torch.Tensor, torch.Tensor]
44
+
45
+
46
+class GPTBigCodeAttention(nn.Module):
47
+
48
+ def __init__(self, config: GPTBigCodeConfig):
49
+ super().__init__()
50
+ self.hidden_size = config.hidden_size
51
+ total_num_heads = config.num_attention_heads
52
+ tensor_model_parallel_world_size = get_tensor_model_parallel_world_size()
53
+ assert total_num_heads % tensor_model_parallel_world_size == 0
54
+ self.num_heads = total_num_heads // tensor_model_parallel_world_size
55
+ self.head_dim = self.hidden_size // total_num_heads
56
+ self.scale = self.head_dim ** -0.5
57
+
58
+ self.c_attn = ColumnParallelLinear(self.hidden_size, 3 * self.hidden_size,
59
+ bias=True, gather_output=False,
60
+ perform_initialization=False)
61
+ self.c_proj = RowParallelLinear(self.hidden_size, self.hidden_size,
62
+ bias=True, input_is_parallel=True,
63
+ perform_initialization=False)
64
+ self.attn = PagedAttention(self.num_heads, self.head_dim,
65
+ scale=self.scale)
66
+
67
+ def forward(
68
+ self,
69
+ hidden_states: torch.Tensor,
70
+ kv_cache: KVCache,
71
+ input_metadata: InputMetadata,
72
+ cache_event: Optional[torch.cuda.Event],
73
+ ) -> torch.Tensor:
74
+ qkv, _ = self.c_attn(hidden_states)
75
+ q, k, v = qkv.chunk(chunks=3, dim=-1)
76
+ key_cache, value_cache = kv_cache
77
+ attn_output = self.attn(
78
+ q, k, v, key_cache, value_cache, input_metadata, cache_event)
79
+ attn_output, _ = self.c_proj(attn_output)
80
+ return attn_output
81
+
82
+
83
+class GPTBigMLP(nn.Module):
84
+
85
+ def __init__(
86
+ self,
87
+ intermediate_size: int,
88
+ config: GPTBigCodeConfig,
89
+ ):
90
+ super().__init__()
91
+ hidden_size = config.hidden_size
92
+ self.c_fc = ColumnParallelLinear(hidden_size, intermediate_size,
93
+ bias=True, gather_output=False,
94
+ perform_initialization=False)
95
+ self.c_proj = RowParallelLinear(intermediate_size, hidden_size,
96
+ bias=True, input_is_parallel=True,
97
+ perform_initialization=False)
98
+ self.act = get_act_fn(config.activation_function)
99
+
100
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
101
+ hidden_states, _ = self.c_fc(hidden_states)
102
+ hidden_states = self.act(hidden_states)
103
+ hidden_states, _ = self.c_proj(hidden_states)
104
+ return hidden_states
105
+
106
+
107
+class GPTBigCodeBlock(nn.Module):
108
+
109
+ def __init__(self, config: GPTBigCodeConfig):
110
+ super().__init__()
111
+ hidden_size = config.hidden_size
112
+ inner_dim = config.n_inner if config.n_inner is not None else 4 * hidden_size
113
+
114
+ self.ln_1 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
115
+ self.attn = GPTBigCodeAttention(config)
116
+ self.ln_2 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
117
+ self.mlp = GPTBigMLP(inner_dim, config)
118
+
119
+ def forward(
120
+ self,
121
+ hidden_states: torch.Tensor,
122
+ kv_cache: KVCache,
123
+ input_metadata: InputMetadata,
124
+ cache_event: Optional[torch.cuda.Event],
125
+ ) -> torch.Tensor:
126
+ residual = hidden_states
127
+ hidden_states = self.ln_1(hidden_states)
128
+ attn_output = self.attn(
129
+ hidden_states=hidden_states,
130
+ kv_cache=kv_cache,
131
+ input_metadata=input_metadata,
132
+ cache_event=cache_event,
133
+ )
134
+ # residual connection
135
+ hidden_states = attn_output + residual
136
+
137
+ residual = hidden_states
138
+ hidden_states = self.ln_2(hidden_states)
139
+ feed_forward_hidden_states = self.mlp(hidden_states)
140
+ # residual connection
141
+ hidden_states = residual + feed_forward_hidden_states
142
+ return hidden_states
143
+
144
+
145
+class GPTBigCodeModel(nn.Module):
146
+
147
+ def __init__(self, config: GPTBigCodeConfig):
148
+ super().__init__()
149
+ self.config = config
150
+ assert config.add_cross_attention == False
151
+
152
+ self.embed_dim = config.hidden_size
153
+
154
+ # Optimization: While the vocab size of GPT-2 is 50257, we extend it
155
+ # to 50304 in order to make it divisible by 64.
156
+ # This improves performance since GPUs are faster if the dimension
157
+ # is divisible by 64. In addition, it allows us to shard the embedding
158
+ # layer across 2, 4, 8, or more GPUs.
159
+ vocab_size = ((config.vocab_size + 63) // 64) * 64
160
+ self.wte = VocabParallelEmbedding(vocab_size, self.embed_dim)
161
+ self.wpe = nn.Embedding(config.max_position_embeddings, self.embed_dim)
162
+ self.h = nn.ModuleList(
163
+ [GPTBigCodeBlock(config) for _ in range(config.num_hidden_layers)])
164
+ self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
165
+
166
+ def forward(
167
+ self,
168
+ input_ids: torch.Tensor,
169
+ position_ids: torch.Tensor,
170
+ kv_caches: List[KVCache],
171
+ input_metadata: InputMetadata,
172
+ cache_events: Optional[List[torch.cuda.Event]],
173
+ ) -> torch.Tensor:
174
+ inputs_embeds = self.wte(input_ids)
175
+ position_embeds = self.wpe(position_ids)
176
+ hidden_states = inputs_embeds + position_embeds
177
+
178
+ for i in range(len(self.h)):
179
+ if cache_events is None:
180
+ cache_event = None
181
+ else:
182
+ cache_event = cache_events[i]
183
+ layer = self.h[i]
184
+ hidden_states = layer(
185
+ hidden_states, kv_caches[i], input_metadata, cache_event)
186
+
187
+ hidden_states = self.ln_f(hidden_states)
188
+ return hidden_states
189
+
190
+
191
+class GPTBigCodeForCausalLM(nn.Module):
192
+
193
+ def __init__(self, config: GPTBigCodeConfig):
194
+ super().__init__()
195
+ self.config = config
196
+ self.transformer = GPTBigCodeModel(config)
197
+ # TODO(zhuohan): create a new weight after implementing pipeline
198
+ # parallelism
199
+ self.lm_head_weight = self.transformer.wte.weight
200
+ self.sampler = Sampler(config.vocab_size)
201
+
202
+ def forward(
203
+ self,
204
+ input_ids: torch.Tensor,
205
+ positions: torch.Tensor,
206
+ kv_caches: List[KVCache],
207
+ input_metadata: InputMetadata,
208
+ cache_events: Optional[List[torch.cuda.Event]],
209
+ ) -> Dict[int, SequenceOutputs]:
210
+ hidden_states = self.transformer(
211
+ input_ids, positions, kv_caches, input_metadata, cache_events)
212
+ next_tokens = self.sampler(
213
+ self.lm_head_weight, hidden_states, input_metadata)
214
+ return next_tokens
215
+
216
+ _column_parallel_weights = ["wte.weight", "c_fc.weight", "c_fc.bias"]
217
+ _row_parallel_weights = ["c_proj.weight"]
218
+
219
+ def load_weights(self, model_name_or_path: str,
220
+ cache_dir: Optional[str] = None,
221
+ use_np_cache: bool = False):
222
+ tensor_model_parallel_world_size = get_tensor_model_parallel_world_size()
223
+ tensor_model_parallel_rank = get_tensor_model_parallel_rank()
224
+ state_dict = self.state_dict()
225
+
226
+ for name, loaded_weight in hf_model_weights_iterator(
227
+ model_name_or_path, cache_dir, use_np_cache):
228
+ if "lm_head.weight" in name:
229
+ # GPT-2 ties the weights of the embedding layer and the final
230
+ # linear layer.
231
+ continue
232
+ if ".attn.bias" in name:
233
+ # Skip attention mask.
234
+ # NOTE: "c_attn.bias" should not be skipped.
235
+ continue
236
+
237
+ param = state_dict[name]
238
+
239
+ def _expand_mqa_mha(qkv_array, n_head, head_dim):
240
+ """manipulates along axis=0 from MQA to MHA
241
+ inputs: qkv_array.shape=((n_heads + 2) * head_dim, hidden_dim)
242
+ with n_heads for q, then 1 for k, 1 for 1 v, times head dim
243
+ return: qkv_array.shape=(3 * n_heads * head_dim, hidden_dim)
244
+
245
+ TODO: this function is no longer needed once vllm supports MQA.
246
+ """
247
+ qkv_array = qkv_array.numpy()
248
+
249
+ dims_q = n_head * head_dim
250
+ q, k, v = np.split(qkv_array, (dims_q, dims_q + head_dim), axis=0)
251
+ # q is fine, but k & v have not replicated shape along the first axis
252
+ # as long as MQA is not nativly supported, increase memory and replicated
253
+ # (head_dim, hidden_dim) to (n_heads * head_dim, hidden_dim)
254
+ if k.ndim == 2 and v.ndim == 2:
255
+ replication = (n_head, 1) # weights
256
+ else:
257
+ replication = n_head # biases
258
+ # replicate n_head times for q, v
259
+ k, v = np.tile(k, replication), np.tile(v, replication)
260
+ # concat q, k, v along the first axis (n_heads * head_dim, hidden_dim)
261
+ # to (3 * n_heads * head_dim, hidden_dim)
262
+ qkv_array = np.concatenate((q, k, v), axis=0)
263
+ return torch.from_numpy(qkv_array)
264
+
265
+ # For the fused QKV linear layer, manually shard the weights.
266
+ if "c_attn" in name:
267
+ # GPT-2's fused QKV has the shape of [3 * num_heads * head_size, hidden_size].
268
+ # When tensor parallelism is used, we shard the weights along the head dimension.
269
+ total_num_heads = self.config.num_attention_heads
270
+ hidden_size = self.config.hidden_size
271
+ head_size = hidden_size // total_num_heads
272
+ num_heads = total_num_heads // tensor_model_parallel_world_size
273
+ head_start = tensor_model_parallel_rank * num_heads
274
+ head_end = (tensor_model_parallel_rank + 1) * num_heads
275
+
276
+ if name.endswith(".weight"):
277
+ loaded_weight = _expand_mqa_mha(loaded_weight, n_head=total_num_heads, head_dim=head_size)
278
+ loaded_weight = loaded_weight.view(3, total_num_heads, head_size, hidden_size)
279
+ loaded_weight = loaded_weight[:, head_start:head_end, :, :]
280
+ loaded_weight = loaded_weight.reshape(-1, hidden_size)
281
+ elif name.endswith(".bias"):
282
+ loaded_weight = _expand_mqa_mha(loaded_weight, n_head=total_num_heads, head_dim=head_size)
283
+ loaded_weight = loaded_weight.view(3, total_num_heads, head_size)
284
+ loaded_weight = loaded_weight[:, head_start:head_end, :]
285
+ loaded_weight = loaded_weight.reshape(-1)
286
+ else:
287
+ raise ValueError(f"Unexpected parameter name {name}")
288
+ load_tensor_parallel_weights(param, loaded_weight, name,
289
+ self._column_parallel_weights,
290
+ self._row_parallel_weights,
291
+ tensor_model_parallel_rank)