A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
Add support for baichuan (#365)
codethazine committed
Jul 17, 2023 at 21:50 UTC
20b0d88d1630aec3a18a80590080b0ab1d16969f
6 files changed
+361
vllm/model_executor/model_loader.py
+1
@@ -11,6 +11,7 @@ from vllm.model_executor.weight_utils import initialize_dummy_weights
11
12
# TODO(woosuk): Lazy-load the model classes.
13
_MODEL_REGISTRY = {
14
+ "BaiChuanForCausalLM": BaiChuanForCausalLM,
15
"BloomForCausalLM": BloomForCausalLM,
16
"GPT2LMHeadModel": GPT2LMHeadModel,
17
"GPTBigCodeForCausalLM": GPTBigCodeForCausalLM,
vllm/model_executor/models/__init__.py
+2
@@ -1,3 +1,4 @@
1
+from vllm.model_executor.models.baichuan import BaiChuanForCausalLM
2
from vllm.model_executor.models.bloom import BloomForCausalLM
3
from vllm.model_executor.models.gpt2 import GPT2LMHeadModel
4
from vllm.model_executor.models.gpt_bigcode import GPTBigCodeForCausalLM
@@ -8,6 +9,7 @@ from vllm.model_executor.models.mpt import MPTForCausalLM
9
from vllm.model_executor.models.opt import OPTForCausalLM
10
11
__all__ = [
12
+ "BaiChuanForCausalLM",
13
"BloomForCausalLM",
14
"GPT2LMHeadModel",
15
"GPTBigCodeForCausalLM",
vllm/model_executor/models/baichuan.py
new
+293
@@ -0,0 +1,293 @@
1
+# coding=utf-8
2
+# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
3
+#
4
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
5
+# and OPT implementations in this library. It has been modified from its
6
+# original forms to accommodate minor architectural differences compared
7
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
8
+#
9
+# Licensed under the Apache License, Version 2.0 (the "License");
10
+# you may not use this file except in compliance with the License.
11
+# You may obtain a copy of the License at
12
+#
13
+# http://www.apache.org/licenses/LICENSE-2.0
14
+#
15
+# Unless required by applicable law or agreed to in writing, software
16
+# distributed under the License is distributed on an "AS IS" BASIS,
17
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18
+# See the License for the specific language governing permissions and
19
+# limitations under the License.
20
+"""Inference-only BaiChuan model compatible with HuggingFace weights.
21
+
22
+The input of the model is flattened to a 1D tensor of tokens. The model uses
23
+InputMetadata to extract the original 2D shape of the input.
24
+"""
25
+from typing import Dict, List, Optional, Tuple
26
+
27
+import torch
28
+from torch import nn
29
+
30
+from vllm.sequence import SequenceOutputs
31
+from vllm.model_executor.input_metadata import InputMetadata
32
+from vllm.model_executor.layers.activation import SiluAndMul
33
+from vllm.model_executor.layers.layernorm import RMSNorm
34
+from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
35
+from vllm.model_executor.layers.sampler import Sampler
36
+from vllm.model_executor.weight_utils import (hf_model_weights_iterator,
37
+ load_tensor_parallel_weights)
38
+from vllm.model_executor.parallel_utils.parallel_state import (
39
+ get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
40
+from vllm.model_executor.parallel_utils.tensor_parallel import (
41
+ VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
42
+from vllm.transformers_utils.configs.baichuan import BaiChuanConfig
43
+
44
+KVCache = Tuple[torch.Tensor, torch.Tensor]
45
+
46
+
47
+class BaiChuanMLP(nn.Module):
48
+
49
+ def __init__(
50
+ self,
51
+ hidden_size: int,
52
+ intermediate_size: int,
53
+ hidden_act: str,
54
+ ):
55
+ super().__init__()
56
+ self.gate_up_proj = ColumnParallelLinear(hidden_size,
57
+ 2 * intermediate_size,
58
+ bias=False,
59
+ gather_output=False,
60
+ perform_initialization=False)
61
+ self.down_proj = RowParallelLinear(intermediate_size,
62
+ hidden_size,
63
+ bias=False,
64
+ input_is_parallel=True,
65
+ perform_initialization=False)
66
+ if hidden_act != "silu":
67
+ raise ValueError(f"Unsupported activation: {hidden_act}. "
68
+ "Only silu is supported for now.")
69
+ self.act_fn = SiluAndMul()
70
+
71
+ def forward(self, x):
72
+ gate_up, _ = self.gate_up_proj(x)
73
+ x = self.act_fn(gate_up)
74
+ x, _ = self.down_proj(x)
75
+ return x
76
+
77
+
78
+class BaiChuanAttention(nn.Module):
79
+ """Multi-headed attention from 'Attention Is All You Need' paper"""
80
+
81
+ def __init__(
82
+ self,
83
+ hidden_size: int,
84
+ num_heads: int,
85
+ ):
86
+ super().__init__()
87
+ self.hidden_size = hidden_size
88
+ tensor_model_parallel_world_size = get_tensor_model_parallel_world_size(
89
+ )
90
+ self.total_num_heads = num_heads
91
+ assert self.total_num_heads % tensor_model_parallel_world_size == 0
92
+ self.num_heads = (self.total_num_heads //
93
+ tensor_model_parallel_world_size)
94
+ self.head_dim = hidden_size // self.total_num_heads
95
+ self.scaling = self.head_dim**-0.5
96
+
97
+ # pylint: disable=invalid-name
98
+ self.W_pack = ColumnParallelLinear(
99
+ hidden_size,
100
+ 3 * hidden_size,
101
+ bias=False,
102
+ gather_output=False,
103
+ perform_initialization=False,
104
+ )
105
+ self.o_proj = RowParallelLinear(
106
+ self.total_num_heads * self.head_dim,
107
+ hidden_size,
108
+ bias=False,
109
+ input_is_parallel=True,
110
+ perform_initialization=False,
111
+ )
112
+
113
+ self.attn = PagedAttentionWithRoPE(self.num_heads,
114
+ self.head_dim,
115
+ self.scaling,
116
+ rotary_dim=self.head_dim)
117
+
118
+ def forward(
119
+ self,
120
+ positions: torch.Tensor,
121
+ hidden_states: torch.Tensor,
122
+ kv_cache: KVCache,
123
+ input_metadata: InputMetadata,
124
+ cache_event: Optional[torch.cuda.Event],
125
+ ) -> torch.Tensor:
126
+ qkv, _ = self.W_pack(hidden_states)
127
+ q, k, v = qkv.chunk(chunks=3, dim=-1)
128
+ k_cache, v_cache = kv_cache
129
+ attn_output = self.attn(positions, q, k, v, k_cache, v_cache,
130
+ input_metadata, cache_event)
131
+ output, _ = self.o_proj(attn_output)
132
+ return output
133
+
134
+
135
+class BaiChuanDecoderLayer(nn.Module):
136
+
137
+ def __init__(self, config: BaiChuanConfig):
138
+ super().__init__()
139
+ self.hidden_size = config.hidden_size
140
+ self.self_attn = BaiChuanAttention(
141
+ hidden_size=self.hidden_size,
142
+ num_heads=config.num_attention_heads,
143
+ )
144
+ self.mlp = BaiChuanMLP(
145
+ hidden_size=self.hidden_size,
146
+ intermediate_size=config.intermediate_size,
147
+ hidden_act=config.hidden_act,
148
+ )
149
+ self.input_layernorm = RMSNorm(config.hidden_size,
150
+ eps=config.rms_norm_eps)
151
+ self.post_attention_layernorm = RMSNorm(config.hidden_size,
152
+ eps=config.rms_norm_eps)
153
+
154
+ def forward(
155
+ self,
156
+ positions: torch.Tensor,
157
+ hidden_states: torch.Tensor,
158
+ kv_cache: KVCache,
159
+ input_metadata: InputMetadata,
160
+ cache_event: Optional[torch.cuda.Event],
161
+ ) -> torch.Tensor:
162
+ # Self Attention
163
+ residual = hidden_states
164
+ hidden_states = self.input_layernorm(hidden_states)
165
+ hidden_states = self.self_attn(
166
+ positions=positions,
167
+ hidden_states=hidden_states,
168
+ kv_cache=kv_cache,
169
+ input_metadata=input_metadata,
170
+ cache_event=cache_event,
171
+ )
172
+ hidden_states = residual + hidden_states
173
+
174
+ # Fully Connected
175
+ residual = hidden_states
176
+ hidden_states = self.post_attention_layernorm(hidden_states)
177
+ hidden_states = self.mlp(hidden_states)
178
+ hidden_states = residual + hidden_states
179
+ return hidden_states
180
+
181
+
182
+class BaiChuanModel(nn.Module):
183
+
184
+ def __init__(self, config: BaiChuanConfig):
185
+ super().__init__()
186
+ self.config = config
187
+ self.padding_idx = config.pad_token_id
188
+ self.vocab_size = config.vocab_size
189
+
190
+ self.embed_tokens = VocabParallelEmbedding(
191
+ config.vocab_size,
192
+ config.hidden_size,
193
+ perform_initialization=False)
194
+ self.layers = nn.ModuleList([
195
+ BaiChuanDecoderLayer(config)
196
+ for _ in range(config.num_hidden_layers)
197
+ ])
198
+ self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
199
+
200
+ def forward(
201
+ self,
202
+ input_ids: torch.Tensor,
203
+ positions: torch.Tensor,
204
+ kv_caches: List[KVCache],
205
+ input_metadata: InputMetadata,
206
+ cache_events: Optional[List[torch.cuda.Event]],
207
+ ) -> torch.Tensor:
208
+ hidden_states = self.embed_tokens(input_ids)
209
+ for i in range(len(self.layers)):
210
+ if cache_events is None:
211
+ cache_event = None
212
+ else:
213
+ cache_event = cache_events[i]
214
+ layer = self.layers[i]
215
+ hidden_states = layer(
216
+ positions,
217
+ hidden_states,
218
+ kv_caches[i],
219
+ input_metadata,
220
+ cache_event,
221
+ )
222
+ hidden_states = self.norm(hidden_states)
223
+ return hidden_states
224
+
225
+
226
+class BaiChuanForCausalLM(nn.Module):
227
+
228
+ def __init__(self, config):
229
+ super().__init__()
230
+ self.config = config
231
+ self.model = BaiChuanModel(config)
232
+ self.lm_head = ColumnParallelLinear(config.hidden_size,
233
+ config.vocab_size,
234
+ bias=False,
235
+ gather_output=False,
236
+ perform_initialization=False)
237
+ self.sampler = Sampler(config.vocab_size)
238
+
239
+ def forward(
240
+ self,
241
+ input_ids: torch.Tensor,
242
+ positions: torch.Tensor,
243
+ kv_caches: List[KVCache],
244
+ input_metadata: InputMetadata,
245
+ cache_events: Optional[List[torch.cuda.Event]],
246
+ ) -> Dict[int, SequenceOutputs]:
247
+ hidden_states = self.model(input_ids, positions, kv_caches,
248
+ input_metadata, cache_events)
249
+ next_tokens = self.sampler(self.lm_head.weight, hidden_states,
250
+ input_metadata)
251
+ return next_tokens
252
+
253
+ _column_parallel_weights = [
254
+ "embed_tokens.weight", "lm_head.weight", "W_pack.weight",
255
+ "gate_proj.weight", "up_proj.weight"
256
+ ]
257
+ _row_parallel_weights = ["o_proj.weight", "down_proj.weight"]
258
+
259
+ def load_weights(self,
260
+ model_name_or_path: str,
261
+ cache_dir: Optional[str] = None,
262
+ use_np_cache: bool = False):
263
+ tensor_model_parallel_rank = get_tensor_model_parallel_rank()
264
+ state_dict = self.state_dict()
265
+
266
+ for name, loaded_weight in hf_model_weights_iterator(
267
+ model_name_or_path, cache_dir, use_np_cache):
268
+ if "rotary_emb.inv_freq" in name:
269
+ continue
270
+
271
+ is_gate_up_weight = False
272
+ for stride_id, weight_name in enumerate(["gate_proj", "up_proj"]):
273
+ if weight_name not in name:
274
+ continue
275
+ param = state_dict[name.replace(weight_name, "gate_up_proj")]
276
+ shard_size = param.shape[0] // 2
277
+ loaded_weight = loaded_weight[
278
+ shard_size * tensor_model_parallel_rank:shard_size *
279
+ (tensor_model_parallel_rank + 1)]
280
+ param_slice = param.data[shard_size * stride_id:shard_size *
281
+ (stride_id + 1)]
282
+ assert param_slice.shape == loaded_weight.shape
283
+ param_slice.copy_(loaded_weight)
284
+ is_gate_up_weight = True
285
+ break
286
+ if is_gate_up_weight:
287
+ continue
288
+
289
+ param = state_dict[name]
290
+ load_tensor_parallel_weights(param, loaded_weight, name,
291
+ self._column_parallel_weights,
292
+ self._row_parallel_weights,
293
+ tensor_model_parallel_rank)
vllm/transformers_utils/config.py
+1
@@ -4,6 +4,7 @@ from vllm.transformers_utils.configs import * # pylint: disable=wildcard-import
4
5
_CONFIG_REGISTRY = {
6
"mpt": MPTConfig,
7
+ "baichuan": BaiChuanConfig,
8
}
9
10
vllm/transformers_utils/configs/__init__.py
+2
@@ -1,5 +1,7 @@
1
from vllm.transformers_utils.configs.mpt import MPTConfig
2
+from vllm.transformers_utils.configs.baichuan import BaiChuanConfig
3
4
__all__ = [
5
"MPTConfig",
6
+ "BaiChuanConfig",
7
]
vllm/transformers_utils/configs/baichuan.py
new
+62
@@ -0,0 +1,62 @@
1
+# coding=utf-8
2
+# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
3
+#
4
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
5
+# and OPT implementations in this library. It has been modified from its
6
+# original forms to accommodate minor architectural differences compared
7
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
8
+#
9
+# Licensed under the Apache License, Version 2.0 (the "License");
10
+# you may not use this file except in compliance with the License.
11
+# You may obtain a copy of the License at
12
+#
13
+# http://www.apache.org/licenses/LICENSE-2.0
14
+#
15
+# Unless required by applicable law or agreed to in writing, software
16
+# distributed under the License is distributed on an "AS IS" BASIS,
17
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18
+# See the License for the specific language governing permissions and
19
+# limitations under the License.
20
+
21
+from transformers.configuration_utils import PretrainedConfig
22
+
23
+
24
+class BaiChuanConfig(PretrainedConfig):
25
+ model_type = "baichuan"
26
+ keys_to_ignore_at_inference = ["past_key_values"]
27
+
28
+ def __init__(
29
+ self,
30
+ vocab_size=64000,
31
+ hidden_size=4096,
32
+ intermediate_size=11008,
33
+ num_hidden_layers=32,
34
+ num_attention_heads=32,
35
+ hidden_act="silu",
36
+ max_position_embeddings=4096,
37
+ initializer_range=0.02,
38
+ rms_norm_eps=1e-6,
39
+ use_cache=True,
40
+ pad_token_id=0,
41
+ bos_token_id=1,
42
+ eos_token_id=2,
43
+ tie_word_embeddings=False,
44
+ **kwargs,
45
+ ):
46
+ self.vocab_size = vocab_size
47
+ self.max_position_embeddings = max_position_embeddings
48
+ self.hidden_size = hidden_size
49
+ self.intermediate_size = intermediate_size
50
+ self.num_hidden_layers = num_hidden_layers
51
+ self.num_attention_heads = num_attention_heads
52
+ self.hidden_act = hidden_act
53
+ self.initializer_range = initializer_range
54
+ self.rms_norm_eps = rms_norm_eps
55
+ self.use_cache = use_cache
56
+ super().__init__(
57
+ pad_token_id=pad_token_id,
58
+ bos_token_id=bos_token_id,
59
+ eos_token_id=eos_token_id,
60
+ tie_word_embeddings=tie_word_embeddings,
61
+ **kwargs,
62
+ )