A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Support starcoder2 architecture (#3089)
Seonghyeon committed
Feb 29, 2024 at 17:51 UTC
bfdcfa6a053c693800551bd1bd71acabbe1941e8
7 files changed
+452
README.md
+1
@@ -78,6 +78,7 @@ vLLM seamlessly supports many Hugging Face models, including the following archi
78
- Qwen (`Qwen/Qwen-7B`, `Qwen/Qwen-7B-Chat`, etc.)
79
- Qwen2 (`Qwen/Qwen2-7B-beta`, `Qwen/Qwen-7B-Chat-beta`, etc.)
80
- StableLM(`stabilityai/stablelm-3b-4e1t`, `stabilityai/stablelm-base-alpha-7b-v2`, etc.)
81
+- Starcoder2(`bigcode/starcoder2-3b`, `bigcode/starcoder2-7b`, `bigcode/starcoder2-15b`, etc.)
82
- Yi (`01-ai/Yi-6B`, `01-ai/Yi-34B`, etc.)
83
84
Install vLLM with pip or [from source](https://vllm.readthedocs.io/en/latest/getting_started/installation.html#build-from-source):
tests/models/test_models.py
+1
@@ -19,6 +19,7 @@ MODELS = [
19
"microsoft/phi-2",
20
"stabilityai/stablelm-3b-4e1t",
21
"allenai/OLMo-1B",
22
+ "bigcode/starcoder2-3b",
23
]
24
25
vllm/model_executor/models/__init__.py
+1
@@ -45,6 +45,7 @@ _MODELS = {
45
"RWForCausalLM": ("falcon", "FalconForCausalLM"),
46
"StableLMEpochForCausalLM": ("stablelm", "StablelmForCausalLM"),
47
"StableLmForCausalLM": ("stablelm", "StablelmForCausalLM"),
48
+ "Starcoder2ForCausalLM": ("starcoder2", "Starcoder2ForCausalLM"),
49
}
50
51
# Models not supported by ROCm.
vllm/model_executor/models/starcoder2.py
new
+310
@@ -0,0 +1,310 @@
1
+# coding=utf-8
2
+# Copyright 2024 BigCode and the HuggingFace Inc. team. All rights reserved.
3
+#
4
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
5
+# and OPT implementations in this library. It has been modified from its
6
+# original forms to accommodate minor architectural differences compared
7
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
8
+#
9
+# Licensed under the Apache License, Version 2.0 (the "License");
10
+# you may not use this file except in compliance with the License.
11
+# You may obtain a copy of the License at
12
+#
13
+# http://www.apache.org/licenses/LICENSE-2.0
14
+#
15
+# Unless required by applicable law or agreed to in writing, software
16
+# distributed under the License is distributed on an "AS IS" BASIS,
17
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18
+# See the License for the specific language governing permissions and
19
+# limitations under the License.
20
+""" PyTorch Starcoder2 model."""
21
+from typing import List, Optional, Tuple
22
+
23
+import torch
24
+from torch import nn
25
+
26
+from vllm.model_executor.input_metadata import InputMetadata
27
+from vllm.model_executor.sampling_metadata import SamplingMetadata
28
+from vllm.model_executor.layers.attention import PagedAttention
29
+from vllm.model_executor.layers.activation import get_act_fn
30
+from vllm.model_executor.layers.rotary_embedding import get_rope
31
+from vllm.model_executor.layers.linear import (ColumnParallelLinear,
32
+ LinearMethodBase,
33
+ QKVParallelLinear,
34
+ RowParallelLinear)
35
+from vllm.model_executor.layers.sampler import Sampler
36
+from vllm.model_executor.layers.vocab_parallel_embedding import (
37
+ VocabParallelEmbedding, ParallelLMHead, DEFAULT_VOCAB_PADDING_SIZE)
38
+from vllm.model_executor.parallel_utils.parallel_state import get_tensor_model_parallel_world_size
39
+from vllm.model_executor.weight_utils import (default_weight_loader,
40
+ hf_model_weights_iterator)
41
+from vllm.sequence import SamplerOutput
42
+
43
+try:
44
+ from transformers import Starcoder2Config
45
+except ImportError:
46
+ # fallback to PretrainedConfig
47
+ # NOTE: Please install transformers from source or use transformers>=4.39.0
48
+ from transformers import PretrainedConfig as Starcoder2Config
49
+
50
+KVCache = Tuple[torch.Tensor, torch.Tensor]
51
+
52
+
53
+class Starcoder2Attention(nn.Module):
54
+
55
+ def __init__(self,
56
+ config: Starcoder2Config,
57
+ linear_method: Optional[LinearMethodBase] = None):
58
+ super().__init__()
59
+ self.config = config
60
+
61
+ self.hidden_size = config.hidden_size
62
+ tp_size = get_tensor_model_parallel_world_size()
63
+ self.total_num_heads = config.num_attention_heads
64
+ assert self.total_num_heads % tp_size == 0
65
+ self.num_heads = self.total_num_heads // tp_size
66
+ self.total_num_kv_heads = config.num_key_value_heads
67
+ if self.total_num_kv_heads >= tp_size:
68
+ # Number of KV heads is greater than TP size, so we partition
69
+ # the KV heads across multiple tensor parallel GPUs.
70
+ assert self.total_num_kv_heads % tp_size == 0
71
+ else:
72
+ # Number of KV heads is less than TP size, so we replicate
73
+ # the KV heads across multiple tensor parallel GPUs.
74
+ assert tp_size % self.total_num_kv_heads == 0
75
+ self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
76
+ self.head_dim = self.hidden_size // self.total_num_heads
77
+ self.q_size = self.num_heads * self.head_dim
78
+ self.kv_size = self.num_kv_heads * self.head_dim
79
+ self.scaling = self.head_dim**-0.5
80
+ self.rope_theta = config.rope_theta
81
+ self.max_position_embeddings = config.max_position_embeddings
82
+ self.use_bias = config.use_bias
83
+ self.sliding_window = config.sliding_window
84
+
85
+ self.qkv_proj = QKVParallelLinear(
86
+ self.hidden_size,
87
+ self.head_dim,
88
+ self.total_num_heads,
89
+ self.total_num_kv_heads,
90
+ bias=self.use_bias,
91
+ linear_method=linear_method,
92
+ )
93
+ self.o_proj = RowParallelLinear(
94
+ self.total_num_heads * self.head_dim,
95
+ self.hidden_size,
96
+ bias=self.use_bias,
97
+ linear_method=linear_method,
98
+ )
99
+ self.rotary_emb = get_rope(
100
+ self.head_dim,
101
+ rotary_dim=self.head_dim,
102
+ max_position=self.max_position_embeddings,
103
+ base=int(self.rope_theta),
104
+ is_neox_style=True,
105
+ )
106
+ self.attn = PagedAttention(
107
+ self.num_heads,
108
+ self.head_dim,
109
+ self.scaling,
110
+ num_kv_heads=self.num_kv_heads,
111
+ sliding_window=self.sliding_window,
112
+ )
113
+
114
+ def forward(
115
+ self,
116
+ positions: torch.Tensor,
117
+ hidden_states: torch.Tensor,
118
+ kv_cache: KVCache,
119
+ input_metadata: InputMetadata,
120
+ ) -> torch.Tensor:
121
+ qkv, _ = self.qkv_proj(hidden_states)
122
+ q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
123
+ q, k = self.rotary_emb(positions, q, k)
124
+ k_cache, v_cache = kv_cache
125
+ attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
126
+ output, _ = self.o_proj(attn_output)
127
+ return output
128
+
129
+
130
+class Starcoder2MLP(nn.Module):
131
+
132
+ def __init__(self,
133
+ config: Starcoder2Config,
134
+ linear_method: Optional[LinearMethodBase] = None):
135
+ super().__init__()
136
+ self.c_fc = ColumnParallelLinear(
137
+ config.hidden_size,
138
+ config.intermediate_size,
139
+ bias=config.use_bias,
140
+ linear_method=linear_method,
141
+ )
142
+ self.c_proj = RowParallelLinear(
143
+ config.intermediate_size,
144
+ config.hidden_size,
145
+ bias=config.use_bias,
146
+ linear_method=linear_method,
147
+ )
148
+ self.act = get_act_fn(config.hidden_act,
149
+ intermediate_size=config.intermediate_size)
150
+
151
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
152
+ hidden_states, _ = self.c_fc(hidden_states)
153
+ hidden_states = self.act(hidden_states)
154
+ hidden_states, _ = self.c_proj(hidden_states)
155
+ return hidden_states
156
+
157
+
158
+class Starcoder2DecoderLayer(nn.Module):
159
+
160
+ def __init__(self,
161
+ config: Starcoder2Config,
162
+ linear_method: Optional[LinearMethodBase] = None):
163
+ super().__init__()
164
+ self.hidden_size = config.hidden_size
165
+ self.self_attn = Starcoder2Attention(config,
166
+ linear_method=linear_method)
167
+ self.mlp = Starcoder2MLP(config, linear_method=linear_method)
168
+ self.input_layernorm = nn.LayerNorm(config.hidden_size,
169
+ eps=config.norm_epsilon)
170
+ self.post_attention_layernorm = nn.LayerNorm(config.hidden_size,
171
+ eps=config.norm_epsilon)
172
+
173
+ def forward(
174
+ self,
175
+ positions: torch.Tensor,
176
+ hidden_states: torch.Tensor,
177
+ kv_cache: KVCache,
178
+ input_metadata: InputMetadata,
179
+ ) -> torch.Tensor:
180
+ # Self Attention
181
+ residual = hidden_states
182
+ hidden_states = self.input_layernorm(hidden_states)
183
+ hidden_states = self.self_attn(
184
+ positions=positions,
185
+ hidden_states=hidden_states,
186
+ kv_cache=kv_cache,
187
+ input_metadata=input_metadata,
188
+ )
189
+ hidden_states = residual + hidden_states
190
+
191
+ # Fully Connected
192
+ residual = hidden_states
193
+ hidden_states = self.post_attention_layernorm(hidden_states)
194
+ hidden_states = self.mlp(hidden_states)
195
+ hidden_states = residual + hidden_states
196
+
197
+ return hidden_states
198
+
199
+
200
+class Starcoder2Model(nn.Module):
201
+
202
+ def __init__(self,
203
+ config: Starcoder2Config,
204
+ linear_method: Optional[LinearMethodBase] = None):
205
+ super().__init__()
206
+ self.config = config
207
+ self.padding_idx = config.pad_token_id
208
+ self.vocab_size = config.vocab_size
209
+
210
+ # TODO: consider padding_idx (currently removed)
211
+ self.embed_tokens = VocabParallelEmbedding(config.vocab_size,
212
+ config.hidden_size)
213
+ self.layers = nn.ModuleList([
214
+ Starcoder2DecoderLayer(config, linear_method=linear_method)
215
+ for _ in range(config.num_hidden_layers)
216
+ ])
217
+ self.norm = nn.LayerNorm(config.hidden_size, eps=config.norm_epsilon)
218
+
219
+ def forward(
220
+ self,
221
+ input_ids: torch.Tensor,
222
+ positions: torch.Tensor,
223
+ kv_caches: List[KVCache],
224
+ input_metadata: InputMetadata,
225
+ ) -> torch.Tensor:
226
+ hidden_states = self.embed_tokens(input_ids)
227
+ for i in range(len(self.layers)):
228
+ layer = self.layers[i]
229
+ hidden_states = layer(positions, hidden_states, kv_caches[i],
230
+ input_metadata)
231
+ hidden_states = self.norm(hidden_states)
232
+ return hidden_states
233
+
234
+
235
+class Starcoder2ForCausalLM(nn.Module):
236
+
237
+ def __init__(self,
238
+ config: Starcoder2Config,
239
+ linear_method: Optional[LinearMethodBase] = None):
240
+ super().__init__()
241
+ self.config = config
242
+ self.model = Starcoder2Model(config, linear_method=linear_method)
243
+ self.vocab_size = config.vocab_size
244
+ self.unpadded_vocab_size = config.vocab_size
245
+ if config.tie_word_embeddings:
246
+ self.lm_head_weight = self.model.embed_tokens.weight
247
+ else:
248
+ self.unpadded_vocab_size = config.vocab_size
249
+ self.lm_head = ParallelLMHead(
250
+ self.unpadded_vocab_size,
251
+ config.hidden_size,
252
+ org_num_embeddings=config.vocab_size,
253
+ padding_size=DEFAULT_VOCAB_PADDING_SIZE,
254
+ )
255
+ self.lm_head_weight = self.lm_head.weight
256
+ self.sampler = Sampler(self.unpadded_vocab_size, config.vocab_size)
257
+
258
+ def forward(
259
+ self,
260
+ input_ids: torch.Tensor,
261
+ positions: torch.Tensor,
262
+ kv_caches: List[KVCache],
263
+ input_metadata: InputMetadata,
264
+ ) -> torch.Tensor:
265
+ hidden_states = self.model(input_ids, positions, kv_caches,
266
+ input_metadata)
267
+ return hidden_states
268
+
269
+ def sample(
270
+ self,
271
+ hidden_states: Optional[torch.Tensor],
272
+ sampling_metadata: SamplingMetadata,
273
+ ) -> Optional[SamplerOutput]:
274
+ next_tokens = self.sampler(self.lm_head_weight, hidden_states,
275
+ sampling_metadata)
276
+ return next_tokens
277
+
278
+ def load_weights(self,
279
+ model_name_or_path: str,
280
+ cache_dir: Optional[str] = None,
281
+ load_format: str = "auto",
282
+ revision: Optional[str] = None):
283
+ stacked_params_mapping = [
284
+ # (param_name, shard_name, shard_id)
285
+ ("qkv_proj", "q_proj", "q"),
286
+ ("qkv_proj", "k_proj", "k"),
287
+ ("qkv_proj", "v_proj", "v"),
288
+ ]
289
+
290
+ params_dict = dict(self.named_parameters(remove_duplicate=False))
291
+ for name, loaded_weight in hf_model_weights_iterator(
292
+ model_name_or_path, cache_dir, load_format, revision):
293
+ if "rotary_emb.inv_freq" in name:
294
+ continue
295
+
296
+ for (param_name, weight_name, shard_id) in stacked_params_mapping:
297
+ if weight_name not in name:
298
+ continue
299
+ name = name.replace(weight_name, param_name)
300
+ param = params_dict[name]
301
+ weight_loader = param.weight_loader
302
+ weight_loader(param, loaded_weight, shard_id)
303
+ break
304
+ else:
305
+ if self.config.tie_word_embeddings and "lm_head.weight" in name:
306
+ continue
307
+ param = params_dict[name]
308
+ weight_loader = getattr(param, "weight_loader",
309
+ default_weight_loader)
310
+ weight_loader(param, loaded_weight)
vllm/transformers_utils/config.py
+10
@@ -9,6 +9,7 @@ _CONFIG_REGISTRY = {
9
"mpt": MPTConfig,
10
"RefinedWeb": RWConfig, # For tiiuae/falcon-40b(-instruct)
11
"RefinedWebModel": RWConfig, # For tiiuae/falcon-7b(-instruct)
12
+ "starcoder2": Starcoder2Config,
13
}
14
15
@@ -16,6 +17,15 @@ def get_config(model: str,
17
trust_remote_code: bool,
18
revision: Optional[str] = None,
19
code_revision: Optional[str] = None) -> PretrainedConfig:
20
+ # FIXME(woosuk): This is a temporary fix for StarCoder2.
21
+ # Remove this when the model is supported by HuggingFace transformers.
22
+ if "bigcode" in model and "starcoder2" in model:
23
+ config_class = _CONFIG_REGISTRY["starcoder2"]
24
+ config = config_class.from_pretrained(model,
25
+ revision=revision,
26
+ code_revision=code_revision)
27
+ return config
28
+
29
try:
30
config = AutoConfig.from_pretrained(
31
model,
vllm/transformers_utils/configs/__init__.py
+2
@@ -4,9 +4,11 @@ from vllm.transformers_utils.configs.mpt import MPTConfig
4
# tiiuae/falcon-7b(-instruct) models. Newer Falcon models will use the
5
# `FalconConfig` class from the official HuggingFace transformers library.
6
from vllm.transformers_utils.configs.falcon import RWConfig
7
+from vllm.transformers_utils.configs.starcoder2 import Starcoder2Config
8
9
__all__ = [
10
"ChatGLMConfig",
11
"MPTConfig",
12
"RWConfig",
13
+ "Starcoder2Config",
14
]
vllm/transformers_utils/configs/starcoder2.py
new
+127
@@ -0,0 +1,127 @@
1
+from transformers import PretrainedConfig
2
+
3
+
4
+class Starcoder2Config(PretrainedConfig):
5
+ r"""
6
+ This is the configuration class to store the configuration of a [`Starcoder2Model`]. It is used to instantiate a
7
+ Starcoder2 model according to the specified arguments, defining the model architecture. Instantiating a configuration
8
+ with the defaults will yield a similar configuration to that of the [bigcode/starcoder2-7b_16k](https://huggingface.co/bigcode/starcoder2-7b_16k) model.
9
+
10
+
11
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
12
+ documentation from [`PretrainedConfig`] for more information.
13
+
14
+
15
+ Args:
16
+ vocab_size (`int`, *optional*, defaults to 49152):
17
+ Vocabulary size of the Starcoder2 model. Defines the number of different tokens that can be represented by the
18
+ `inputs_ids` passed when calling [`Starcoder2Model`]
19
+ hidden_size (`int`, *optional*, defaults to 3072):
20
+ Dimension of the hidden representations.
21
+ intermediate_size (`int`, *optional*, defaults to 12288):
22
+ Dimension of the MLP representations.
23
+ num_hidden_layers (`int`, *optional*, defaults to 30):
24
+ Number of hidden layers in the Transformer encoder.
25
+ num_attention_heads (`int`, *optional*, defaults to 24):
26
+ Number of attention heads for each attention layer in the Transformer encoder.
27
+ num_key_value_heads (`int`, *optional*, defaults to 2):
28
+ This is the number of key_value heads that should be used to implement Grouped Query Attention. If
29
+ `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
30
+ `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When
31
+ converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
32
+ by meanpooling all the original heads within that group. For more details checkout [this
33
+ paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to `8`.
34
+ hidden_act (`str` or `function`, *optional*, defaults to `"gelu_pytorch_tanh"`):
35
+ The non-linear activation function (function or string) in the decoder.
36
+ max_position_embeddings (`int`, *optional*, defaults to 4096):
37
+ The maximum sequence length that this model might ever be used with. Starcoder2's sliding window attention
38
+ allows sequence of up to 4096*32 tokens.
39
+ initializer_range (`float`, *optional*, defaults to 0.02):
40
+ The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
41
+ norm_epsilon (`float`, *optional*, defaults to 1e-05):
42
+ Epsilon value for the layer norm
43
+ use_cache (`bool`, *optional*, defaults to `True`):
44
+ Whether or not the model should return the last key/values attentions (not used by all models). Only
45
+ relevant if `config.is_decoder=True`.
46
+ bos_token_id (`int`, *optional*, defaults to 50256):
47
+ The id of the "beginning-of-sequence" token.
48
+ eos_token_id (`int`, *optional*, defaults to 50256):
49
+ The id of the "end-of-sequence" token.
50
+ rope_theta (`float`, *optional*, defaults to 10000.0):
51
+ The base period of the RoPE embeddings.
52
+ sliding_window (`int`, *optional*):
53
+ Sliding window attention window size. If not specified, will default to `None` (no sliding window).
54
+ attention_dropout (`float`, *optional*, defaults to 0.0):
55
+ The dropout ratio for the attention probabilities.
56
+ residual_dropout (`float`, *optional*, defaults to 0.0):
57
+ Residual connection dropout value.
58
+ embedding_dropout (`float`, *optional*, defaults to 0.0):
59
+ Embedding dropout.
60
+ use_bias (`bool`, *optional*, defaults to `True`):
61
+ Whether to use bias term on linear layers of the model.
62
+
63
+
64
+ ```python
65
+ >>> from transformers import Starcoder2Model, Starcoder2Config
66
+
67
+ >>> # Initializing a Starcoder2 7B style configuration
68
+ >>> configuration = Starcoder2Config()
69
+
70
+ >>> # Initializing a model from the Starcoder2 7B style configuration
71
+ >>> model = Starcoder2Model(configuration)
72
+
73
+ >>> # Accessing the model configuration
74
+ >>> configuration = model.config
75
+ ```"""
76
+
77
+ model_type = "starcoder2"
78
+ keys_to_ignore_at_inference = ["past_key_values"]
79
+
80
+ def __init__(
81
+ self,
82
+ vocab_size=49152,
83
+ hidden_size=3072,
84
+ intermediate_size=12288,
85
+ num_hidden_layers=30,
86
+ num_attention_heads=24,
87
+ num_key_value_heads=2,
88
+ hidden_act="gelu_pytorch_tanh",
89
+ max_position_embeddings=4096,
90
+ initializer_range=0.018042,
91
+ norm_epsilon=1e-5,
92
+ use_cache=True,
93
+ bos_token_id=50256,
94
+ eos_token_id=50256,
95
+ rope_theta=10000.0,
96
+ sliding_window=None,
97
+ attention_dropout=0.0,
98
+ residual_dropout=0.0,
99
+ embedding_dropout=0.0,
100
+ use_bias=True,
101
+ **kwargs,
102
+ ):
103
+ self.vocab_size = vocab_size
104
+ self.max_position_embeddings = max_position_embeddings
105
+ self.hidden_size = hidden_size
106
+ self.intermediate_size = intermediate_size
107
+ self.num_hidden_layers = num_hidden_layers
108
+ self.num_attention_heads = num_attention_heads
109
+ self.sliding_window = sliding_window
110
+ self.use_bias = use_bias
111
+ self.num_key_value_heads = num_key_value_heads
112
+ self.hidden_act = hidden_act
113
+ self.initializer_range = initializer_range
114
+ self.norm_epsilon = norm_epsilon
115
+ self.use_cache = use_cache
116
+ self.rope_theta = rope_theta
117
+ self.attention_dropout = attention_dropout
118
+ self.residual_dropout = residual_dropout
119
+ self.embedding_dropout = embedding_dropout
120
+
121
+ super().__init__(
122
+ bos_token_id=bos_token_id,
123
+ eos_token_id=eos_token_id,
124
+ **kwargs,
125
+ )
126
+ if self.architectures is None:
127
+ self.architectures = ['Starcoder2ForCausalLM']