@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bfdcfa6a05
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Support starcoder2 architecture (#3089)

Seonghyeon committed Feb 29, 2024 at 17:51 UTC bfdcfa6a053c693800551bd1bd71acabbe1941e8
7 files changed +452
README.md
+1
@@ -78,6 +78,7 @@ vLLM seamlessly supports many Hugging Face models, including the following archi
78 - Qwen (`Qwen/Qwen-7B`, `Qwen/Qwen-7B-Chat`, etc.)
79 - Qwen2 (`Qwen/Qwen2-7B-beta`, `Qwen/Qwen-7B-Chat-beta`, etc.)
80 - StableLM(`stabilityai/stablelm-3b-4e1t`, `stabilityai/stablelm-base-alpha-7b-v2`, etc.)
81 +- Starcoder2(`bigcode/starcoder2-3b`, `bigcode/starcoder2-7b`, `bigcode/starcoder2-15b`, etc.)
82 - Yi (`01-ai/Yi-6B`, `01-ai/Yi-34B`, etc.)
83
84 Install vLLM with pip or [from source](https://vllm.readthedocs.io/en/latest/getting_started/installation.html#build-from-source):
tests/models/test_models.py
+1
@@ -19,6 +19,7 @@ MODELS = [
19 "microsoft/phi-2",
20 "stabilityai/stablelm-3b-4e1t",
21 "allenai/OLMo-1B",
22 + "bigcode/starcoder2-3b",
23 ]
24
25
vllm/model_executor/models/__init__.py
+1
@@ -45,6 +45,7 @@ _MODELS = {
45 "RWForCausalLM": ("falcon", "FalconForCausalLM"),
46 "StableLMEpochForCausalLM": ("stablelm", "StablelmForCausalLM"),
47 "StableLmForCausalLM": ("stablelm", "StablelmForCausalLM"),
48 + "Starcoder2ForCausalLM": ("starcoder2", "Starcoder2ForCausalLM"),
49 }
50
51 # Models not supported by ROCm.
vllm/model_executor/models/starcoder2.py new
+310
@@ -0,0 +1,310 @@
1 +# coding=utf-8
2 +# Copyright 2024 BigCode and the HuggingFace Inc. team. All rights reserved.
3 +#
4 +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
5 +# and OPT implementations in this library. It has been modified from its
6 +# original forms to accommodate minor architectural differences compared
7 +# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
8 +#
9 +# Licensed under the Apache License, Version 2.0 (the "License");
10 +# you may not use this file except in compliance with the License.
11 +# You may obtain a copy of the License at
12 +#
13 +# http://www.apache.org/licenses/LICENSE-2.0
14 +#
15 +# Unless required by applicable law or agreed to in writing, software
16 +# distributed under the License is distributed on an "AS IS" BASIS,
17 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18 +# See the License for the specific language governing permissions and
19 +# limitations under the License.
20 +""" PyTorch Starcoder2 model."""
21 +from typing import List, Optional, Tuple
22 +
23 +import torch
24 +from torch import nn
25 +
26 +from vllm.model_executor.input_metadata import InputMetadata
27 +from vllm.model_executor.sampling_metadata import SamplingMetadata
28 +from vllm.model_executor.layers.attention import PagedAttention
29 +from vllm.model_executor.layers.activation import get_act_fn
30 +from vllm.model_executor.layers.rotary_embedding import get_rope
31 +from vllm.model_executor.layers.linear import (ColumnParallelLinear,
32 + LinearMethodBase,
33 + QKVParallelLinear,
34 + RowParallelLinear)
35 +from vllm.model_executor.layers.sampler import Sampler
36 +from vllm.model_executor.layers.vocab_parallel_embedding import (
37 + VocabParallelEmbedding, ParallelLMHead, DEFAULT_VOCAB_PADDING_SIZE)
38 +from vllm.model_executor.parallel_utils.parallel_state import get_tensor_model_parallel_world_size
39 +from vllm.model_executor.weight_utils import (default_weight_loader,
40 + hf_model_weights_iterator)
41 +from vllm.sequence import SamplerOutput
42 +
43 +try:
44 + from transformers import Starcoder2Config
45 +except ImportError:
46 + # fallback to PretrainedConfig
47 + # NOTE: Please install transformers from source or use transformers>=4.39.0
48 + from transformers import PretrainedConfig as Starcoder2Config
49 +
50 +KVCache = Tuple[torch.Tensor, torch.Tensor]
51 +
52 +
53 +class Starcoder2Attention(nn.Module):
54 +
55 + def __init__(self,
56 + config: Starcoder2Config,
57 + linear_method: Optional[LinearMethodBase] = None):
58 + super().__init__()
59 + self.config = config
60 +
61 + self.hidden_size = config.hidden_size
62 + tp_size = get_tensor_model_parallel_world_size()
63 + self.total_num_heads = config.num_attention_heads
64 + assert self.total_num_heads % tp_size == 0
65 + self.num_heads = self.total_num_heads // tp_size
66 + self.total_num_kv_heads = config.num_key_value_heads
67 + if self.total_num_kv_heads >= tp_size:
68 + # Number of KV heads is greater than TP size, so we partition
69 + # the KV heads across multiple tensor parallel GPUs.
70 + assert self.total_num_kv_heads % tp_size == 0
71 + else:
72 + # Number of KV heads is less than TP size, so we replicate
73 + # the KV heads across multiple tensor parallel GPUs.
74 + assert tp_size % self.total_num_kv_heads == 0
75 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
76 + self.head_dim = self.hidden_size // self.total_num_heads
77 + self.q_size = self.num_heads * self.head_dim
78 + self.kv_size = self.num_kv_heads * self.head_dim
79 + self.scaling = self.head_dim**-0.5
80 + self.rope_theta = config.rope_theta
81 + self.max_position_embeddings = config.max_position_embeddings
82 + self.use_bias = config.use_bias
83 + self.sliding_window = config.sliding_window
84 +
85 + self.qkv_proj = QKVParallelLinear(
86 + self.hidden_size,
87 + self.head_dim,
88 + self.total_num_heads,
89 + self.total_num_kv_heads,
90 + bias=self.use_bias,
91 + linear_method=linear_method,
92 + )
93 + self.o_proj = RowParallelLinear(
94 + self.total_num_heads * self.head_dim,
95 + self.hidden_size,
96 + bias=self.use_bias,
97 + linear_method=linear_method,
98 + )
99 + self.rotary_emb = get_rope(
100 + self.head_dim,
101 + rotary_dim=self.head_dim,
102 + max_position=self.max_position_embeddings,
103 + base=int(self.rope_theta),
104 + is_neox_style=True,
105 + )
106 + self.attn = PagedAttention(
107 + self.num_heads,
108 + self.head_dim,
109 + self.scaling,
110 + num_kv_heads=self.num_kv_heads,
111 + sliding_window=self.sliding_window,
112 + )
113 +
114 + def forward(
115 + self,
116 + positions: torch.Tensor,
117 + hidden_states: torch.Tensor,
118 + kv_cache: KVCache,
119 + input_metadata: InputMetadata,
120 + ) -> torch.Tensor:
121 + qkv, _ = self.qkv_proj(hidden_states)
122 + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
123 + q, k = self.rotary_emb(positions, q, k)
124 + k_cache, v_cache = kv_cache
125 + attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
126 + output, _ = self.o_proj(attn_output)
127 + return output
128 +
129 +
130 +class Starcoder2MLP(nn.Module):
131 +
132 + def __init__(self,
133 + config: Starcoder2Config,
134 + linear_method: Optional[LinearMethodBase] = None):
135 + super().__init__()
136 + self.c_fc = ColumnParallelLinear(
137 + config.hidden_size,
138 + config.intermediate_size,
139 + bias=config.use_bias,
140 + linear_method=linear_method,
141 + )
142 + self.c_proj = RowParallelLinear(
143 + config.intermediate_size,
144 + config.hidden_size,
145 + bias=config.use_bias,
146 + linear_method=linear_method,
147 + )
148 + self.act = get_act_fn(config.hidden_act,
149 + intermediate_size=config.intermediate_size)
150 +
151 + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
152 + hidden_states, _ = self.c_fc(hidden_states)
153 + hidden_states = self.act(hidden_states)
154 + hidden_states, _ = self.c_proj(hidden_states)
155 + return hidden_states
156 +
157 +
158 +class Starcoder2DecoderLayer(nn.Module):
159 +
160 + def __init__(self,
161 + config: Starcoder2Config,
162 + linear_method: Optional[LinearMethodBase] = None):
163 + super().__init__()
164 + self.hidden_size = config.hidden_size
165 + self.self_attn = Starcoder2Attention(config,
166 + linear_method=linear_method)
167 + self.mlp = Starcoder2MLP(config, linear_method=linear_method)
168 + self.input_layernorm = nn.LayerNorm(config.hidden_size,
169 + eps=config.norm_epsilon)
170 + self.post_attention_layernorm = nn.LayerNorm(config.hidden_size,
171 + eps=config.norm_epsilon)
172 +
173 + def forward(
174 + self,
175 + positions: torch.Tensor,
176 + hidden_states: torch.Tensor,
177 + kv_cache: KVCache,
178 + input_metadata: InputMetadata,
179 + ) -> torch.Tensor:
180 + # Self Attention
181 + residual = hidden_states
182 + hidden_states = self.input_layernorm(hidden_states)
183 + hidden_states = self.self_attn(
184 + positions=positions,
185 + hidden_states=hidden_states,
186 + kv_cache=kv_cache,
187 + input_metadata=input_metadata,
188 + )
189 + hidden_states = residual + hidden_states
190 +
191 + # Fully Connected
192 + residual = hidden_states
193 + hidden_states = self.post_attention_layernorm(hidden_states)
194 + hidden_states = self.mlp(hidden_states)
195 + hidden_states = residual + hidden_states
196 +
197 + return hidden_states
198 +
199 +
200 +class Starcoder2Model(nn.Module):
201 +
202 + def __init__(self,
203 + config: Starcoder2Config,
204 + linear_method: Optional[LinearMethodBase] = None):
205 + super().__init__()
206 + self.config = config
207 + self.padding_idx = config.pad_token_id
208 + self.vocab_size = config.vocab_size
209 +
210 + # TODO: consider padding_idx (currently removed)
211 + self.embed_tokens = VocabParallelEmbedding(config.vocab_size,
212 + config.hidden_size)
213 + self.layers = nn.ModuleList([
214 + Starcoder2DecoderLayer(config, linear_method=linear_method)
215 + for _ in range(config.num_hidden_layers)
216 + ])
217 + self.norm = nn.LayerNorm(config.hidden_size, eps=config.norm_epsilon)
218 +
219 + def forward(
220 + self,
221 + input_ids: torch.Tensor,
222 + positions: torch.Tensor,
223 + kv_caches: List[KVCache],
224 + input_metadata: InputMetadata,
225 + ) -> torch.Tensor:
226 + hidden_states = self.embed_tokens(input_ids)
227 + for i in range(len(self.layers)):
228 + layer = self.layers[i]
229 + hidden_states = layer(positions, hidden_states, kv_caches[i],
230 + input_metadata)
231 + hidden_states = self.norm(hidden_states)
232 + return hidden_states
233 +
234 +
235 +class Starcoder2ForCausalLM(nn.Module):
236 +
237 + def __init__(self,
238 + config: Starcoder2Config,
239 + linear_method: Optional[LinearMethodBase] = None):
240 + super().__init__()
241 + self.config = config
242 + self.model = Starcoder2Model(config, linear_method=linear_method)
243 + self.vocab_size = config.vocab_size
244 + self.unpadded_vocab_size = config.vocab_size
245 + if config.tie_word_embeddings:
246 + self.lm_head_weight = self.model.embed_tokens.weight
247 + else:
248 + self.unpadded_vocab_size = config.vocab_size
249 + self.lm_head = ParallelLMHead(
250 + self.unpadded_vocab_size,
251 + config.hidden_size,
252 + org_num_embeddings=config.vocab_size,
253 + padding_size=DEFAULT_VOCAB_PADDING_SIZE,
254 + )
255 + self.lm_head_weight = self.lm_head.weight
256 + self.sampler = Sampler(self.unpadded_vocab_size, config.vocab_size)
257 +
258 + def forward(
259 + self,
260 + input_ids: torch.Tensor,
261 + positions: torch.Tensor,
262 + kv_caches: List[KVCache],
263 + input_metadata: InputMetadata,
264 + ) -> torch.Tensor:
265 + hidden_states = self.model(input_ids, positions, kv_caches,
266 + input_metadata)
267 + return hidden_states
268 +
269 + def sample(
270 + self,
271 + hidden_states: Optional[torch.Tensor],
272 + sampling_metadata: SamplingMetadata,
273 + ) -> Optional[SamplerOutput]:
274 + next_tokens = self.sampler(self.lm_head_weight, hidden_states,
275 + sampling_metadata)
276 + return next_tokens
277 +
278 + def load_weights(self,
279 + model_name_or_path: str,
280 + cache_dir: Optional[str] = None,
281 + load_format: str = "auto",
282 + revision: Optional[str] = None):
283 + stacked_params_mapping = [
284 + # (param_name, shard_name, shard_id)
285 + ("qkv_proj", "q_proj", "q"),
286 + ("qkv_proj", "k_proj", "k"),
287 + ("qkv_proj", "v_proj", "v"),
288 + ]
289 +
290 + params_dict = dict(self.named_parameters(remove_duplicate=False))
291 + for name, loaded_weight in hf_model_weights_iterator(
292 + model_name_or_path, cache_dir, load_format, revision):
293 + if "rotary_emb.inv_freq" in name:
294 + continue
295 +
296 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
297 + if weight_name not in name:
298 + continue
299 + name = name.replace(weight_name, param_name)
300 + param = params_dict[name]
301 + weight_loader = param.weight_loader
302 + weight_loader(param, loaded_weight, shard_id)
303 + break
304 + else:
305 + if self.config.tie_word_embeddings and "lm_head.weight" in name:
306 + continue
307 + param = params_dict[name]
308 + weight_loader = getattr(param, "weight_loader",
309 + default_weight_loader)
310 + weight_loader(param, loaded_weight)
vllm/transformers_utils/config.py
+10
@@ -9,6 +9,7 @@ _CONFIG_REGISTRY = {
9 "mpt": MPTConfig,
10 "RefinedWeb": RWConfig, # For tiiuae/falcon-40b(-instruct)
11 "RefinedWebModel": RWConfig, # For tiiuae/falcon-7b(-instruct)
12 + "starcoder2": Starcoder2Config,
13 }
14
15
@@ -16,6 +17,15 @@ def get_config(model: str,
17 trust_remote_code: bool,
18 revision: Optional[str] = None,
19 code_revision: Optional[str] = None) -> PretrainedConfig:
20 + # FIXME(woosuk): This is a temporary fix for StarCoder2.
21 + # Remove this when the model is supported by HuggingFace transformers.
22 + if "bigcode" in model and "starcoder2" in model:
23 + config_class = _CONFIG_REGISTRY["starcoder2"]
24 + config = config_class.from_pretrained(model,
25 + revision=revision,
26 + code_revision=code_revision)
27 + return config
28 +
29 try:
30 config = AutoConfig.from_pretrained(
31 model,
vllm/transformers_utils/configs/__init__.py
+2
@@ -4,9 +4,11 @@ from vllm.transformers_utils.configs.mpt import MPTConfig
4 # tiiuae/falcon-7b(-instruct) models. Newer Falcon models will use the
5 # `FalconConfig` class from the official HuggingFace transformers library.
6 from vllm.transformers_utils.configs.falcon import RWConfig
7 +from vllm.transformers_utils.configs.starcoder2 import Starcoder2Config
8
9 __all__ = [
10 "ChatGLMConfig",
11 "MPTConfig",
12 "RWConfig",
13 + "Starcoder2Config",
14 ]
vllm/transformers_utils/configs/starcoder2.py new
+127
@@ -0,0 +1,127 @@
1 +from transformers import PretrainedConfig
2 +
3 +
4 +class Starcoder2Config(PretrainedConfig):
5 + r"""
6 + This is the configuration class to store the configuration of a [`Starcoder2Model`]. It is used to instantiate a
7 + Starcoder2 model according to the specified arguments, defining the model architecture. Instantiating a configuration
8 + with the defaults will yield a similar configuration to that of the [bigcode/starcoder2-7b_16k](https://huggingface.co/bigcode/starcoder2-7b_16k) model.
9 +
10 +
11 + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
12 + documentation from [`PretrainedConfig`] for more information.
13 +
14 +
15 + Args:
16 + vocab_size (`int`, *optional*, defaults to 49152):
17 + Vocabulary size of the Starcoder2 model. Defines the number of different tokens that can be represented by the
18 + `inputs_ids` passed when calling [`Starcoder2Model`]
19 + hidden_size (`int`, *optional*, defaults to 3072):
20 + Dimension of the hidden representations.
21 + intermediate_size (`int`, *optional*, defaults to 12288):
22 + Dimension of the MLP representations.
23 + num_hidden_layers (`int`, *optional*, defaults to 30):
24 + Number of hidden layers in the Transformer encoder.
25 + num_attention_heads (`int`, *optional*, defaults to 24):
26 + Number of attention heads for each attention layer in the Transformer encoder.
27 + num_key_value_heads (`int`, *optional*, defaults to 2):
28 + This is the number of key_value heads that should be used to implement Grouped Query Attention. If
29 + `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
30 + `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When
31 + converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
32 + by meanpooling all the original heads within that group. For more details checkout [this
33 + paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to `8`.
34 + hidden_act (`str` or `function`, *optional*, defaults to `"gelu_pytorch_tanh"`):
35 + The non-linear activation function (function or string) in the decoder.
36 + max_position_embeddings (`int`, *optional*, defaults to 4096):
37 + The maximum sequence length that this model might ever be used with. Starcoder2's sliding window attention
38 + allows sequence of up to 4096*32 tokens.
39 + initializer_range (`float`, *optional*, defaults to 0.02):
40 + The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
41 + norm_epsilon (`float`, *optional*, defaults to 1e-05):
42 + Epsilon value for the layer norm
43 + use_cache (`bool`, *optional*, defaults to `True`):
44 + Whether or not the model should return the last key/values attentions (not used by all models). Only
45 + relevant if `config.is_decoder=True`.
46 + bos_token_id (`int`, *optional*, defaults to 50256):
47 + The id of the "beginning-of-sequence" token.
48 + eos_token_id (`int`, *optional*, defaults to 50256):
49 + The id of the "end-of-sequence" token.
50 + rope_theta (`float`, *optional*, defaults to 10000.0):
51 + The base period of the RoPE embeddings.
52 + sliding_window (`int`, *optional*):
53 + Sliding window attention window size. If not specified, will default to `None` (no sliding window).
54 + attention_dropout (`float`, *optional*, defaults to 0.0):
55 + The dropout ratio for the attention probabilities.
56 + residual_dropout (`float`, *optional*, defaults to 0.0):
57 + Residual connection dropout value.
58 + embedding_dropout (`float`, *optional*, defaults to 0.0):
59 + Embedding dropout.
60 + use_bias (`bool`, *optional*, defaults to `True`):
61 + Whether to use bias term on linear layers of the model.
62 +
63 +
64 + ```python
65 + >>> from transformers import Starcoder2Model, Starcoder2Config
66 +
67 + >>> # Initializing a Starcoder2 7B style configuration
68 + >>> configuration = Starcoder2Config()
69 +
70 + >>> # Initializing a model from the Starcoder2 7B style configuration
71 + >>> model = Starcoder2Model(configuration)
72 +
73 + >>> # Accessing the model configuration
74 + >>> configuration = model.config
75 + ```"""
76 +
77 + model_type = "starcoder2"
78 + keys_to_ignore_at_inference = ["past_key_values"]
79 +
80 + def __init__(
81 + self,
82 + vocab_size=49152,
83 + hidden_size=3072,
84 + intermediate_size=12288,
85 + num_hidden_layers=30,
86 + num_attention_heads=24,
87 + num_key_value_heads=2,
88 + hidden_act="gelu_pytorch_tanh",
89 + max_position_embeddings=4096,
90 + initializer_range=0.018042,
91 + norm_epsilon=1e-5,
92 + use_cache=True,
93 + bos_token_id=50256,
94 + eos_token_id=50256,
95 + rope_theta=10000.0,
96 + sliding_window=None,
97 + attention_dropout=0.0,
98 + residual_dropout=0.0,
99 + embedding_dropout=0.0,
100 + use_bias=True,
101 + **kwargs,
102 + ):
103 + self.vocab_size = vocab_size
104 + self.max_position_embeddings = max_position_embeddings
105 + self.hidden_size = hidden_size
106 + self.intermediate_size = intermediate_size
107 + self.num_hidden_layers = num_hidden_layers
108 + self.num_attention_heads = num_attention_heads
109 + self.sliding_window = sliding_window
110 + self.use_bias = use_bias
111 + self.num_key_value_heads = num_key_value_heads
112 + self.hidden_act = hidden_act
113 + self.initializer_range = initializer_range
114 + self.norm_epsilon = norm_epsilon
115 + self.use_cache = use_cache
116 + self.rope_theta = rope_theta
117 + self.attention_dropout = attention_dropout
118 + self.residual_dropout = residual_dropout
119 + self.embedding_dropout = embedding_dropout
120 +
121 + super().__init__(
122 + bos_token_id=bos_token_id,
123 + eos_token_id=eos_token_id,
124 + **kwargs,
125 + )
126 + if self.architectures is None:
127 + self.architectures = ['Starcoder2ForCausalLM']