A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Add StableLM3B model (#2372)
Hyunsung Lee committed
Jan 17, 2024 at 13:32 UTC
e1957c6ebdd4860f832c26ae4de4195d10803723
5 files changed
+310
-13
README.md
+1
@@ -77,6 +77,7 @@ vLLM seamlessly supports many Hugging Face models, including the following archi
77
- OPT (`facebook/opt-66b`, `facebook/opt-iml-max-30b`, etc.)
78
- Phi (`microsoft/phi-1_5`, `microsoft/phi-2`, etc.)
79
- Qwen (`Qwen/Qwen-7B`, `Qwen/Qwen-7B-Chat`, etc.)
80
+- StableLM(`stabilityai/stablelm-3b-4e1t`, `stabilityai/stablelm-base-alpha-7b-v2`, etc.)
81
- Yi (`01-ai/Yi-6B`, `01-ai/Yi-34B`, etc.)
82
83
Install vLLM with pip or [from source](https://vllm.readthedocs.io/en/latest/getting_started/installation.html#build-from-source):
docs/source/models/supported_models.rst
+3
@@ -68,6 +68,9 @@ Alongside each architecture, we include some popular models that use it.
68
* - :code:`QWenLMHeadModel`
69
- Qwen
70
- :code:`Qwen/Qwen-7B`, :code:`Qwen/Qwen-7B-Chat`, etc.
71
+ * - :code:`StableLMEpochForCausalLM`
72
+ - StableLM
73
+ - :code:`stabilityai/stablelm-3b-4e1t/` , :code:`stabilityai/stablelm-base-alpha-7b-v2`, etc.
74
* - :code:`YiForCausalLM`
75
- Yi
76
- :code:`01-ai/Yi-6B`, :code:`01-ai/Yi-34B`, etc.
tests/models/test_models.py
+5
-12
@@ -5,18 +5,11 @@ Run `pytest tests/models/test_models.py --forked`.
5
import pytest
6
7
MODELS = [
8
- "facebook/opt-125m",
9
- "meta-llama/Llama-2-7b-hf",
10
- "mistralai/Mistral-7B-v0.1",
11
- "Deci/DeciLM-7b",
12
- "tiiuae/falcon-7b",
13
- "gpt2",
14
- "bigcode/tiny_starcoder_py",
15
- "EleutherAI/gpt-j-6b",
16
- "EleutherAI/pythia-70m",
17
- "bigscience/bloom-560m",
18
- "mosaicml/mpt-7b",
19
- "microsoft/phi-2",
8
+ "facebook/opt-125m", "meta-llama/Llama-2-7b-hf",
9
+ "mistralai/Mistral-7B-v0.1", "Deci/DeciLM-7b", "tiiuae/falcon-7b", "gpt2",
10
+ "bigcode/tiny_starcoder_py", "EleutherAI/gpt-j-6b",
11
+ "EleutherAI/pythia-70m", "bigscience/bloom-560m", "mosaicml/mpt-7b",
12
+ "microsoft/phi-2", "stabilityai/stablelm-3b-4e1t"
13
]
14
15
vllm/model_executor/models/__init__.py
+2
-1
@@ -36,7 +36,8 @@ _MODELS = {
36
"PhiForCausalLM": ("phi", "PhiForCausalLM"),
37
"QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
38
"RWForCausalLM": ("falcon", "FalconForCausalLM"),
39
- "YiForCausalLM": ("yi", "YiForCausalLM"),
39
+ "StableLMEpochForCausalLM": ("stablelm", "StablelmForCausalLM"),
40
+ "YiForCausalLM": ("yi", "YiForCausalLM")
41
}
42
43
# Models not supported by ROCm.
vllm/model_executor/models/stablelm.py
new
+299
@@ -0,0 +1,299 @@
1
+# coding=utf-8
2
+# Copyright 2023 Stability AI, EleutherAI, and The HuggingFace Inc. team. All rights reserved.
3
+#
4
+# Licensed under the Apache License, Version 2.0 (the "License");
5
+# you may not use this file except in compliance with the License.
6
+# You may obtain a copy of the License at
7
+#
8
+# http://www.apache.org/licenses/LICENSE-2.0
9
+#
10
+# Unless required by applicable law or agreed to in writing, software
11
+# distributed under the License is distributed on an "AS IS" BASIS,
12
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+# See the License for the specific language governing permissions and
14
+# limitations under the License.
15
+#
16
+# This code is based off the following work:
17
+# https://huggingface.co/stabilityai/stablelm-3b-4e1t/blob/main/modeling_stablelm_epoch.py
18
+# https://huggingface.co/stabilityai/stablelm-3b-4e1t/blob/main/config.json
19
+"""Inference-only StabeLM (https://github.com/Stability-AI/StableLM) model compatible with HuggingFace weights."""
20
+from typing import List, Optional, Tuple
21
+
22
+import torch
23
+from torch import nn
24
+from transformers import PretrainedConfig
25
+
26
+from vllm.model_executor.input_metadata import InputMetadata
27
+from vllm.model_executor.layers.activation import SiluAndMul
28
+from vllm.model_executor.layers.attention import PagedAttention
29
+from vllm.model_executor.layers.linear import (LinearMethodBase,
30
+ MergedColumnParallelLinear,
31
+ QKVParallelLinear,
32
+ RowParallelLinear)
33
+from vllm.model_executor.layers.rotary_embedding import get_rope
34
+from vllm.model_executor.layers.sampler import Sampler
35
+from vllm.model_executor.layers.vocab_parallel_embedding import (
36
+ VocabParallelEmbedding, ParallelLMHead)
37
+from vllm.model_executor.parallel_utils.parallel_state import (
38
+ get_tensor_model_parallel_world_size)
39
+from vllm.model_executor.sampling_metadata import SamplingMetadata
40
+from vllm.model_executor.weight_utils import (default_weight_loader,
41
+ hf_model_weights_iterator)
42
+from vllm.sequence import SamplerOutput
43
+
44
+KVCache = Tuple[torch.Tensor, torch.Tensor]
45
+
46
+
47
+class StablelmMLP(nn.Module):
48
+
49
+ def __init__(self,
50
+ config: PretrainedConfig,
51
+ linear_method: Optional[LinearMethodBase] = None) -> None:
52
+ super().__init__()
53
+ self.config = config
54
+ self.hidden_size = config.hidden_size
55
+ self.intermediate_size = config.intermediate_size
56
+ self.gate_up_proj = MergedColumnParallelLinear(
57
+ config.hidden_size, [config.intermediate_size] * 2,
58
+ bias=False,
59
+ linear_method=linear_method)
60
+ self.down_proj = RowParallelLinear(config.intermediate_size,
61
+ config.hidden_size,
62
+ bias=False)
63
+ self.act_fn = SiluAndMul()
64
+
65
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
66
+ gate_up, _ = self.gate_up_proj(x)
67
+ x = self.act_fn(gate_up)
68
+ x, _ = self.down_proj(x)
69
+ return x
70
+
71
+
72
+class StablelmAttention(nn.Module):
73
+
74
+ def __init__(self,
75
+ config: PretrainedConfig,
76
+ linear_method: Optional[LinearMethodBase] = None) -> None:
77
+ super().__init__()
78
+ self.config = config
79
+ self.hidden_size = config.hidden_size
80
+ tp_size = get_tensor_model_parallel_world_size()
81
+ self.total_num_heads = config.num_attention_heads
82
+ self.num_heads = self.total_num_heads // tp_size
83
+
84
+ self.total_num_key_value_heads = config.num_key_value_heads
85
+ if self.total_num_key_value_heads >= tp_size:
86
+ # Number of KV heads is greater than TP size, so we partition
87
+ # the KV heads across multiple tensor parallel GPUs.
88
+ assert self.total_num_key_value_heads % tp_size == 0
89
+ else:
90
+ # Number of KV heads is less than TP size, so we replicate
91
+ # the KV heads across multiple tensor parallel GPUs.
92
+ assert tp_size % self.total_num_key_value_heads == 0
93
+ self.num_key_value_heads = max(
94
+ 1, self.total_num_key_value_heads // tp_size)
95
+ self.head_dim = self.hidden_size // self.total_num_heads
96
+ self.max_position_embeddings = config.max_position_embeddings
97
+ self.rotary_ndims = int(self.head_dim * self.config.rope_pct)
98
+ self.scaling = self.head_dim**-0.5
99
+ self.q_size = self.num_heads * self.head_dim
100
+ self.kv_size = self.num_key_value_heads * self.head_dim
101
+
102
+ if (self.head_dim * self.num_heads) != self.hidden_size:
103
+ raise ValueError(
104
+ f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}"
105
+ f" and `num_heads`: {self.num_heads}).")
106
+
107
+ self.qkv_proj = QKVParallelLinear(self.hidden_size,
108
+ self.head_dim,
109
+ self.total_num_heads,
110
+ self.total_num_key_value_heads,
111
+ bias=False,
112
+ linear_method=linear_method)
113
+ self.o_proj = RowParallelLinear(self.total_num_heads * self.head_dim,
114
+ self.hidden_size,
115
+ bias=False,
116
+ linear_method=linear_method)
117
+ self.rotary_ndims = int(self.head_dim * self.config.rope_pct)
118
+ self.rotary_emb = get_rope(
119
+ self.head_dim,
120
+ rotary_dim=self.rotary_ndims,
121
+ max_position=self.config.max_position_embeddings,
122
+ base=self.config.rope_theta,
123
+ )
124
+ self.attn = PagedAttention(self.num_heads,
125
+ self.head_dim,
126
+ self.scaling,
127
+ num_kv_heads=self.num_key_value_heads)
128
+
129
+ def forward(
130
+ self,
131
+ positions: torch.Tensor,
132
+ hidden_states: torch.Tensor,
133
+ kv_cache: KVCache,
134
+ input_metadata: InputMetadata,
135
+ ) -> torch.Tensor:
136
+ qkv, _ = self.qkv_proj(hidden_states)
137
+ q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
138
+ q, k = self.rotary_emb(positions, q, k)
139
+ k_cache, v_cache = kv_cache
140
+ attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
141
+ output, _ = self.o_proj(attn_output)
142
+ return output
143
+
144
+
145
+class StablelmDecoderLayer(nn.Module):
146
+
147
+ def __init__(
148
+ self,
149
+ config: PretrainedConfig,
150
+ linear_method: Optional[LinearMethodBase] = None,
151
+ ) -> None:
152
+ super().__init__()
153
+ self.self_attn = StablelmAttention(config)
154
+ self.mlp = StablelmMLP(config, linear_method)
155
+ self.input_layernorm = nn.LayerNorm(config.hidden_size,
156
+ eps=config.norm_eps)
157
+ self.post_attention_layernorm = nn.LayerNorm(config.hidden_size,
158
+ eps=config.norm_eps)
159
+
160
+ def forward(
161
+ self,
162
+ positions: torch.Tensor,
163
+ hidden_states: torch.Tensor,
164
+ kv_cache: KVCache,
165
+ input_metadata: InputMetadata,
166
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
167
+ # Self Attention
168
+ residual = hidden_states
169
+ hidden_states = self.input_layernorm(hidden_states)
170
+ hidden_states = self.self_attn(
171
+ positions=positions,
172
+ hidden_states=hidden_states,
173
+ kv_cache=kv_cache,
174
+ input_metadata=input_metadata,
175
+ )
176
+ hidden_states = residual + hidden_states
177
+
178
+ # Fully Connected
179
+ residual = hidden_states
180
+ hidden_states = self.post_attention_layernorm(hidden_states)
181
+ hidden_states = self.mlp(hidden_states)
182
+ hidden_states = residual + hidden_states
183
+
184
+ return hidden_states, residual
185
+
186
+
187
+class StableLMEpochModel(nn.Module):
188
+
189
+ def __init__(self,
190
+ config: PretrainedConfig,
191
+ linear_method: Optional[LinearMethodBase] = None) -> None:
192
+ super().__init__()
193
+ # self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, config.pad_token_id)
194
+ self.embed_tokens = VocabParallelEmbedding(
195
+ config.vocab_size,
196
+ config.hidden_size,
197
+ )
198
+ self.layers = nn.ModuleList([
199
+ StablelmDecoderLayer(config, linear_method)
200
+ for _ in range(config.num_hidden_layers)
201
+ ])
202
+ self.norm = nn.LayerNorm(config.hidden_size, eps=config.norm_eps)
203
+
204
+ def forward(
205
+ self,
206
+ input_ids: torch.Tensor,
207
+ positions: torch.Tensor,
208
+ kv_caches: List[KVCache],
209
+ input_metadata: InputMetadata,
210
+ ) -> torch.Tensor:
211
+ hidden_states = self.embed_tokens(input_ids)
212
+ for i in range(len(self.layers)):
213
+ layer = self.layers[i]
214
+ hidden_states, residual = layer(
215
+ positions,
216
+ hidden_states,
217
+ kv_caches[i],
218
+ input_metadata,
219
+ )
220
+ hidden_states = self.norm(hidden_states)
221
+ return hidden_states
222
+
223
+
224
+class StablelmForCausalLM(nn.Module):
225
+
226
+ def __init__(
227
+ self,
228
+ config: PretrainedConfig,
229
+ linear_method: Optional[LinearMethodBase] = None,
230
+ ) -> None:
231
+ super().__init__()
232
+ self.config = config
233
+ self.linear_method = linear_method
234
+ self.model = StableLMEpochModel(config, linear_method)
235
+ self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
236
+ self.sampler = Sampler(config.vocab_size)
237
+
238
+ def forward(
239
+ self,
240
+ input_ids: torch.Tensor,
241
+ positions: torch.Tensor,
242
+ kv_caches: List[KVCache],
243
+ input_metadata: InputMetadata,
244
+ ) -> torch.Tensor:
245
+ hidden_states = self.model(input_ids, positions, kv_caches,
246
+ input_metadata)
247
+ return hidden_states
248
+
249
+ def sample(
250
+ self,
251
+ hidden_states: torch.Tensor,
252
+ sampling_metadata: SamplingMetadata,
253
+ ) -> Optional[SamplerOutput]:
254
+ next_tokens = self.sampler(self.lm_head.weight, hidden_states,
255
+ sampling_metadata)
256
+ return next_tokens
257
+
258
+ def load_weights(self,
259
+ model_name_or_path: str,
260
+ cache_dir: Optional[str] = None,
261
+ load_format: str = "auto",
262
+ revision: Optional[str] = None):
263
+ stacked_params_mapping = [
264
+ # (param_name, shard_name, shard_id)
265
+ ("qkv_proj", "q_proj", "q"),
266
+ ("qkv_proj", "k_proj", "k"),
267
+ ("qkv_proj", "v_proj", "v"),
268
+ ("gate_up_proj", "gate_proj", 0),
269
+ ("gate_up_proj", "up_proj", 1),
270
+ ]
271
+ params_dict = dict(self.named_parameters())
272
+ for name, loaded_weight in hf_model_weights_iterator(
273
+ model_name_or_path, cache_dir, load_format, revision):
274
+ if "rotary_emb.inv_freq" in name:
275
+ continue
276
+ if ("rotary_emb.cos_cached" in name
277
+ or "rotary_emb.sin_cached" in name):
278
+ # Models trained using ColossalAI may include these tensors in
279
+ # the checkpoint. Skip them.
280
+ continue
281
+ for (param_name, weight_name, shard_id) in stacked_params_mapping:
282
+ if weight_name not in name:
283
+ continue
284
+ name = name.replace(weight_name, param_name)
285
+ # Skip loading extra bias for GPTQ models.
286
+ if name.endswith(".bias") and name not in params_dict:
287
+ continue
288
+ param = params_dict[name]
289
+ weight_loader = param.weight_loader
290
+ weight_loader(param, loaded_weight, shard_id)
291
+ break
292
+ else:
293
+ # Skip loading extra bias for GPTQ models.
294
+ if name.endswith(".bias") and name not in params_dict:
295
+ continue
296
+ param = params_dict[name]
297
+ weight_loader = getattr(param, "weight_loader",
298
+ default_weight_loader)
299
+ weight_loader(param, loaded_weight)