@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e1957c6ebd
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Add StableLM3B model (#2372)

Hyunsung Lee committed Jan 17, 2024 at 13:32 UTC e1957c6ebdd4860f832c26ae4de4195d10803723
5 files changed +310 -13
README.md
+1
@@ -77,6 +77,7 @@ vLLM seamlessly supports many Hugging Face models, including the following archi
77 - OPT (`facebook/opt-66b`, `facebook/opt-iml-max-30b`, etc.)
78 - Phi (`microsoft/phi-1_5`, `microsoft/phi-2`, etc.)
79 - Qwen (`Qwen/Qwen-7B`, `Qwen/Qwen-7B-Chat`, etc.)
80 +- StableLM(`stabilityai/stablelm-3b-4e1t`, `stabilityai/stablelm-base-alpha-7b-v2`, etc.)
81 - Yi (`01-ai/Yi-6B`, `01-ai/Yi-34B`, etc.)
82
83 Install vLLM with pip or [from source](https://vllm.readthedocs.io/en/latest/getting_started/installation.html#build-from-source):
docs/source/models/supported_models.rst
+3
@@ -68,6 +68,9 @@ Alongside each architecture, we include some popular models that use it.
68 * - :code:`QWenLMHeadModel`
69 - Qwen
70 - :code:`Qwen/Qwen-7B`, :code:`Qwen/Qwen-7B-Chat`, etc.
71 + * - :code:`StableLMEpochForCausalLM`
72 + - StableLM
73 + - :code:`stabilityai/stablelm-3b-4e1t/` , :code:`stabilityai/stablelm-base-alpha-7b-v2`, etc.
74 * - :code:`YiForCausalLM`
75 - Yi
76 - :code:`01-ai/Yi-6B`, :code:`01-ai/Yi-34B`, etc.
tests/models/test_models.py
+5 -12
@@ -5,18 +5,11 @@ Run `pytest tests/models/test_models.py --forked`.
5 import pytest
6
7 MODELS = [
8 - "facebook/opt-125m",
9 - "meta-llama/Llama-2-7b-hf",
10 - "mistralai/Mistral-7B-v0.1",
11 - "Deci/DeciLM-7b",
12 - "tiiuae/falcon-7b",
13 - "gpt2",
14 - "bigcode/tiny_starcoder_py",
15 - "EleutherAI/gpt-j-6b",
16 - "EleutherAI/pythia-70m",
17 - "bigscience/bloom-560m",
18 - "mosaicml/mpt-7b",
19 - "microsoft/phi-2",
8 + "facebook/opt-125m", "meta-llama/Llama-2-7b-hf",
9 + "mistralai/Mistral-7B-v0.1", "Deci/DeciLM-7b", "tiiuae/falcon-7b", "gpt2",
10 + "bigcode/tiny_starcoder_py", "EleutherAI/gpt-j-6b",
11 + "EleutherAI/pythia-70m", "bigscience/bloom-560m", "mosaicml/mpt-7b",
12 + "microsoft/phi-2", "stabilityai/stablelm-3b-4e1t"
13 ]
14
15
vllm/model_executor/models/__init__.py
+2 -1
@@ -36,7 +36,8 @@ _MODELS = {
36 "PhiForCausalLM": ("phi", "PhiForCausalLM"),
37 "QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
38 "RWForCausalLM": ("falcon", "FalconForCausalLM"),
39 - "YiForCausalLM": ("yi", "YiForCausalLM"),
39 + "StableLMEpochForCausalLM": ("stablelm", "StablelmForCausalLM"),
40 + "YiForCausalLM": ("yi", "YiForCausalLM")
41 }
42
43 # Models not supported by ROCm.
vllm/model_executor/models/stablelm.py new
+299
@@ -0,0 +1,299 @@
1 +# coding=utf-8
2 +# Copyright 2023 Stability AI, EleutherAI, and The HuggingFace Inc. team. All rights reserved.
3 +#
4 +# Licensed under the Apache License, Version 2.0 (the "License");
5 +# you may not use this file except in compliance with the License.
6 +# You may obtain a copy of the License at
7 +#
8 +# http://www.apache.org/licenses/LICENSE-2.0
9 +#
10 +# Unless required by applicable law or agreed to in writing, software
11 +# distributed under the License is distributed on an "AS IS" BASIS,
12 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 +# See the License for the specific language governing permissions and
14 +# limitations under the License.
15 +#
16 +# This code is based off the following work:
17 +# https://huggingface.co/stabilityai/stablelm-3b-4e1t/blob/main/modeling_stablelm_epoch.py
18 +# https://huggingface.co/stabilityai/stablelm-3b-4e1t/blob/main/config.json
19 +"""Inference-only StabeLM (https://github.com/Stability-AI/StableLM) model compatible with HuggingFace weights."""
20 +from typing import List, Optional, Tuple
21 +
22 +import torch
23 +from torch import nn
24 +from transformers import PretrainedConfig
25 +
26 +from vllm.model_executor.input_metadata import InputMetadata
27 +from vllm.model_executor.layers.activation import SiluAndMul
28 +from vllm.model_executor.layers.attention import PagedAttention
29 +from vllm.model_executor.layers.linear import (LinearMethodBase,
30 + MergedColumnParallelLinear,
31 + QKVParallelLinear,
32 + RowParallelLinear)
33 +from vllm.model_executor.layers.rotary_embedding import get_rope
34 +from vllm.model_executor.layers.sampler import Sampler
35 +from vllm.model_executor.layers.vocab_parallel_embedding import (
36 + VocabParallelEmbedding, ParallelLMHead)
37 +from vllm.model_executor.parallel_utils.parallel_state import (
38 + get_tensor_model_parallel_world_size)
39 +from vllm.model_executor.sampling_metadata import SamplingMetadata
40 +from vllm.model_executor.weight_utils import (default_weight_loader,
41 + hf_model_weights_iterator)
42 +from vllm.sequence import SamplerOutput
43 +
44 +KVCache = Tuple[torch.Tensor, torch.Tensor]
45 +
46 +
47 +class StablelmMLP(nn.Module):
48 +
49 + def __init__(self,
50 + config: PretrainedConfig,
51 + linear_method: Optional[LinearMethodBase] = None) -> None:
52 + super().__init__()
53 + self.config = config
54 + self.hidden_size = config.hidden_size
55 + self.intermediate_size = config.intermediate_size
56 + self.gate_up_proj = MergedColumnParallelLinear(
57 + config.hidden_size, [config.intermediate_size] * 2,
58 + bias=False,
59 + linear_method=linear_method)
60 + self.down_proj = RowParallelLinear(config.intermediate_size,
61 + config.hidden_size,
62 + bias=False)
63 + self.act_fn = SiluAndMul()
64 +
65 + def forward(self, x: torch.Tensor) -> torch.Tensor:
66 + gate_up, _ = self.gate_up_proj(x)
67 + x = self.act_fn(gate_up)
68 + x, _ = self.down_proj(x)
69 + return x
70 +
71 +
72 +class StablelmAttention(nn.Module):
73 +
74 + def __init__(self,
75 + config: PretrainedConfig,
76 + linear_method: Optional[LinearMethodBase] = None) -> None:
77 + super().__init__()
78 + self.config = config
79 + self.hidden_size = config.hidden_size
80 + tp_size = get_tensor_model_parallel_world_size()
81 + self.total_num_heads = config.num_attention_heads
82 + self.num_heads = self.total_num_heads // tp_size
83 +
84 + self.total_num_key_value_heads = config.num_key_value_heads
85 + if self.total_num_key_value_heads >= tp_size:
86 + # Number of KV heads is greater than TP size, so we partition
87 + # the KV heads across multiple tensor parallel GPUs.
88 + assert self.total_num_key_value_heads % tp_size == 0
89 + else:
90 + # Number of KV heads is less than TP size, so we replicate
91 + # the KV heads across multiple tensor parallel GPUs.
92 + assert tp_size % self.total_num_key_value_heads == 0
93 + self.num_key_value_heads = max(
94 + 1, self.total_num_key_value_heads // tp_size)
95 + self.head_dim = self.hidden_size // self.total_num_heads
96 + self.max_position_embeddings = config.max_position_embeddings
97 + self.rotary_ndims = int(self.head_dim * self.config.rope_pct)
98 + self.scaling = self.head_dim**-0.5
99 + self.q_size = self.num_heads * self.head_dim
100 + self.kv_size = self.num_key_value_heads * self.head_dim
101 +
102 + if (self.head_dim * self.num_heads) != self.hidden_size:
103 + raise ValueError(
104 + f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}"
105 + f" and `num_heads`: {self.num_heads}).")
106 +
107 + self.qkv_proj = QKVParallelLinear(self.hidden_size,
108 + self.head_dim,
109 + self.total_num_heads,
110 + self.total_num_key_value_heads,
111 + bias=False,
112 + linear_method=linear_method)
113 + self.o_proj = RowParallelLinear(self.total_num_heads * self.head_dim,
114 + self.hidden_size,
115 + bias=False,
116 + linear_method=linear_method)
117 + self.rotary_ndims = int(self.head_dim * self.config.rope_pct)
118 + self.rotary_emb = get_rope(
119 + self.head_dim,
120 + rotary_dim=self.rotary_ndims,
121 + max_position=self.config.max_position_embeddings,
122 + base=self.config.rope_theta,
123 + )
124 + self.attn = PagedAttention(self.num_heads,
125 + self.head_dim,
126 + self.scaling,
127 + num_kv_heads=self.num_key_value_heads)
128 +
129 + def forward(
130 + self,
131 + positions: torch.Tensor,
132 + hidden_states: torch.Tensor,
133 + kv_cache: KVCache,
134 + input_metadata: InputMetadata,
135 + ) -> torch.Tensor:
136 + qkv, _ = self.qkv_proj(hidden_states)
137 + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
138 + q, k = self.rotary_emb(positions, q, k)
139 + k_cache, v_cache = kv_cache
140 + attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
141 + output, _ = self.o_proj(attn_output)
142 + return output
143 +
144 +
145 +class StablelmDecoderLayer(nn.Module):
146 +
147 + def __init__(
148 + self,
149 + config: PretrainedConfig,
150 + linear_method: Optional[LinearMethodBase] = None,
151 + ) -> None:
152 + super().__init__()
153 + self.self_attn = StablelmAttention(config)
154 + self.mlp = StablelmMLP(config, linear_method)
155 + self.input_layernorm = nn.LayerNorm(config.hidden_size,
156 + eps=config.norm_eps)
157 + self.post_attention_layernorm = nn.LayerNorm(config.hidden_size,
158 + eps=config.norm_eps)
159 +
160 + def forward(
161 + self,
162 + positions: torch.Tensor,
163 + hidden_states: torch.Tensor,
164 + kv_cache: KVCache,
165 + input_metadata: InputMetadata,
166 + ) -> Tuple[torch.Tensor, torch.Tensor]:
167 + # Self Attention
168 + residual = hidden_states
169 + hidden_states = self.input_layernorm(hidden_states)
170 + hidden_states = self.self_attn(
171 + positions=positions,
172 + hidden_states=hidden_states,
173 + kv_cache=kv_cache,
174 + input_metadata=input_metadata,
175 + )
176 + hidden_states = residual + hidden_states
177 +
178 + # Fully Connected
179 + residual = hidden_states
180 + hidden_states = self.post_attention_layernorm(hidden_states)
181 + hidden_states = self.mlp(hidden_states)
182 + hidden_states = residual + hidden_states
183 +
184 + return hidden_states, residual
185 +
186 +
187 +class StableLMEpochModel(nn.Module):
188 +
189 + def __init__(self,
190 + config: PretrainedConfig,
191 + linear_method: Optional[LinearMethodBase] = None) -> None:
192 + super().__init__()
193 + # self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, config.pad_token_id)
194 + self.embed_tokens = VocabParallelEmbedding(
195 + config.vocab_size,
196 + config.hidden_size,
197 + )
198 + self.layers = nn.ModuleList([
199 + StablelmDecoderLayer(config, linear_method)
200 + for _ in range(config.num_hidden_layers)
201 + ])
202 + self.norm = nn.LayerNorm(config.hidden_size, eps=config.norm_eps)
203 +
204 + def forward(
205 + self,
206 + input_ids: torch.Tensor,
207 + positions: torch.Tensor,
208 + kv_caches: List[KVCache],
209 + input_metadata: InputMetadata,
210 + ) -> torch.Tensor:
211 + hidden_states = self.embed_tokens(input_ids)
212 + for i in range(len(self.layers)):
213 + layer = self.layers[i]
214 + hidden_states, residual = layer(
215 + positions,
216 + hidden_states,
217 + kv_caches[i],
218 + input_metadata,
219 + )
220 + hidden_states = self.norm(hidden_states)
221 + return hidden_states
222 +
223 +
224 +class StablelmForCausalLM(nn.Module):
225 +
226 + def __init__(
227 + self,
228 + config: PretrainedConfig,
229 + linear_method: Optional[LinearMethodBase] = None,
230 + ) -> None:
231 + super().__init__()
232 + self.config = config
233 + self.linear_method = linear_method
234 + self.model = StableLMEpochModel(config, linear_method)
235 + self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
236 + self.sampler = Sampler(config.vocab_size)
237 +
238 + def forward(
239 + self,
240 + input_ids: torch.Tensor,
241 + positions: torch.Tensor,
242 + kv_caches: List[KVCache],
243 + input_metadata: InputMetadata,
244 + ) -> torch.Tensor:
245 + hidden_states = self.model(input_ids, positions, kv_caches,
246 + input_metadata)
247 + return hidden_states
248 +
249 + def sample(
250 + self,
251 + hidden_states: torch.Tensor,
252 + sampling_metadata: SamplingMetadata,
253 + ) -> Optional[SamplerOutput]:
254 + next_tokens = self.sampler(self.lm_head.weight, hidden_states,
255 + sampling_metadata)
256 + return next_tokens
257 +
258 + def load_weights(self,
259 + model_name_or_path: str,
260 + cache_dir: Optional[str] = None,
261 + load_format: str = "auto",
262 + revision: Optional[str] = None):
263 + stacked_params_mapping = [
264 + # (param_name, shard_name, shard_id)
265 + ("qkv_proj", "q_proj", "q"),
266 + ("qkv_proj", "k_proj", "k"),
267 + ("qkv_proj", "v_proj", "v"),
268 + ("gate_up_proj", "gate_proj", 0),
269 + ("gate_up_proj", "up_proj", 1),
270 + ]
271 + params_dict = dict(self.named_parameters())
272 + for name, loaded_weight in hf_model_weights_iterator(
273 + model_name_or_path, cache_dir, load_format, revision):
274 + if "rotary_emb.inv_freq" in name:
275 + continue
276 + if ("rotary_emb.cos_cached" in name
277 + or "rotary_emb.sin_cached" in name):
278 + # Models trained using ColossalAI may include these tensors in
279 + # the checkpoint. Skip them.
280 + continue
281 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
282 + if weight_name not in name:
283 + continue
284 + name = name.replace(weight_name, param_name)
285 + # Skip loading extra bias for GPTQ models.
286 + if name.endswith(".bias") and name not in params_dict:
287 + continue
288 + param = params_dict[name]
289 + weight_loader = param.weight_loader
290 + weight_loader(param, loaded_weight, shard_id)
291 + break
292 + else:
293 + # Skip loading extra bias for GPTQ models.
294 + if name.endswith(".bias") and name not in params_dict:
295 + continue
296 + param = params_dict[name]
297 + weight_loader = getattr(param, "weight_loader",
298 + default_weight_loader)
299 + weight_loader(param, loaded_weight)