@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ea356004d4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Revert "Refactor llama family models (#2637)" (#2851)

This reverts commit 5c976a7e1a1bec875bf6474824b7dff39e38de18.

Philipp Moritz committed Feb 13, 2024 at 09:24 UTC ea356004d4749627bd1c65b7f71c76f51b5c45be
17 files changed +2720 -236
vllm/model_executor/layers/layernorm.py
-25
@@ -7,31 +7,6 @@ import torch.nn as nn
7 from vllm._C import ops
8
9
10 -class LayerNorm(nn.LayerNorm):
11 -
12 - def __init__(
13 - self,
14 - hidden_size: int,
15 - eps: float = 1e-6,
16 - ) -> None:
17 - super().__init__(hidden_size, eps=eps)
18 -
19 - def forward(
20 - self,
21 - x: torch.Tensor,
22 - residual: Optional[torch.Tensor] = None,
23 - ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
24 - """normalization."""
25 - if residual is not None:
26 - x = x + residual
27 - residual = x
28 - x = super().forward(x)
29 - if residual is None:
30 - return x
31 - else:
32 - return x, residual
33 -
34 -
10 class RMSNorm(nn.Module):
11 """Root mean square normalization.
12
vllm/model_executor/models/__init__.py
+5 -4
@@ -10,8 +10,8 @@ logger = init_logger(__name__)
10
11 # Architecture -> (module, class).
12 _MODELS = {
13 - "AquilaModel": ("llama", "LlamaForCausalLM"),
14 - "AquilaForCausalLM": ("llama", "LlamaForCausalLM"), # AquilaChat2
13 + "AquilaModel": ("aquila", "AquilaForCausalLM"),
14 + "AquilaForCausalLM": ("aquila", "AquilaForCausalLM"), # AquilaChat2
15 "BaiChuanForCausalLM": ("baichuan", "BaiChuanForCausalLM"), # baichuan-7b
16 "BaichuanForCausalLM": ("baichuan", "BaichuanForCausalLM"), # baichuan-13b
17 "BloomForCausalLM": ("bloom", "BloomForCausalLM"),
@@ -24,12 +24,12 @@ _MODELS = {
24 "GPTBigCodeForCausalLM": ("gpt_bigcode", "GPTBigCodeForCausalLM"),
25 "GPTJForCausalLM": ("gpt_j", "GPTJForCausalLM"),
26 "GPTNeoXForCausalLM": ("gpt_neox", "GPTNeoXForCausalLM"),
27 - "InternLMForCausalLM": ("llama", "LlamaForCausalLM"),
27 + "InternLMForCausalLM": ("internlm", "InternLMForCausalLM"),
28 "InternLM2ForCausalLM": ("internlm2", "InternLM2ForCausalLM"),
29 "LlamaForCausalLM": ("llama", "LlamaForCausalLM"),
30 # For decapoda-research/llama-*
31 "LLaMAForCausalLM": ("llama", "LlamaForCausalLM"),
32 - "MistralForCausalLM": ("llama", "LlamaForCausalLM"),
32 + "MistralForCausalLM": ("mistral", "MistralForCausalLM"),
33 "MixtralForCausalLM": ("mixtral", "MixtralForCausalLM"),
34 "QuantMixtralForCausalLM": ("mixtral_quant", "MixtralForCausalLM"),
35 # transformers's mpt class has lower case
@@ -41,6 +41,7 @@ _MODELS = {
41 "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
42 "RWForCausalLM": ("falcon", "FalconForCausalLM"),
43 "StableLMEpochForCausalLM": ("stablelm", "StablelmForCausalLM"),
44 + "YiForCausalLM": ("yi", "YiForCausalLM")
45 }
46
47 # Models not supported by ROCm.
vllm/model_executor/models/aquila.py new
+342
@@ -0,0 +1,342 @@
1 +# coding=utf-8
2 +# Adapted from
3 +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
4 +# Copyright 2023 The vLLM team.
5 +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
6 +#
7 +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
8 +# and OPT implementations in this library. It has been modified from its
9 +# original forms to accommodate minor architectural differences compared
10 +# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
11 +#
12 +# Licensed under the Apache License, Version 2.0 (the "License");
13 +# you may not use this file except in compliance with the License.
14 +# You may obtain a copy of the License at
15 +#
16 +# http://www.apache.org/licenses/LICENSE-2.0
17 +#
18 +# Unless required by applicable law or agreed to in writing, software
19 +# distributed under the License is distributed on an "AS IS" BASIS,
20 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21 +# See the License for the specific language governing permissions and
22 +# limitations under the License.
23 +"""Inference-only LLaMA model compatible with HuggingFace weights."""
24 +from typing import Any, Dict, List, Optional, Tuple
25 +
26 +import torch
27 +from torch import nn
28 +
29 +from vllm.model_executor.input_metadata import InputMetadata
30 +from vllm.model_executor.layers.activation import SiluAndMul
31 +from vllm.model_executor.layers.attention import PagedAttention
32 +from vllm.model_executor.layers.linear import (LinearMethodBase,
33 + MergedColumnParallelLinear,
34 + QKVParallelLinear,
35 + RowParallelLinear)
36 +from vllm.model_executor.layers.rotary_embedding import get_rope
37 +from vllm.model_executor.layers.sampler import Sampler
38 +from vllm.model_executor.layers.vocab_parallel_embedding import (
39 + VocabParallelEmbedding, ParallelLMHead)
40 +from vllm.model_executor.parallel_utils.parallel_state import (
41 + get_tensor_model_parallel_world_size)
42 +from vllm.model_executor.sampling_metadata import SamplingMetadata
43 +from vllm.model_executor.weight_utils import (default_weight_loader,
44 + hf_model_weights_iterator)
45 +from vllm.sequence import SamplerOutput
46 +from vllm.transformers_utils.configs.aquila import AquilaConfig
47 +
48 +KVCache = Tuple[torch.Tensor, torch.Tensor]
49 +
50 +
51 +class AquilaMLP(nn.Module):
52 +
53 + def __init__(
54 + self,
55 + hidden_size: int,
56 + intermediate_size: int,
57 + hidden_act: str,
58 + linear_method: Optional[LinearMethodBase] = None,
59 + ):
60 + super().__init__()
61 + self.gate_up_proj = MergedColumnParallelLinear(
62 + hidden_size, [intermediate_size] * 2,
63 + bias=False,
64 + linear_method=linear_method)
65 + self.down_proj = RowParallelLinear(intermediate_size,
66 + hidden_size,
67 + bias=False,
68 + linear_method=linear_method)
69 + if hidden_act != "silu":
70 + raise ValueError(f"Unsupported activation: {hidden_act}. "
71 + "Only silu is supported for now.")
72 + self.act_fn = SiluAndMul()
73 +
74 + def forward(self, x):
75 + gate_up, _ = self.gate_up_proj(x)
76 + x = self.act_fn(gate_up)
77 + x, _ = self.down_proj(x)
78 + return x
79 +
80 +
81 +class AquilaRMSNorm(nn.Module):
82 +
83 + def __init__(self, hidden_size, eps=1e-6):
84 + """
85 + AquilaRMSNorm is equivalent to T5LayerNorm
86 + """
87 + super().__init__()
88 + self.weight = nn.Parameter(torch.ones(hidden_size))
89 + self.variance_epsilon = eps
90 +
91 + def forward(self, hidden_states):
92 + input_dtype = hidden_states.dtype
93 + variance = hidden_states.to(torch.float32).pow(2).mean(-1,
94 + keepdim=True)
95 + hidden_states = hidden_states * torch.rsqrt(variance +
96 + self.variance_epsilon)
97 +
98 + return (self.weight * hidden_states).to(input_dtype)
99 +
100 +
101 +class AquilaAttention(nn.Module):
102 +
103 + def __init__(
104 + self,
105 + hidden_size: int,
106 + num_heads: int,
107 + num_kv_heads: int,
108 + rope_theta: float = 10000,
109 + max_position_embeddings: int = 8192,
110 + rope_scaling: Optional[Dict[str, Any]] = None,
111 + linear_method: Optional[LinearMethodBase] = None,
112 + ):
113 + super().__init__()
114 + self.hidden_size = hidden_size
115 + tp_size = get_tensor_model_parallel_world_size()
116 + self.total_num_heads = num_heads
117 + assert self.total_num_heads % tp_size == 0
118 + self.num_heads = self.total_num_heads // tp_size
119 + self.total_num_kv_heads = num_kv_heads
120 + assert self.total_num_kv_heads % tp_size == 0
121 + self.num_kv_heads = self.total_num_kv_heads // tp_size
122 + self.head_dim = hidden_size // self.total_num_heads
123 + self.q_size = self.num_heads * self.head_dim
124 + self.kv_size = self.num_kv_heads * self.head_dim
125 + self.scaling = self.head_dim**-0.5
126 + self.rope_theta = rope_theta
127 + self.max_position_embeddings = max_position_embeddings
128 +
129 + self.qkv_proj = QKVParallelLinear(
130 + hidden_size,
131 + self.head_dim,
132 + self.total_num_heads,
133 + self.total_num_kv_heads,
134 + bias=False,
135 + linear_method=linear_method,
136 + )
137 + self.o_proj = RowParallelLinear(
138 + self.total_num_heads * self.head_dim,
139 + hidden_size,
140 + bias=False,
141 + linear_method=linear_method,
142 + )
143 + self.rotary_emb = get_rope(
144 + self.head_dim,
145 + rotary_dim=self.head_dim,
146 + max_position=self.max_position_embeddings,
147 + base=self.rope_theta,
148 + rope_scaling=rope_scaling,
149 + )
150 + self.attn = PagedAttention(self.num_heads,
151 + self.head_dim,
152 + self.scaling,
153 + num_kv_heads=self.num_kv_heads)
154 +
155 + def forward(
156 + self,
157 + positions: torch.Tensor,
158 + hidden_states: torch.Tensor,
159 + kv_cache: KVCache,
160 + input_metadata: InputMetadata,
161 + ) -> torch.Tensor:
162 + qkv, _ = self.qkv_proj(hidden_states)
163 + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
164 + q, k = self.rotary_emb(positions, q, k)
165 + k_cache, v_cache = kv_cache
166 + attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
167 + output, _ = self.o_proj(attn_output)
168 + return output
169 +
170 +
171 +class AquilaDecoderLayer(nn.Module):
172 +
173 + def __init__(
174 + self,
175 + config: AquilaConfig,
176 + linear_method: Optional[LinearMethodBase] = None,
177 + ):
178 + super().__init__()
179 + self.hidden_size = config.hidden_size
180 + rope_theta = getattr(config, "rope_theta", 10000)
181 + rope_scaling = getattr(config, "rope_scaling", None)
182 + max_position_embeddings = getattr(config, "max_position_embeddings",
183 + 8192)
184 + self.self_attn = AquilaAttention(
185 + hidden_size=self.hidden_size,
186 + num_heads=config.num_attention_heads,
187 + num_kv_heads=config.num_key_value_heads,
188 + rope_theta=rope_theta,
189 + max_position_embeddings=max_position_embeddings,
190 + rope_scaling=rope_scaling,
191 + linear_method=linear_method,
192 + )
193 + self.mlp = AquilaMLP(
194 + hidden_size=self.hidden_size,
195 + intermediate_size=config.intermediate_size,
196 + hidden_act=config.hidden_act,
197 + linear_method=linear_method,
198 + )
199 + self.input_layernorm = AquilaRMSNorm(config.hidden_size,
200 + eps=config.rms_norm_eps)
201 + self.post_attention_layernorm = AquilaRMSNorm(config.hidden_size,
202 + eps=config.rms_norm_eps)
203 +
204 + def forward(
205 + self,
206 + positions: torch.Tensor,
207 + hidden_states: torch.Tensor,
208 + kv_cache: KVCache,
209 + input_metadata: InputMetadata,
210 + ) -> torch.Tensor:
211 + # Self Attention
212 + residual = hidden_states
213 + hidden_states = self.input_layernorm(hidden_states)
214 + hidden_states = self.self_attn(
215 + positions=positions,
216 + hidden_states=hidden_states,
217 + kv_cache=kv_cache,
218 + input_metadata=input_metadata,
219 + )
220 + hidden_states = residual + hidden_states
221 +
222 + # Fully Connected
223 + residual = hidden_states
224 + hidden_states = self.post_attention_layernorm(hidden_states)
225 + hidden_states = self.mlp(hidden_states)
226 + hidden_states = residual + hidden_states
227 + return hidden_states
228 +
229 +
230 +class AquilaModel(nn.Module):
231 +
232 + def __init__(
233 + self,
234 + config: AquilaConfig,
235 + linear_method: Optional[LinearMethodBase] = None,
236 + ):
237 + super().__init__()
238 + self.config = config
239 + self.padding_idx = config.pad_token_id
240 + self.vocab_size = config.vocab_size
241 + self.embed_tokens = VocabParallelEmbedding(
242 + config.vocab_size,
243 + config.hidden_size,
244 + )
245 + self.layers = nn.ModuleList([
246 + AquilaDecoderLayer(config, linear_method)
247 + for _ in range(config.num_hidden_layers)
248 + ])
249 + self.norm = AquilaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
250 +
251 + def forward(
252 + self,
253 + input_ids: torch.Tensor,
254 + positions: torch.Tensor,
255 + kv_caches: List[KVCache],
256 + input_metadata: InputMetadata,
257 + ) -> torch.Tensor:
258 + hidden_states = self.embed_tokens(input_ids)
259 + for i in range(len(self.layers)):
260 + layer = self.layers[i]
261 + hidden_states = layer(
262 + positions,
263 + hidden_states,
264 + kv_caches[i],
265 + input_metadata,
266 + )
267 + hidden_states = self.norm(hidden_states)
268 +
269 + return hidden_states
270 +
271 +
272 +class AquilaForCausalLM(nn.Module):
273 +
274 + def __init__(
275 + self,
276 + config,
277 + linear_method: Optional[LinearMethodBase] = None,
278 + ):
279 + super().__init__()
280 + self.config = config
281 + self.linear_method = linear_method
282 + self.model = AquilaModel(config, linear_method)
283 + self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
284 + self.sampler = Sampler(config.vocab_size)
285 +
286 + def forward(
287 + self,
288 + input_ids: torch.Tensor,
289 + positions: torch.Tensor,
290 + kv_caches: List[KVCache],
291 + input_metadata: InputMetadata,
292 + ) -> torch.Tensor:
293 + hidden_states = self.model(input_ids, positions, kv_caches,
294 + input_metadata)
295 + return hidden_states
296 +
297 + def sample(
298 + self,
299 + hidden_states: torch.Tensor,
300 + sampling_metadata: SamplingMetadata,
301 + ) -> Optional[SamplerOutput]:
302 + next_tokens = self.sampler(self.lm_head.weight, hidden_states,
303 + sampling_metadata)
304 + return next_tokens
305 +
306 + def load_weights(self,
307 + model_name_or_path: str,
308 + cache_dir: Optional[str] = None,
309 + load_format: str = "auto",
310 + revision: Optional[str] = None):
311 + stacked_params_mapping = [
312 + # (param_name, shard_name, shard_id)
313 + ("qkv_proj", "q_proj", "q"),
314 + ("qkv_proj", "k_proj", "k"),
315 + ("qkv_proj", "v_proj", "v"),
316 + ("gate_up_proj", "gate_proj", 0),
317 + ("gate_up_proj", "up_proj", 1),
318 + ]
319 + params_dict = dict(self.named_parameters())
320 + for name, loaded_weight in hf_model_weights_iterator(
321 + model_name_or_path, cache_dir, load_format, revision):
322 + if "rotary_emb.inv_freq" in name:
323 + continue
324 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
325 + if weight_name not in name:
326 + continue
327 + name = name.replace(weight_name, param_name)
328 + # Skip loading extra bias for GPTQ models.
329 + if name.endswith(".bias") and name not in params_dict:
330 + continue
331 + param = params_dict[name]
332 + weight_loader = param.weight_loader
333 + weight_loader(param, loaded_weight, shard_id)
334 + break
335 + else:
336 + # Skip loading extra bias for GPTQ models.
337 + if name.endswith(".bias") and name not in params_dict:
338 + continue
339 + param = params_dict[name]
340 + weight_loader = getattr(param, "weight_loader",
341 + default_weight_loader)
342 + weight_loader(param, loaded_weight)
vllm/model_executor/models/baichuan.py
+303 -32
@@ -18,19 +18,305 @@
18 # See the License for the specific language governing permissions and
19 # limitations under the License.
20 """Inference-only BaiChuan model compatible with HuggingFace weights."""
21 -from typing import Optional
21 +import math
22 +from typing import List, Optional, Tuple
23
24 import torch
24 -from transformers import PretrainedConfig
25 -from vllm.config import LoRAConfig
25 +from torch import nn
26
27 -from vllm.model_executor.layers.linear import LinearMethodBase
28 -from vllm.model_executor.models.llama import LlamaForCausalLM
27 +from vllm.model_executor.input_metadata import InputMetadata
28 +from vllm.model_executor.layers.activation import SiluAndMul
29 +from vllm.model_executor.layers.attention import PagedAttention
30 +from vllm.model_executor.layers.layernorm import RMSNorm
31 +from vllm.model_executor.layers.linear import (LinearMethodBase,
32 + MergedColumnParallelLinear,
33 + QKVParallelLinear,
34 + RowParallelLinear)
35 +from vllm.model_executor.layers.rotary_embedding import get_rope
36 +from vllm.model_executor.layers.sampler import Sampler
37 +from vllm.model_executor.layers.vocab_parallel_embedding import (
38 + VocabParallelEmbedding, ParallelLMHead)
39 +from vllm.model_executor.parallel_utils.parallel_state import (
40 + get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
41 +from vllm.model_executor.sampling_metadata import SamplingMetadata
42 from vllm.model_executor.weight_utils import (default_weight_loader,
43 hf_model_weights_iterator)
44 +from vllm.sequence import SamplerOutput
45 +from vllm.transformers_utils.configs.baichuan import BaiChuanConfig
46
47 +KVCache = Tuple[torch.Tensor, torch.Tensor]
48
33 -class BaiChuanBaseForCausalLM(LlamaForCausalLM):
49 +
50 +def _get_alibi_slopes(total_num_heads: int) -> torch.Tensor:
51 + closest_power_of_2 = 2**math.floor(math.log2(total_num_heads))
52 + base = torch.tensor(
53 + 2**(-(2**-(math.log2(closest_power_of_2) - 3))),
54 + dtype=torch.float32,
55 + )
56 + powers = torch.arange(1, 1 + closest_power_of_2, dtype=torch.int32)
57 + slopes = torch.pow(base, powers)
58 +
59 + if closest_power_of_2 != total_num_heads:
60 + extra_base = torch.tensor(
61 + 2**(-(2**-(math.log2(2 * closest_power_of_2) - 3))),
62 + dtype=torch.float32,
63 + )
64 + num_remaining_heads = min(closest_power_of_2,
65 + total_num_heads - closest_power_of_2)
66 + extra_powers = torch.arange(start=1,
67 + end=1 + 2 * num_remaining_heads,
68 + step=2,
69 + dtype=torch.int32)
70 + slopes = torch.cat(
71 + [slopes, torch.pow(extra_base, extra_powers)], dim=0)
72 + return slopes
73 +
74 +
75 +class BaiChuanMLP(nn.Module):
76 +
77 + def __init__(
78 + self,
79 + hidden_size: int,
80 + intermediate_size: int,
81 + hidden_act: str,
82 + linear_method: Optional[LinearMethodBase] = None,
83 + ):
84 + super().__init__()
85 + self.gate_up_proj = MergedColumnParallelLinear(
86 + hidden_size, [intermediate_size] * 2,
87 + bias=False,
88 + linear_method=linear_method)
89 + self.down_proj = RowParallelLinear(intermediate_size,
90 + hidden_size,
91 + bias=False,
92 + linear_method=linear_method)
93 + if hidden_act != "silu":
94 + raise ValueError(f"Unsupported activation: {hidden_act}. "
95 + "Only silu is supported for now.")
96 + self.act_fn = SiluAndMul()
97 +
98 + def forward(self, x):
99 + gate_up, _ = self.gate_up_proj(x)
100 + x = self.act_fn(gate_up)
101 + x, _ = self.down_proj(x)
102 + return x
103 +
104 +
105 +class BaiChuanAttention(nn.Module):
106 + """Multi-headed attention from 'Attention Is All You Need' paper"""
107 +
108 + def __init__(
109 + self,
110 + hidden_size: int,
111 + num_heads: int,
112 + position_embedding: str,
113 + rope_theta: float = 10000,
114 + max_position_embeddings: int = 8192,
115 + linear_method: Optional[LinearMethodBase] = None,
116 + ):
117 + super().__init__()
118 + self.hidden_size = hidden_size
119 + tensor_model_parallel_world_size = get_tensor_model_parallel_world_size(
120 + )
121 + self.total_num_heads = num_heads
122 + assert self.total_num_heads % tensor_model_parallel_world_size == 0
123 + self.num_heads = (self.total_num_heads //
124 + tensor_model_parallel_world_size)
125 + self.head_dim = hidden_size // self.total_num_heads
126 + self.postion_embedding = position_embedding
127 + self.rope_theta = rope_theta
128 + self.max_position_embeddings = max_position_embeddings
129 +
130 + # pylint: disable=invalid-name
131 + self.W_pack = QKVParallelLinear(
132 + hidden_size,
133 + self.head_dim,
134 + self.total_num_heads,
135 + self.total_num_heads,
136 + bias=False,
137 + linear_method=linear_method,
138 + )
139 + self.o_proj = RowParallelLinear(
140 + self.total_num_heads * self.head_dim,
141 + hidden_size,
142 + bias=False,
143 + linear_method=linear_method,
144 + )
145 + # Create the alibi slopes and slice them.
146 + if self.postion_embedding == "ALIBI":
147 + tp_rank = get_tensor_model_parallel_rank()
148 + head_start = tp_rank * self.num_heads
149 + head_end = (tp_rank + 1) * self.num_heads
150 + alibi_slopes = _get_alibi_slopes(self.total_num_heads)
151 + alibi_slopes = alibi_slopes[head_start:head_end].tolist()
152 +
153 + scaling = self.head_dim**-0.5
154 + self.attn = PagedAttention(self.num_heads,
155 + self.head_dim,
156 + scaling,
157 + alibi_slopes=alibi_slopes)
158 + else:
159 + self.rotary_emb = get_rope(
160 + self.head_dim,
161 + rotary_dim=self.head_dim,
162 + max_position=self.max_position_embeddings,
163 + base=self.rope_theta,
164 + )
165 + self.scaling = self.head_dim**-0.5
166 + self.attn = PagedAttention(self.num_heads, self.head_dim,
167 + self.scaling)
168 +
169 + def forward(
170 + self,
171 + positions: torch.Tensor,
172 + hidden_states: torch.Tensor,
173 + kv_cache: KVCache,
174 + input_metadata: InputMetadata,
175 + ) -> torch.Tensor:
176 + qkv, _ = self.W_pack(hidden_states)
177 + q, k, v = qkv.chunk(chunks=3, dim=-1)
178 + if self.postion_embedding != "ALIBI":
179 + q, k = self.rotary_emb(positions, q, k)
180 + k_cache, v_cache = kv_cache
181 + attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
182 + output, _ = self.o_proj(attn_output)
183 + return output
184 +
185 +
186 +class BaiChuanDecoderLayer(nn.Module):
187 +
188 + def __init__(self,
189 + config: BaiChuanConfig,
190 + position_embedding: str,
191 + linear_method: Optional[LinearMethodBase] = None):
192 + super().__init__()
193 + self.hidden_size = config.hidden_size
194 + rope_theta = getattr(config, "rope_theta", 10000)
195 + max_position_embeddings = getattr(config, "max_position_embeddings",
196 + 8192)
197 + self.self_attn = BaiChuanAttention(
198 + hidden_size=self.hidden_size,
199 + num_heads=config.num_attention_heads,
200 + position_embedding=position_embedding,
201 + rope_theta=rope_theta,
202 + max_position_embeddings=max_position_embeddings,
203 + linear_method=linear_method,
204 + )
205 + self.mlp = BaiChuanMLP(
206 + hidden_size=self.hidden_size,
207 + intermediate_size=config.intermediate_size,
208 + hidden_act=config.hidden_act,
209 + linear_method=linear_method,
210 + )
211 + self.input_layernorm = RMSNorm(config.hidden_size,
212 + eps=config.rms_norm_eps)
213 + self.post_attention_layernorm = RMSNorm(config.hidden_size,
214 + eps=config.rms_norm_eps)
215 +
216 + def forward(
217 + self,
218 + positions: torch.Tensor,
219 + hidden_states: torch.Tensor,
220 + kv_cache: KVCache,
221 + input_metadata: InputMetadata,
222 + residual: Optional[torch.Tensor],
223 + ) -> Tuple[torch.Tensor, torch.Tensor]:
224 + # Self Attention
225 + if residual is None:
226 + residual = hidden_states
227 + hidden_states = self.input_layernorm(hidden_states)
228 + else:
229 + hidden_states, residual = self.input_layernorm(
230 + hidden_states, residual)
231 + hidden_states = self.self_attn(
232 + positions=positions,
233 + hidden_states=hidden_states,
234 + kv_cache=kv_cache,
235 + input_metadata=input_metadata,
236 + )
237 +
238 + # Fully Connected
239 + hidden_states, residual = self.post_attention_layernorm(
240 + hidden_states, residual)
241 + hidden_states = self.mlp(hidden_states)
242 + return hidden_states, residual
243 +
244 +
245 +class BaiChuanModel(nn.Module):
246 +
247 + def __init__(self,
248 + config: BaiChuanConfig,
249 + position_embedding: str,
250 + linear_method: Optional[LinearMethodBase] = None):
251 + super().__init__()
252 + self.config = config
253 + self.padding_idx = config.pad_token_id
254 + self.vocab_size = config.vocab_size
255 +
256 + self.embed_tokens = VocabParallelEmbedding(
257 + config.vocab_size,
258 + config.hidden_size,
259 + )
260 + self.layers = nn.ModuleList([
261 + BaiChuanDecoderLayer(config, position_embedding, linear_method)
262 + for _ in range(config.num_hidden_layers)
263 + ])
264 + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
265 +
266 + def forward(
267 + self,
268 + input_ids: torch.Tensor,
269 + positions: torch.Tensor,
270 + kv_caches: List[KVCache],
271 + input_metadata: InputMetadata,
272 + ) -> torch.Tensor:
273 + hidden_states = self.embed_tokens(input_ids)
274 + residual = None
275 + for i in range(len(self.layers)):
276 + layer = self.layers[i]
277 + hidden_states, residual = layer(
278 + positions,
279 + hidden_states,
280 + kv_caches[i],
281 + input_metadata,
282 + residual,
283 + )
284 + hidden_states, _ = self.norm(hidden_states, residual)
285 + return hidden_states
286 +
287 +
288 +class BaiChuanBaseForCausalLM(nn.Module):
289 +
290 + def __init__(self,
291 + config,
292 + position_embedding: str,
293 + linear_method: Optional[LinearMethodBase] = None):
294 + super().__init__()
295 + self.config = config
296 + self.linear_method = linear_method
297 + self.model = BaiChuanModel(config, position_embedding, linear_method)
298 + self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
299 + self.sampler = Sampler(config.vocab_size)
300 +
301 + def forward(
302 + self,
303 + input_ids: torch.Tensor,
304 + positions: torch.Tensor,
305 + kv_caches: List[KVCache],
306 + input_metadata: InputMetadata,
307 + ) -> torch.Tensor:
308 + hidden_states = self.model(input_ids, positions, kv_caches,
309 + input_metadata)
310 + return hidden_states
311 +
312 + def sample(
313 + self,
314 + hidden_states: torch.Tensor,
315 + sampling_metadata: SamplingMetadata,
316 + ) -> Optional[SamplerOutput]:
317 + next_tokens = self.sampler(self.lm_head.weight, hidden_states,
318 + sampling_metadata)
319 + return next_tokens
320
321 def load_weights(self,
322 model_name_or_path: str,
@@ -42,15 +328,9 @@ class BaiChuanBaseForCausalLM(LlamaForCausalLM):
328 ("gate_up_proj", "gate_proj", 0),
329 ("gate_up_proj", "up_proj", 1),
330 ]
45 - param_weight_map = [
46 - ("qkv_proj", "W_pack"),
47 - ]
331 params_dict = dict(self.named_parameters())
332 for name, loaded_weight in hf_model_weights_iterator(
333 model_name_or_path, cache_dir, load_format, revision):
51 - for (param_name, weight_name) in param_weight_map:
52 - name = name.replace(weight_name, param_name)
53 -
334 if "rotary_emb.inv_freq" in name:
335 continue
336 if name == "lm_head.weight":
@@ -88,28 +368,19 @@ class BaiChuanBaseForCausalLM(LlamaForCausalLM):
368 class BaichuanForCausalLM(BaiChuanBaseForCausalLM):
369 """Baichuan 13B and Baichuan2 7B/13B."""
370
91 - def __init__(
92 - self,
93 - config: Optional[PretrainedConfig] = None,
94 - linear_method: Optional[LinearMethodBase] = None,
95 - lora_config: Optional[LoRAConfig] = None,
96 - ) -> None:
97 - if config.hidden_size != 4096: # baichuan 13b, baichuan2 13b
98 - config.postion_embedding = "ALIBI"
99 - super().__init__(config=config,
100 - linear_method=linear_method,
101 - lora_config=lora_config)
371 + def __init__(self,
372 + config,
373 + linear_method: Optional[LinearMethodBase] = None):
374 + if config.hidden_size == 4096: # baichuan2 7b
375 + super().__init__(config, "ROPE", linear_method)
376 + else: # baichuan 13b, baichuan2 13b
377 + super().__init__(config, "ALIBI", linear_method)
378
379
380 class BaiChuanForCausalLM(BaiChuanBaseForCausalLM):
381 """Baichuan 7B."""
382
107 - def __init__(
108 - self,
109 - config: Optional[PretrainedConfig] = None,
110 - linear_method: Optional[LinearMethodBase] = None,
111 - lora_config: Optional[LoRAConfig] = None,
112 - ) -> None:
113 - super().__init__(config=config,
114 - linear_method=linear_method,
115 - lora_config=lora_config)
383 + def __init__(self,
384 + config,
385 + linear_method: Optional[LinearMethodBase] = None):
386 + super().__init__(config, "ROPE", linear_method)
vllm/model_executor/models/internlm.py new
+299
@@ -0,0 +1,299 @@
1 +# -*- coding: utf-8 -*-
2 +from typing import Any, Dict, List, Optional, Tuple
3 +
4 +import torch
5 +from torch import nn
6 +from transformers import LlamaConfig
7 +
8 +from vllm.model_executor.input_metadata import InputMetadata
9 +from vllm.model_executor.layers.activation import SiluAndMul
10 +from vllm.model_executor.layers.attention import PagedAttention
11 +from vllm.model_executor.layers.layernorm import RMSNorm
12 +from vllm.model_executor.layers.linear import (LinearMethodBase,
13 + MergedColumnParallelLinear,
14 + QKVParallelLinear,
15 + RowParallelLinear)
16 +from vllm.model_executor.layers.rotary_embedding import get_rope
17 +from vllm.model_executor.layers.sampler import Sampler
18 +from vllm.model_executor.layers.vocab_parallel_embedding import (
19 + VocabParallelEmbedding, ParallelLMHead)
20 +from vllm.model_executor.parallel_utils.parallel_state import (
21 + get_tensor_model_parallel_world_size)
22 +from vllm.model_executor.sampling_metadata import SamplingMetadata
23 +from vllm.model_executor.weight_utils import (default_weight_loader,
24 + hf_model_weights_iterator)
25 +from vllm.sequence import SamplerOutput
26 +
27 +KVCache = Tuple[torch.Tensor, torch.Tensor]
28 +
29 +
30 +class InternLMMLP(nn.Module):
31 +
32 + def __init__(
33 + self,
34 + hidden_size: int,
35 + intermediate_size: int,
36 + hidden_act: str,
37 + linear_method: Optional[LinearMethodBase] = None,
38 + ):
39 + super().__init__()
40 + self.gate_up_proj = MergedColumnParallelLinear(
41 + hidden_size, [intermediate_size] * 2,
42 + bias=False,
43 + linear_method=linear_method)
44 + self.down_proj = RowParallelLinear(intermediate_size,
45 + hidden_size,
46 + bias=False,
47 + linear_method=linear_method)
48 + if hidden_act != "silu":
49 + raise ValueError(f"Unsupported activation: {hidden_act}. "
50 + "Only silu is supported for now.")
51 + self.act_fn = SiluAndMul()
52 +
53 + def forward(self, x):
54 + gate_up, _ = self.gate_up_proj(x)
55 + x = self.act_fn(gate_up)
56 + x, _ = self.down_proj(x)
57 + return x
58 +
59 +
60 +class InternLMAttention(nn.Module):
61 +
62 + def __init__(
63 + self,
64 + hidden_size: int,
65 + num_heads: int,
66 + bias: bool,
67 + rope_theta: float = 10000,
68 + max_position_embeddings: int = 8192,
69 + linear_method: Optional[LinearMethodBase] = None,
70 + rope_scaling: Optional[Dict[str, Any]] = None,
71 + ):
72 + super().__init__()
73 + self.hidden_size = hidden_size
74 + tensor_model_parallel_world_size = (
75 + get_tensor_model_parallel_world_size())
76 + self.total_num_heads = num_heads
77 + assert self.total_num_heads % tensor_model_parallel_world_size == 0
78 + self.num_heads = (self.total_num_heads //
79 + tensor_model_parallel_world_size)
80 + self.head_dim = hidden_size // self.total_num_heads
81 + self.scaling = self.head_dim**-0.5
82 + self.rope_theta = rope_theta
83 + self.max_position_embeddings = max_position_embeddings
84 +
85 + self.qkv_proj = QKVParallelLinear(
86 + hidden_size,
87 + self.head_dim,
88 + self.total_num_heads,
89 + bias=bias,
90 + linear_method=linear_method,
91 + )
92 + self.o_proj = RowParallelLinear(
93 + self.total_num_heads * self.head_dim,
94 + hidden_size,
95 + bias=bias,
96 + linear_method=linear_method,
97 + )
98 + self.rotary_emb = get_rope(
99 + self.head_dim,
100 + rotary_dim=self.head_dim,
101 + max_position=self.max_position_embeddings,
102 + base=self.rope_theta,
103 + rope_scaling=rope_scaling,
104 + )
105 + self.attn = PagedAttention(self.num_heads, self.head_dim, self.scaling)
106 +
107 + def forward(
108 + self,
109 + positions: torch.Tensor,
110 + hidden_states: torch.Tensor,
111 + kv_cache: KVCache,
112 + input_metadata: InputMetadata,
113 + ) -> torch.Tensor:
114 + qkv, _ = self.qkv_proj(hidden_states)
115 + q, k, v = qkv.chunk(chunks=3, dim=-1)
116 + q, k = self.rotary_emb(positions, q, k)
117 + k_cache, v_cache = kv_cache
118 + attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
119 + output, _ = self.o_proj(attn_output)
120 + return output
121 +
122 +
123 +class InternLMDecoderLayer(nn.Module):
124 +
125 + def __init__(
126 + self,
127 + config: LlamaConfig,
128 + linear_method: Optional[LinearMethodBase] = None,
129 + ):
130 + super().__init__()
131 + self.hidden_size = config.hidden_size
132 + rope_theta = getattr(config, "rope_theta", 10000)
133 + max_position_embeddings = getattr(config, "max_position_embeddings",
134 + 8192)
135 + self.self_attn = InternLMAttention(
136 + hidden_size=self.hidden_size,
137 + num_heads=config.num_attention_heads,
138 + bias=config.bias,
139 + rope_theta=rope_theta,
140 + max_position_embeddings=max_position_embeddings,
141 + linear_method=linear_method,
142 + rope_scaling=getattr(config, "rope_scaling", None),
143 + )
144 + self.mlp = InternLMMLP(
145 + hidden_size=self.hidden_size,
146 + intermediate_size=config.intermediate_size,
147 + hidden_act=config.hidden_act,
148 + linear_method=linear_method,
149 + )
150 + self.input_layernorm = RMSNorm(config.hidden_size,
151 + eps=config.rms_norm_eps)
152 + self.post_attention_layernorm = RMSNorm(config.hidden_size,
153 + eps=config.rms_norm_eps)
154 +
155 + def forward(
156 + self,
157 + positions: torch.Tensor,
158 + hidden_states: torch.Tensor,
159 + kv_cache: KVCache,
160 + input_metadata: InputMetadata,
161 + residual: Optional[torch.Tensor],
162 + ) -> Tuple[torch.Tensor, torch.Tensor]:
163 + # Self Attention
164 + if residual is None:
165 + residual = hidden_states
166 + hidden_states = self.input_layernorm(hidden_states)
167 + else:
168 + hidden_states, residual = self.input_layernorm(
169 + hidden_states, residual)
170 + hidden_states = self.self_attn(
171 + positions=positions,
172 + hidden_states=hidden_states,
173 + kv_cache=kv_cache,
174 + input_metadata=input_metadata,
175 + )
176 +
177 + # Fully Connected
178 + hidden_states, residual = self.post_attention_layernorm(
179 + hidden_states, residual)
180 + hidden_states = self.mlp(hidden_states)
181 + return hidden_states, residual
182 +
183 +
184 +class InternLMModel(nn.Module):
185 +
186 + def __init__(
187 + self,
188 + config: LlamaConfig,
189 + linear_method: Optional[LinearMethodBase] = None,
190 + ):
191 + super().__init__()
192 + self.config = config
193 + self.padding_idx = config.pad_token_id
194 + self.vocab_size = config.vocab_size
195 +
196 + vocab_size = ((config.vocab_size + 63) // 64) * 64
197 + self.embed_tokens = VocabParallelEmbedding(
198 + vocab_size,
199 + config.hidden_size,
200 + )
201 + self.layers = nn.ModuleList([
202 + InternLMDecoderLayer(config, linear_method)
203 + for _ in range(config.num_hidden_layers)
204 + ])
205 + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
206 +
207 + def forward(
208 + self,
209 + input_ids: torch.Tensor,
210 + positions: torch.Tensor,
211 + kv_caches: List[KVCache],
212 + input_metadata: InputMetadata,
213 + ) -> torch.Tensor:
214 + hidden_states = self.embed_tokens(input_ids)
215 + residual = None
216 + for i in range(len(self.layers)):
217 + layer = self.layers[i]
218 + hidden_states, residual = layer(
219 + positions,
220 + hidden_states,
221 + kv_caches[i],
222 + input_metadata,
223 + residual,
224 + )
225 + hidden_states, _ = self.norm(hidden_states, residual)
226 + return hidden_states
227 +
228 +
229 +class InternLMForCausalLM(nn.Module):
230 +
231 + def __init__(
232 + self,
233 + config,
234 + linear_method: Optional[LinearMethodBase] = None,
235 + ):
236 + super().__init__()
237 + self.config = config
238 + self.linear_method = linear_method
239 + self.model = InternLMModel(config, linear_method)
240 + self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
241 + self.sampler = Sampler(config.vocab_size)
242 +
243 + def forward(
244 + self,
245 + input_ids: torch.Tensor,
246 + positions: torch.Tensor,
247 + kv_caches: List[KVCache],
248 + input_metadata: InputMetadata,
249 + ) -> torch.Tensor:
250 + hidden_states = self.model(input_ids, positions, kv_caches,
251 + input_metadata)
252 + return hidden_states
253 +
254 + def sample(
255 + self,
256 + hidden_states: torch.Tensor,
257 + sampling_metadata: SamplingMetadata,
258 + ) -> Optional[SamplerOutput]:
259 + next_tokens = self.sampler(self.lm_head.weight, hidden_states,
260 + sampling_metadata)
261 + return next_tokens
262 +
263 + def load_weights(self,
264 + model_name_or_path: str,
265 + cache_dir: Optional[str] = None,
266 + load_format: str = "auto",
267 + revision: Optional[str] = None):
268 + stacked_params_mapping = [
269 + # (param_name, shard_name, shard_id)
270 + ("qkv_proj", "q_proj", "q"),
271 + ("qkv_proj", "k_proj", "k"),
272 + ("qkv_proj", "v_proj", "v"),
273 + ("gate_up_proj", "gate_proj", 0),
274 + ("gate_up_proj", "up_proj", 1),
275 + ]
276 + params_dict = dict(self.named_parameters())
277 + for name, loaded_weight in hf_model_weights_iterator(
278 + model_name_or_path, cache_dir, load_format, revision):
279 + if "rotary_emb.inv_freq" in name:
280 + continue
281 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
282 + if weight_name not in name:
283 + continue
284 + name = name.replace(weight_name, param_name)
285 + # Skip loading extra bias for GPTQ models.
286 + if name.endswith(".bias") and name not in params_dict:
287 + continue
288 + param = params_dict[name]
289 + weight_loader = param.weight_loader
290 + weight_loader(param, loaded_weight, shard_id)
291 + break
292 + else:
293 + # Skip loading extra bias for GPTQ models.
294 + if name.endswith(".bias") and name not in params_dict:
295 + continue
296 + param = params_dict[name]
297 + weight_loader = getattr(param, "weight_loader",
298 + default_weight_loader)
299 + weight_loader(param, loaded_weight)
vllm/model_executor/models/internlm2.py
+260 -25
@@ -1,27 +1,276 @@
1 # -*- coding: utf-8 -*-
2 -from typing import Optional
2 +from typing import Any, Dict, List, Optional, Tuple
3
4 import torch
5 +from torch import nn
6 from transformers import PretrainedConfig
6 -from vllm.config import LoRAConfig
7
8 -from vllm.model_executor.layers.linear import LinearMethodBase
9 -from vllm.model_executor.models.llama import LlamaForCausalLM
8 +from vllm.model_executor.input_metadata import InputMetadata
9 +from vllm.model_executor.layers.activation import SiluAndMul
10 +from vllm.model_executor.layers.attention import PagedAttention
11 +from vllm.model_executor.layers.layernorm import RMSNorm
12 +from vllm.model_executor.layers.linear import (LinearMethodBase,
13 + MergedColumnParallelLinear,
14 + QKVParallelLinear,
15 + RowParallelLinear)
16 +from vllm.model_executor.layers.rotary_embedding import get_rope
17 +from vllm.model_executor.layers.sampler import Sampler
18 +from vllm.model_executor.layers.vocab_parallel_embedding import (
19 + VocabParallelEmbedding, ParallelLMHead)
20 +from vllm.model_executor.parallel_utils.parallel_state import (
21 + get_tensor_model_parallel_world_size)
22 +from vllm.model_executor.sampling_metadata import SamplingMetadata
23 from vllm.model_executor.weight_utils import (default_weight_loader,
24 hf_model_weights_iterator)
25 +from vllm.sequence import SamplerOutput
26
27 +KVCache = Tuple[torch.Tensor, torch.Tensor]
28
14 -class InternLM2ForCausalLM(LlamaForCausalLM):
29 +
30 +class InternLM2MLP(nn.Module):
31 +
32 + def __init__(
33 + self,
34 + hidden_size: int,
35 + intermediate_size: int,
36 + hidden_act: str,
37 + linear_method: Optional[LinearMethodBase] = None,
38 + ) -> None:
39 + super().__init__()
40 + self.gate_up_proj = MergedColumnParallelLinear(
41 + hidden_size, [intermediate_size] * 2,
42 + bias=False,
43 + linear_method=linear_method)
44 + self.w2 = RowParallelLinear(intermediate_size,
45 + hidden_size,
46 + bias=False,
47 + linear_method=linear_method)
48 + if hidden_act != "silu":
49 + raise ValueError(f"Unsupported activation: {hidden_act}. "
50 + "Only silu is supported for now.")
51 + self.act_fn = SiluAndMul()
52 +
53 + def forward(self, x):
54 + gate_up, _ = self.gate_up_proj(x)
55 + x = self.act_fn(gate_up)
56 + x, _ = self.w2(x)
57 + return x
58 +
59 +
60 +class InternLM2Attention(nn.Module):
61 +
62 + def __init__(
63 + self,
64 + hidden_size: int,
65 + num_heads: int,
66 + num_kv_heads: int,
67 + rope_theta: float = 10000,
68 + rope_scaling: Optional[Dict[str, Any]] = None,
69 + max_position_embeddings: int = 8192,
70 + linear_method: Optional[LinearMethodBase] = None,
71 + ) -> None:
72 + super().__init__()
73 + self.hidden_size = hidden_size
74 + tp_size = get_tensor_model_parallel_world_size()
75 + self.total_num_heads = num_heads
76 + assert self.total_num_heads % tp_size == 0
77 + self.num_heads = self.total_num_heads // tp_size
78 + self.total_num_kv_heads = num_kv_heads
79 + if self.total_num_kv_heads >= tp_size:
80 + # Number of KV heads is greater than TP size, so we partition
81 + # the KV heads across multiple tensor parallel GPUs.
82 + assert self.total_num_kv_heads % tp_size == 0
83 + else:
84 + # Number of KV heads is less than TP size, so we replicate
85 + # the KV heads across multiple tensor parallel GPUs.
86 + assert tp_size % self.total_num_kv_heads == 0
87 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
88 + self.head_dim = hidden_size // self.total_num_heads
89 + self.q_size = self.num_heads * self.head_dim
90 + self.kv_size = self.num_kv_heads * self.head_dim
91 + self.scaling = self.head_dim**-0.5
92 + self.rope_theta = rope_theta
93 + self.max_position_embeddings = max_position_embeddings
94 +
95 + self.wqkv = QKVParallelLinear(
96 + hidden_size,
97 + self.head_dim,
98 + self.total_num_heads,
99 + self.total_num_kv_heads,
100 + bias=False,
101 + linear_method=linear_method,
102 + )
103 + self.wo = RowParallelLinear(
104 + self.total_num_heads * self.head_dim,
105 + hidden_size,
106 + bias=False,
107 + linear_method=linear_method,
108 + )
109 +
110 + self.rotary_emb = get_rope(
111 + self.head_dim,
112 + rotary_dim=self.head_dim,
113 + max_position=max_position_embeddings,
114 + base=rope_theta,
115 + rope_scaling=rope_scaling,
116 + )
117 + self.attn = PagedAttention(self.num_heads,
118 + self.head_dim,
119 + self.scaling,
120 + num_kv_heads=self.num_kv_heads)
121 +
122 + def forward(
123 + self,
124 + positions: torch.Tensor,
125 + hidden_states: torch.Tensor,
126 + kv_cache: KVCache,
127 + input_metadata: InputMetadata,
128 + ) -> torch.Tensor:
129 + qkv, _ = self.wqkv(hidden_states)
130 + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
131 + q, k = self.rotary_emb(positions, q, k)
132 + k_cache, v_cache = kv_cache
133 + attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
134 + output, _ = self.wo(attn_output)
135 + return output
136 +
137 +
138 +class InternLMDecoderLayer(nn.Module):
139 +
140 + def __init__(
141 + self,
142 + config: PretrainedConfig,
143 + linear_method: Optional[LinearMethodBase] = None,
144 + ) -> None:
145 + super().__init__()
146 + self.hidden_size = config.hidden_size
147 + rope_theta = getattr(config, "rope_theta", 10000)
148 + rope_scaling = getattr(config, "rope_scaling", None)
149 + max_position_embeddings = getattr(config, "max_position_embeddings",
150 + 8192)
151 + self.attention = InternLM2Attention(
152 + hidden_size=self.hidden_size,
153 + num_heads=config.num_attention_heads,
154 + num_kv_heads=config.num_key_value_heads,
155 + rope_theta=rope_theta,
156 + rope_scaling=rope_scaling,
157 + max_position_embeddings=max_position_embeddings,
158 + linear_method=linear_method,
159 + )
160 + self.feed_forward = InternLM2MLP(
161 + hidden_size=self.hidden_size,
162 + intermediate_size=config.intermediate_size,
163 + hidden_act=config.hidden_act,
164 + linear_method=linear_method,
165 + )
166 + self.attention_norm = RMSNorm(config.hidden_size,
167 + eps=config.rms_norm_eps)
168 + self.ffn_norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
169 +
170 + def forward(
171 + self,
172 + positions: torch.Tensor,
173 + hidden_states: torch.Tensor,
174 + kv_cache: KVCache,
175 + input_metadata: InputMetadata,
176 + residual: Optional[torch.Tensor],
177 + ) -> Tuple[torch.Tensor, torch.Tensor]:
178 + # Self Attention
179 + if residual is None:
180 + residual = hidden_states
181 + hidden_states = self.attention_norm(hidden_states)
182 + else:
183 + hidden_states, residual = self.attention_norm(
184 + hidden_states, residual)
185 + hidden_states = self.attention(
186 + positions=positions,
187 + hidden_states=hidden_states,
188 + kv_cache=kv_cache,
189 + input_metadata=input_metadata,
190 + )
191 +
192 + # Fully Connected
193 + hidden_states, residual = self.ffn_norm(hidden_states, residual)
194 + hidden_states = self.feed_forward(hidden_states)
195 + return hidden_states, residual
196 +
197 +
198 +class InternLM2Model(nn.Module):
199
200 def __init__(
201 self,
18 - config: Optional[PretrainedConfig] = None,
202 + config: PretrainedConfig,
203 linear_method: Optional[LinearMethodBase] = None,
20 - lora_config: Optional[LoRAConfig] = None,
204 ) -> None:
22 - super().__init__(config=config,
23 - linear_method=linear_method,
24 - lora_config=lora_config)
205 + super().__init__()
206 + self.config = config
207 + self.padding_idx = config.pad_token_id
208 + self.vocab_size = config.vocab_size
209 + self.tok_embeddings = VocabParallelEmbedding(
210 + config.vocab_size,
211 + config.hidden_size,
212 + )
213 + self.layers = nn.ModuleList([
214 + InternLMDecoderLayer(config, linear_method)
215 + for _ in range(config.num_hidden_layers)
216 + ])
217 + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
218 +
219 + def forward(
220 + self,
221 + input_ids: torch.Tensor,
222 + positions: torch.Tensor,
223 + kv_caches: List[KVCache],
224 + input_metadata: InputMetadata,
225 + ) -> torch.Tensor:
226 + hidden_states = self.tok_embeddings(input_ids)
227 + residual = None
228 + for i in range(len(self.layers)):
229 + layer = self.layers[i]
230 + hidden_states, residual = layer(
231 + positions,
232 + hidden_states,
233 + kv_caches[i],
234 + input_metadata,
235 + residual,
236 + )
237 + hidden_states, _ = self.norm(hidden_states, residual)
238 + return hidden_states
239 +
240 +
241 +class InternLM2ForCausalLM(nn.Module):
242 +
243 + def __init__(
244 + self,
245 + config: PretrainedConfig,
246 + linear_method: Optional[LinearMethodBase] = None,
247 + ) -> None:
248 + super().__init__()
249 + self.config = config
250 + self.linear_method = linear_method
251 + self.model = InternLM2Model(config, linear_method)
252 + self.output = ParallelLMHead(config.vocab_size, config.hidden_size)
253 + self.sampler = Sampler(config.vocab_size)
254 +
255 + def forward(
256 + self,
257 + input_ids: torch.Tensor,
258 + positions: torch.Tensor,
259 + kv_caches: List[KVCache],
260 + input_metadata: InputMetadata,
261 + ) -> torch.Tensor:
262 + hidden_states = self.model(input_ids, positions, kv_caches,
263 + input_metadata)
264 + return hidden_states
265 +
266 + def sample(
267 + self,
268 + hidden_states: torch.Tensor,
269 + sampling_metadata: SamplingMetadata,
270 + ) -> Optional[SamplerOutput]:
271 + next_tokens = self.sampler(self.output.weight, hidden_states,
272 + sampling_metadata)
273 + return next_tokens
274
275 def load_weights(self,
276 model_name_or_path: str,
@@ -33,23 +282,9 @@ class InternLM2ForCausalLM(LlamaForCausalLM):
282 ("gate_up_proj", "w1", 0),
283 ("gate_up_proj", "w3", 1),
284 ]
36 - param_weight_map = [
37 - ("qkv_proj", "wqkv"),
38 - ("o_proj", "wo"),
39 - ("down_proj", "w2"),
40 - ("input_layernorm", "attention_norm"),
41 - ("post_attention_layernorm", "ffn_norm"),
42 - ("embed_tokens", "tok_embeddings"),
43 - (".self_attn.", ".attention."),
44 - ("mlp", "feed_forward"),
45 - ("lm_head", "output"),
46 - ]
285 params_dict = dict(self.named_parameters())
286 for name, loaded_weight in hf_model_weights_iterator(
287 model_name_or_path, cache_dir, load_format, revision):
50 - for (param_name, weight_name) in param_weight_map:
51 - name = name.replace(weight_name, param_name)
52 -
288 if "rotary_emb.inv_freq" in name:
289 continue
290 for (param_name, weight_name, shard_id) in stacked_params_mapping:
@@ -68,7 +303,7 @@ class InternLM2ForCausalLM(LlamaForCausalLM):
303 if name.endswith(".bias") and name not in params_dict:
304 continue
305 param = params_dict[name]
71 - if "qkv_proj" in name:
306 + if "wqkv" in name:
307 config = self.config
308 kv_groups = config.num_attention_heads // config.num_key_value_heads
309 head_dim = config.hidden_size // config.num_attention_heads
vllm/model_executor/models/llama.py
+56 -106
@@ -21,9 +21,8 @@
21 # See the License for the specific language governing permissions and
22 # limitations under the License.
23 """Inference-only LLaMA model compatible with HuggingFace weights."""
24 -from typing import List, Optional, Tuple
24 +from typing import Any, Dict, List, Optional, Tuple
25
26 -import math
26 import torch
27 from torch import nn
28 from transformers import LlamaConfig
@@ -41,60 +40,34 @@ from vllm.model_executor.layers.sampler import Sampler
40 from vllm.model_executor.layers.vocab_parallel_embedding import (
41 VocabParallelEmbedding, ParallelLMHead, DEFAULT_VOCAB_PADDING_SIZE)
42 from vllm.model_executor.parallel_utils.parallel_state import (
44 - get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
43 + get_tensor_model_parallel_world_size)
44 from vllm.model_executor.sampling_metadata import SamplingMetadata
45 from vllm.model_executor.weight_utils import (default_weight_loader,
46 hf_model_weights_iterator)
47 from vllm.sequence import SamplerOutput
48 from vllm.config import LoRAConfig
49
51 -from copy import deepcopy
52 -
50 KVCache = Tuple[torch.Tensor, torch.Tensor]
51
52
56 -def _get_alibi_slopes(total_num_heads: int) -> torch.Tensor:
57 - closest_power_of_2 = 2**math.floor(math.log2(total_num_heads))
58 - base = torch.tensor(
59 - 2**(-(2**-(math.log2(closest_power_of_2) - 3))),
60 - dtype=torch.float32,
61 - )
62 - powers = torch.arange(1, 1 + closest_power_of_2, dtype=torch.int32)
63 - slopes = torch.pow(base, powers)
64 -
65 - if closest_power_of_2 != total_num_heads:
66 - extra_base = torch.tensor(
67 - 2**(-(2**-(math.log2(2 * closest_power_of_2) - 3))),
68 - dtype=torch.float32,
69 - )
70 - num_remaining_heads = min(closest_power_of_2,
71 - total_num_heads - closest_power_of_2)
72 - extra_powers = torch.arange(start=1,
73 - end=1 + 2 * num_remaining_heads,
74 - step=2,
75 - dtype=torch.int32)
76 - slopes = torch.cat(
77 - [slopes, torch.pow(extra_base, extra_powers)], dim=0)
78 - return slopes
79 -
80 -
53 class LlamaMLP(nn.Module):
54
55 def __init__(
56 self,
85 - config: LlamaConfig,
57 + hidden_size: int,
58 + intermediate_size: int,
59 + hidden_act: str,
60 linear_method: Optional[LinearMethodBase] = None,
61 ) -> None:
62 super().__init__()
63 self.gate_up_proj = MergedColumnParallelLinear(
90 - config.hidden_size, [config.intermediate_size] * 2,
64 + hidden_size, [intermediate_size] * 2,
65 bias=False,
66 linear_method=linear_method)
93 - self.down_proj = RowParallelLinear(config.intermediate_size,
94 - config.hidden_size,
67 + self.down_proj = RowParallelLinear(intermediate_size,
68 + hidden_size,
69 bias=False,
70 linear_method=linear_method)
97 - hidden_act = getattr(config, "hidden_act", "silu")
71 if hidden_act != "silu":
72 raise ValueError(f"Unsupported activation: {hidden_act}. "
73 "Only silu is supported for now.")
@@ -111,19 +84,21 @@ class LlamaAttention(nn.Module):
84
85 def __init__(
86 self,
114 - config: LlamaConfig,
87 + hidden_size: int,
88 + num_heads: int,
89 + num_kv_heads: int,
90 + rope_theta: float = 10000,
91 + rope_scaling: Optional[Dict[str, Any]] = None,
92 + max_position_embeddings: int = 8192,
93 linear_method: Optional[LinearMethodBase] = None,
94 ) -> None:
95 super().__init__()
118 - self.hidden_size = config.hidden_size
96 + self.hidden_size = hidden_size
97 tp_size = get_tensor_model_parallel_world_size()
120 - self.total_num_heads = getattr(config, "num_attention_heads", None)
98 + self.total_num_heads = num_heads
99 assert self.total_num_heads % tp_size == 0
100 self.num_heads = self.total_num_heads // tp_size
123 -
124 - # defaut to mha
125 - self.total_num_kv_heads = getattr(config, "num_key_value_heads",
126 - self.total_num_heads)
101 + self.total_num_kv_heads = num_kv_heads
102 if self.total_num_kv_heads >= tp_size:
103 # Number of KV heads is greater than TP size, so we partition
104 # the KV heads across multiple tensor parallel GPUs.
@@ -133,68 +108,39 @@ class LlamaAttention(nn.Module):
108 # the KV heads across multiple tensor parallel GPUs.
109 assert tp_size % self.total_num_kv_heads == 0
110 self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
136 - self.head_dim = self.hidden_size // self.total_num_heads
111 + self.head_dim = hidden_size // self.total_num_heads
112 self.q_size = self.num_heads * self.head_dim
113 self.kv_size = self.num_kv_heads * self.head_dim
114 self.scaling = self.head_dim**-0.5
140 - max_position_embeddings = getattr(config, "max_position_embeddings",
141 - 8192)
142 - self.max_position_embeddings = config.max_position_embeddings
115 + self.rope_theta = rope_theta
116 + self.max_position_embeddings = max_position_embeddings
117
144 - # internlm
145 - bias = getattr(config, "bias", False)
146 -
147 - # stablelm
148 - qkv_bias = getattr(config, "use_qkv_bias", False)
118 self.qkv_proj = QKVParallelLinear(
150 - self.hidden_size,
119 + hidden_size,
120 self.head_dim,
121 self.total_num_heads,
122 self.total_num_kv_heads,
154 - bias=bias or qkv_bias,
123 + bias=False,
124 linear_method=linear_method,
125 )
126 self.o_proj = RowParallelLinear(
127 self.total_num_heads * self.head_dim,
159 - self.hidden_size,
160 - bias=bias,
128 + hidden_size,
129 + bias=False,
130 linear_method=linear_method,
131 )
132
164 - # mistral
165 - sliding_window = getattr(config, "sliding_window", None)
166 -
167 - self.postion_embedding = getattr(config, "postion_embedding", "ROPE")
168 - # Create the alibi slopes and slice them.
169 - if self.postion_embedding == "ALIBI":
170 - tp_rank = get_tensor_model_parallel_rank()
171 - head_start = tp_rank * self.num_heads
172 - head_end = (tp_rank + 1) * self.num_heads
173 - alibi_slopes = _get_alibi_slopes(self.total_num_heads)
174 - alibi_slopes = alibi_slopes[head_start:head_end].tolist()
175 -
176 - self.attn = PagedAttention(self.num_heads,
177 - self.head_dim,
178 - self.scaling,
179 - alibi_slopes=alibi_slopes,
180 - sliding_window=sliding_window)
181 - else:
182 - rope_theta = getattr(config, "rope_theta", 10000)
183 - rope_scaling = getattr(config, "rope_scaling", None)
184 - # stablelm
185 - rope_pct = getattr(config, "rope_pct", 1)
186 - self.rotary_emb = get_rope(
187 - self.head_dim,
188 - rotary_dim=int(self.head_dim * rope_pct),
189 - max_position=max_position_embeddings,
190 - base=rope_theta,
191 - rope_scaling=rope_scaling,
192 - )
193 - self.attn = PagedAttention(self.num_heads,
194 - self.head_dim,
195 - self.scaling,
196 - num_kv_heads=self.num_kv_heads,
197 - sliding_window=sliding_window)
133 + self.rotary_emb = get_rope(
134 + self.head_dim,
135 + rotary_dim=self.head_dim,
136 + max_position=max_position_embeddings,
137 + base=rope_theta,
138 + rope_scaling=rope_scaling,
139 + )
140 + self.attn = PagedAttention(self.num_heads,
141 + self.head_dim,
142 + self.scaling,
143 + num_kv_heads=self.num_kv_heads)
144
145 def forward(
146 self,
@@ -205,8 +151,7 @@ class LlamaAttention(nn.Module):
151 ) -> torch.Tensor:
152 qkv, _ = self.qkv_proj(hidden_states)
153 q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
208 - if self.postion_embedding != "ALIBI":
209 - q, k = self.rotary_emb(positions, q, k)
154 + q, k = self.rotary_emb(positions, q, k)
155 k_cache, v_cache = kv_cache
156 attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
157 output, _ = self.o_proj(attn_output)
@@ -219,20 +164,32 @@ class LlamaDecoderLayer(nn.Module):
164 self,
165 config: LlamaConfig,
166 linear_method: Optional[LinearMethodBase] = None,
222 - norm: Optional[torch.Tensor] = None,
167 ) -> None:
168 super().__init__()
169 self.hidden_size = config.hidden_size
170 + rope_theta = getattr(config, "rope_theta", 10000)
171 + rope_scaling = getattr(config, "rope_scaling", None)
172 + max_position_embeddings = getattr(config, "max_position_embeddings",
173 + 8192)
174 self.self_attn = LlamaAttention(
227 - config,
175 + hidden_size=self.hidden_size,
176 + num_heads=config.num_attention_heads,
177 + num_kv_heads=config.num_key_value_heads,
178 + rope_theta=rope_theta,
179 + rope_scaling=rope_scaling,
180 + max_position_embeddings=max_position_embeddings,
181 linear_method=linear_method,
182 )
183 self.mlp = LlamaMLP(
231 - config,
184 + hidden_size=self.hidden_size,
185 + intermediate_size=config.intermediate_size,
186 + hidden_act=config.hidden_act,
187 linear_method=linear_method,
188 )
234 - self.input_layernorm = deepcopy(norm)
235 - self.post_attention_layernorm = deepcopy(norm)
189 + self.input_layernorm = RMSNorm(config.hidden_size,
190 + eps=config.rms_norm_eps)
191 + self.post_attention_layernorm = RMSNorm(config.hidden_size,
192 + eps=config.rms_norm_eps)
193
194 def forward(
195 self,
@@ -269,7 +226,6 @@ class LlamaModel(nn.Module):
226 self,
227 config: LlamaConfig,
228 linear_method: Optional[LinearMethodBase] = None,
272 - norm: Optional[torch.Tensor] = None,
229 lora_config: Optional[LoRAConfig] = None,
230 ) -> None:
231 super().__init__()
@@ -285,10 +241,10 @@ class LlamaModel(nn.Module):
241 org_num_embeddings=config.vocab_size,
242 )
243 self.layers = nn.ModuleList([
288 - LlamaDecoderLayer(config, linear_method, norm)
244 + LlamaDecoderLayer(config, linear_method)
245 for _ in range(config.num_hidden_layers)
246 ])
291 - self.norm = norm
247 + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
248
249 def forward(
250 self,
@@ -319,18 +275,12 @@ class LlamaForCausalLM(nn.Module):
275 self,
276 config: LlamaConfig,
277 linear_method: Optional[LinearMethodBase] = None,
322 - norm: Optional[torch.Tensor] = None,
278 lora_config: Optional[LoRAConfig] = None,
279 ) -> None:
280 super().__init__()
281 self.config = config
282 self.linear_method = linear_method
328 - if norm is None:
329 - norm = RMSNorm(config.hidden_size, config.rms_norm_eps)
330 - self.model = LlamaModel(config,
331 - linear_method,
332 - norm=norm,
333 - lora_config=lora_config)
283 + self.model = LlamaModel(config, linear_method, lora_config=lora_config)
284 unpadded_vocab_size = config.vocab_size
285 if lora_config:
286 unpadded_vocab_size += lora_config.lora_extra_vocab_size
vllm/model_executor/models/mistral.py new
+352
@@ -0,0 +1,352 @@
1 +# coding=utf-8
2 +# Adapted from
3 +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
4 +# Copyright 2023 The vLLM team.
5 +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
6 +#
7 +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
8 +# and OPT implementations in this library. It has been modified from its
9 +# original forms to accommodate minor architectural differences compared
10 +# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
11 +#
12 +# Licensed under the Apache License, Version 2.0 (the "License");
13 +# you may not use this file except in compliance with the License.
14 +# You may obtain a copy of the License at
15 +#
16 +# http://www.apache.org/licenses/LICENSE-2.0
17 +#
18 +# Unless required by applicable law or agreed to in writing, software
19 +# distributed under the License is distributed on an "AS IS" BASIS,
20 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21 +# See the License for the specific language governing permissions and
22 +# limitations under the License.
23 +"""Inference-only Mistral model compatible with HuggingFace weights."""
24 +from typing import List, Optional, Tuple
25 +
26 +import torch
27 +from torch import nn
28 +from transformers import MistralConfig
29 +
30 +from vllm.model_executor.input_metadata import InputMetadata
31 +from vllm.model_executor.layers.activation import SiluAndMul
32 +from vllm.model_executor.layers.attention import PagedAttention
33 +from vllm.model_executor.layers.layernorm import RMSNorm
34 +from vllm.model_executor.layers.linear import (LinearMethodBase,
35 + MergedColumnParallelLinear,
36 + QKVParallelLinear,
37 + RowParallelLinear)
38 +from vllm.model_executor.layers.rotary_embedding import get_rope
39 +from vllm.model_executor.layers.sampler import Sampler
40 +from vllm.model_executor.layers.vocab_parallel_embedding import (
41 + VocabParallelEmbedding, ParallelLMHead, DEFAULT_VOCAB_PADDING_SIZE)
42 +from vllm.model_executor.parallel_utils.parallel_state import (
43 + get_tensor_model_parallel_world_size)
44 +from vllm.model_executor.sampling_metadata import SamplingMetadata
45 +from vllm.model_executor.weight_utils import (default_weight_loader,
46 + hf_model_weights_iterator)
47 +from vllm.sequence import SamplerOutput
48 +from vllm.config import LoRAConfig
49 +
50 +KVCache = Tuple[torch.Tensor, torch.Tensor]
51 +
52 +
53 +class MistralMLP(nn.Module):
54 +
55 + def __init__(
56 + self,
57 + hidden_size: int,
58 + intermediate_size: int,
59 + hidden_act: str,
60 + linear_method: Optional[LinearMethodBase] = None,
61 + ) -> None:
62 + super().__init__()
63 + self.gate_up_proj = MergedColumnParallelLinear(
64 + hidden_size, [intermediate_size] * 2,
65 + bias=False,
66 + linear_method=linear_method)
67 + self.down_proj = RowParallelLinear(intermediate_size,
68 + hidden_size,
69 + bias=False,
70 + linear_method=linear_method)
71 + if hidden_act != "silu":
72 + raise ValueError(f"Unsupported activation: {hidden_act}. "
73 + "Only silu is supported for now.")
74 + self.act_fn = SiluAndMul()
75 +
76 + def forward(self, x):
77 + gate_up, _ = self.gate_up_proj(x)
78 + x = self.act_fn(gate_up)
79 + x, _ = self.down_proj(x)
80 + return x
81 +
82 +
83 +class MistralAttention(nn.Module):
84 +
85 + def __init__(self,
86 + hidden_size: int,
87 + num_heads: int,
88 + num_kv_heads: int,
89 + max_position: int = 4096 * 32,
90 + rope_theta: float = 10000,
91 + linear_method: Optional[LinearMethodBase] = None,
92 + sliding_window: Optional[int] = None) -> None:
93 + super().__init__()
94 + self.hidden_size = hidden_size
95 + tp_size = get_tensor_model_parallel_world_size()
96 + self.total_num_heads = num_heads
97 + assert self.total_num_heads % tp_size == 0
98 + self.num_heads = self.total_num_heads // tp_size
99 + self.total_num_kv_heads = num_kv_heads
100 + if self.total_num_kv_heads >= tp_size:
101 + # Number of KV heads is greater than TP size, so we partition
102 + # the KV heads across multiple tensor parallel GPUs.
103 + assert self.total_num_kv_heads % tp_size == 0
104 + else:
105 + # Number of KV heads is less than TP size, so we replicate
106 + # the KV heads across multiple tensor parallel GPUs.
107 + assert tp_size % self.total_num_kv_heads == 0
108 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
109 + self.head_dim = hidden_size // self.total_num_heads
110 + self.q_size = self.num_heads * self.head_dim
111 + self.kv_size = self.num_kv_heads * self.head_dim
112 + self.scaling = self.head_dim**-0.5
113 + self.rope_theta = rope_theta
114 + self.sliding_window = sliding_window
115 +
116 + self.qkv_proj = QKVParallelLinear(
117 + hidden_size,
118 + self.head_dim,
119 + self.total_num_heads,
120 + self.total_num_kv_heads,
121 + bias=False,
122 + linear_method=linear_method,
123 + )
124 + self.o_proj = RowParallelLinear(
125 + self.total_num_heads * self.head_dim,
126 + hidden_size,
127 + bias=False,
128 + linear_method=linear_method,
129 + )
130 +
131 + self.rotary_emb = get_rope(
132 + self.head_dim,
133 + rotary_dim=self.head_dim,
134 + max_position=max_position,
135 + base=self.rope_theta,
136 + )
137 + self.attn = PagedAttention(self.num_heads,
138 + self.head_dim,
139 + self.scaling,
140 + num_kv_heads=self.num_kv_heads,
141 + sliding_window=self.sliding_window)
142 +
143 + def forward(
144 + self,
145 + positions: torch.Tensor,
146 + hidden_states: torch.Tensor,
147 + kv_cache: KVCache,
148 + input_metadata: InputMetadata,
149 + ) -> torch.Tensor:
150 + qkv, _ = self.qkv_proj(hidden_states)
151 + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
152 + q, k = self.rotary_emb(positions, q, k)
153 + k_cache, v_cache = kv_cache
154 + attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
155 + output, _ = self.o_proj(attn_output)
156 + return output
157 +
158 +
159 +class MistralDecoderLayer(nn.Module):
160 +
161 + def __init__(
162 + self,
163 + config: MistralConfig,
164 + linear_method: Optional[LinearMethodBase] = None,
165 + ) -> None:
166 + super().__init__()
167 + self.hidden_size = config.hidden_size
168 + # Requires transformers > 4.32.0
169 + rope_theta = getattr(config, "rope_theta", 10000)
170 + self.self_attn = MistralAttention(
171 + hidden_size=self.hidden_size,
172 + num_heads=config.num_attention_heads,
173 + max_position=config.max_position_embeddings,
174 + num_kv_heads=config.num_key_value_heads,
175 + rope_theta=rope_theta,
176 + linear_method=linear_method,
177 + sliding_window=config.sliding_window)
178 + self.mlp = MistralMLP(
179 + hidden_size=self.hidden_size,
180 + intermediate_size=config.intermediate_size,
181 + hidden_act=config.hidden_act,
182 + linear_method=linear_method,
183 + )
184 + self.input_layernorm = RMSNorm(config.hidden_size,
185 + eps=config.rms_norm_eps)
186 + self.post_attention_layernorm = RMSNorm(config.hidden_size,
187 + eps=config.rms_norm_eps)
188 +
189 + def forward(
190 + self,
191 + positions: torch.Tensor,
192 + hidden_states: torch.Tensor,
193 + kv_cache: KVCache,
194 + input_metadata: InputMetadata,
195 + residual: Optional[torch.Tensor],
196 + ) -> Tuple[torch.Tensor, torch.Tensor]:
197 + # Self Attention
198 + if residual is None:
199 + residual = hidden_states
200 + hidden_states = self.input_layernorm(hidden_states)
201 + else:
202 + hidden_states, residual = self.input_layernorm(
203 + hidden_states, residual)
204 + hidden_states = self.self_attn(
205 + positions=positions,
206 + hidden_states=hidden_states,
207 + kv_cache=kv_cache,
208 + input_metadata=input_metadata,
209 + )
210 +
211 + # Fully Connected
212 + hidden_states, residual = self.post_attention_layernorm(
213 + hidden_states, residual)
214 + hidden_states = self.mlp(hidden_states)
215 + return hidden_states, residual
216 +
217 +
218 +class MistralModel(nn.Module):
219 +
220 + def __init__(
221 + self,
222 + config: MistralConfig,
223 + linear_method: Optional[LinearMethodBase] = None,
224 + lora_config: Optional[LoRAConfig] = None,
225 + ) -> None:
226 + super().__init__()
227 + self.config = config
228 + self.padding_idx = config.pad_token_id
229 + lora_vocab = (lora_config.lora_extra_vocab_size *
230 + (lora_config.max_loras or 1)) if lora_config else 0
231 + self.vocab_size = config.vocab_size + lora_vocab
232 + self.org_vocab_size = config.vocab_size
233 +
234 + self.embed_tokens = VocabParallelEmbedding(
235 + self.vocab_size,
236 + config.hidden_size,
237 + org_num_embeddings=config.vocab_size,
238 + )
239 + self.layers = nn.ModuleList([
240 + MistralDecoderLayer(config, linear_method)
241 + for _ in range(config.num_hidden_layers)
242 + ])
243 + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
244 +
245 + def forward(
246 + self,
247 + input_ids: torch.Tensor,
248 + positions: torch.Tensor,
249 + kv_caches: List[KVCache],
250 + input_metadata: InputMetadata,
251 + ) -> torch.Tensor:
252 + hidden_states = self.embed_tokens(input_ids)
253 + residual = None
254 + for i in range(len(self.layers)):
255 + layer = self.layers[i]
256 + hidden_states, residual = layer(
257 + positions,
258 + hidden_states,
259 + kv_caches[i],
260 + input_metadata,
261 + residual,
262 + )
263 + hidden_states, _ = self.norm(hidden_states, residual)
264 + return hidden_states
265 +
266 +
267 +class MistralForCausalLM(nn.Module):
268 + supports_lora = True
269 +
270 + def __init__(
271 + self,
272 + config: MistralConfig,
273 + linear_method: Optional[LinearMethodBase] = None,
274 + lora_config: Optional[LoRAConfig] = None,
275 + ) -> None:
276 + super().__init__()
277 + self.config = config
278 + self.linear_method = linear_method
279 + self.model = MistralModel(config,
280 + linear_method,
281 + lora_config=lora_config)
282 + unpadded_vocab_size = config.vocab_size
283 + if lora_config:
284 + unpadded_vocab_size += lora_config.lora_extra_vocab_size
285 + self.lm_head = ParallelLMHead(
286 + unpadded_vocab_size,
287 + config.hidden_size,
288 + org_num_embeddings=config.vocab_size,
289 + padding_size=DEFAULT_VOCAB_PADDING_SIZE
290 + # We need bigger padding if using lora for kernel
291 + # compatibility
292 + if not lora_config else lora_config.lora_vocab_padding_size,
293 + )
294 + self.sampler = Sampler(unpadded_vocab_size, config.vocab_size)
295 +
296 + def forward(
297 + self,
298 + input_ids: torch.Tensor,
299 + positions: torch.Tensor,
300 + kv_caches: List[KVCache],
301 + input_metadata: InputMetadata,
302 + ) -> torch.Tensor:
303 + hidden_states = self.model(input_ids, positions, kv_caches,
304 + input_metadata)
305 + return hidden_states
306 +
307 + def sample(
308 + self,
309 + hidden_states: torch.Tensor,
310 + sampling_metadata: SamplingMetadata,
311 + ) -> Optional[SamplerOutput]:
312 + next_tokens = self.sampler(self.lm_head.weight, hidden_states,
313 + sampling_metadata)
314 + return next_tokens
315 +
316 + def load_weights(self,
317 + model_name_or_path: str,
318 + cache_dir: Optional[str] = None,
319 + load_format: str = "auto",
320 + revision: Optional[str] = None):
321 + stacked_params_mapping = [
322 + # (param_name, shard_name, shard_id)
323 + ("qkv_proj", "q_proj", "q"),
324 + ("qkv_proj", "k_proj", "k"),
325 + ("qkv_proj", "v_proj", "v"),
326 + ("gate_up_proj", "gate_proj", 0),
327 + ("gate_up_proj", "up_proj", 1),
328 + ]
329 + params_dict = dict(self.named_parameters())
330 + for name, loaded_weight in hf_model_weights_iterator(
331 + model_name_or_path, cache_dir, load_format, revision):
332 + if "rotary_emb.inv_freq" in name:
333 + continue
334 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
335 + if weight_name not in name:
336 + continue
337 + name = name.replace(weight_name, param_name)
338 + # Skip loading extra bias for GPTQ models.
339 + if name.endswith(".bias") and name not in params_dict:
340 + continue
341 + param = params_dict[name]
342 + weight_loader = param.weight_loader
343 + weight_loader(param, loaded_weight, shard_id)
344 + break
345 + else:
346 + # Skip loading extra bias for GPTQ models.
347 + if name.endswith(".bias") and name not in params_dict:
348 + continue
349 + param = params_dict[name]
350 + weight_loader = getattr(param, "weight_loader",
351 + default_weight_loader)
352 + weight_loader(param, loaded_weight)
vllm/model_executor/models/qwen.py
+236 -31
@@ -4,33 +4,253 @@
4 # Copyright (c) Alibaba Cloud.
5 # LICENSE: https://huggingface.co/Qwen/Qwen-7B/blob/main/LICENSE
6 """Inference-only QWen model compatible with HuggingFace weights."""
7 -from typing import Optional
7 +from typing import Any, Dict, List, Optional, Tuple
8
9 -from transformers import PretrainedConfig
10 -from vllm.config import LoRAConfig
9 +import torch
10 +from torch import nn
11
12 -from vllm.model_executor.layers.linear import LinearMethodBase
12 +from vllm.model_executor.input_metadata import InputMetadata
13 +from vllm.model_executor.layers.activation import SiluAndMul
14 +from vllm.model_executor.layers.attention import PagedAttention
15 from vllm.model_executor.layers.layernorm import RMSNorm
14 -from vllm.model_executor.models.llama import LlamaForCausalLM
16 +from vllm.model_executor.layers.linear import (LinearMethodBase,
17 + MergedColumnParallelLinear,
18 + QKVParallelLinear,
19 + RowParallelLinear)
20 +from vllm.model_executor.layers.rotary_embedding import get_rope
21 +from vllm.model_executor.layers.sampler import Sampler
22 +from vllm.model_executor.layers.vocab_parallel_embedding import (
23 + VocabParallelEmbedding, ParallelLMHead)
24 +from vllm.model_executor.parallel_utils.parallel_state import (
25 + get_tensor_model_parallel_world_size)
26 +from vllm.model_executor.sampling_metadata import SamplingMetadata
27 from vllm.model_executor.weight_utils import (default_weight_loader,
28 hf_model_weights_iterator)
29 +from vllm.sequence import SamplerOutput
30 +from vllm.transformers_utils.configs.qwen import QWenConfig
31
32 +KVCache = Tuple[torch.Tensor, torch.Tensor]
33
19 -class QWenLMHeadModel(LlamaForCausalLM):
34 +
35 +class QWenMLP(nn.Module):
36 +
37 + def __init__(
38 + self,
39 + hidden_size: int,
40 + intermediate_size: int,
41 + hidden_act: str = "silu",
42 + linear_method: Optional[LinearMethodBase] = None,
43 + ):
44 + super().__init__()
45 + self.gate_up_proj = MergedColumnParallelLinear(
46 + hidden_size, [intermediate_size] * 2,
47 + bias=False,
48 + linear_method=linear_method)
49 + self.c_proj = RowParallelLinear(intermediate_size,
50 + hidden_size,
51 + bias=False,
52 + linear_method=linear_method)
53 + if hidden_act != "silu":
54 + raise ValueError(f"Unsupported activation: {hidden_act}. "
55 + "Only silu is supported for now.")
56 + self.act_fn = SiluAndMul()
57 +
58 + def forward(self, x):
59 + gate_up, _ = self.gate_up_proj(x)
60 + x = self.act_fn(gate_up)
61 + x, _ = self.c_proj(x)
62 + return x
63 +
64 +
65 +class QWenAttention(nn.Module):
66 +
67 + def __init__(
68 + self,
69 + hidden_size: int,
70 + num_heads: int,
71 + max_position_embeddings: int,
72 + rope_theta: float = 10000,
73 + rope_scaling: Optional[Dict[str, Any]] = None,
74 + linear_method: Optional[LinearMethodBase] = None,
75 + ):
76 + super().__init__()
77 + self.hidden_size = hidden_size
78 + tensor_model_parallel_world_size = get_tensor_model_parallel_world_size(
79 + )
80 + self.total_num_heads = num_heads
81 + assert self.total_num_heads % tensor_model_parallel_world_size == 0
82 + self.num_heads = (self.total_num_heads //
83 + tensor_model_parallel_world_size)
84 + self.head_dim = hidden_size // self.total_num_heads
85 + self.c_attn = QKVParallelLinear(
86 + hidden_size,
87 + self.head_dim,
88 + self.total_num_heads,
89 + bias=True,
90 + linear_method=linear_method,
91 + )
92 + self.c_proj = RowParallelLinear(
93 + self.total_num_heads * self.head_dim,
94 + hidden_size,
95 + bias=False,
96 + linear_method=linear_method,
97 + )
98 + self.scaling = self.head_dim**-0.5
99 +
100 + self.rotary_emb = get_rope(
101 + self.head_dim,
102 + rotary_dim=self.head_dim,
103 + max_position=max_position_embeddings,
104 + base=rope_theta,
105 + rope_scaling=rope_scaling,
106 + )
107 + self.attn = PagedAttention(self.num_heads, self.head_dim, self.scaling)
108 +
109 + def forward(
110 + self,
111 + positions: torch.Tensor,
112 + hidden_states: torch.Tensor,
113 + kv_cache: KVCache,
114 + input_metadata: InputMetadata,
115 + ) -> torch.Tensor:
116 + qkv, _ = self.c_attn(hidden_states)
117 + q, k, v = qkv.chunk(chunks=3, dim=-1)
118 + q, k = self.rotary_emb(positions, q, k)
119 + k_cache, v_cache = kv_cache
120 + attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
121 +
122 + output, _ = self.c_proj(attn_output)
123 + return output
124 +
125 +
126 +class QWenBlock(nn.Module):
127 +
128 + def __init__(
129 + self,
130 + config: QWenConfig,
131 + linear_method: Optional[LinearMethodBase] = None,
132 + ):
133 + super().__init__()
134 + self.ln_1 = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
135 +
136 + rope_theta = getattr(config, "rope_theta", 10000)
137 + rope_scaling = getattr(config, "rope_scaling", None)
138 + self.attn = QWenAttention(config.hidden_size,
139 + config.num_attention_heads,
140 + config.max_position_embeddings,
141 + rope_theta=rope_theta,
142 + rope_scaling=rope_scaling,
143 + linear_method=linear_method)
144 +
145 + self.ln_2 = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
146 +
147 + self.mlp = QWenMLP(config.hidden_size,
148 + config.intermediate_size // 2,
149 + linear_method=linear_method)
150 +
151 + def forward(
152 + self,
153 + positions: torch.Tensor,
154 + hidden_states: torch.Tensor,
155 + kv_cache: KVCache,
156 + input_metadata: InputMetadata,
157 + residual: Optional[torch.Tensor],
158 + ) -> Tuple[torch.Tensor, torch.Tensor]:
159 + # Self Attention
160 + if residual is None:
161 + residual = hidden_states
162 + hidden_states = self.ln_1(hidden_states)
163 + else:
164 + hidden_states, residual = self.ln_1(hidden_states, residual)
165 + hidden_states = self.attn(
166 + positions=positions,
167 + hidden_states=hidden_states,
168 + kv_cache=kv_cache,
169 + input_metadata=input_metadata,
170 + )
171 +
172 + # Fully Connected
173 + hidden_states, residual = self.ln_2(hidden_states, residual)
174 + hidden_states = self.mlp(hidden_states)
175 + return hidden_states, residual
176 +
177 +
178 +class QWenModel(nn.Module):
179
180 def __init__(
181 self,
23 - config: Optional[PretrainedConfig] = None,
182 + config: QWenConfig,
183 linear_method: Optional[LinearMethodBase] = None,
25 - lora_config: Optional[LoRAConfig] = None,
26 - ) -> None:
27 - norm = RMSNorm(config.hidden_size, config.layer_norm_epsilon)
28 - config.use_qkv_bias = True
29 - config.intermediate_size = config.intermediate_size // 2
30 - super().__init__(config=config,
31 - linear_method=linear_method,
32 - norm=norm,
33 - lora_config=lora_config)
184 + ):
185 + super().__init__()
186 + self.config = config
187 + self.vocab_size = config.vocab_size
188 +
189 + self.wte = VocabParallelEmbedding(
190 + config.vocab_size,
191 + config.hidden_size,
192 + )
193 + self.h = nn.ModuleList([
194 + QWenBlock(config, linear_method)
195 + for _ in range(config.num_hidden_layers)
196 + ])
197 + self.ln_f = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
198 +
199 + def forward(
200 + self,
201 + input_ids: torch.Tensor,
202 + positions: torch.Tensor,
203 + kv_caches: List[KVCache],
204 + input_metadata: InputMetadata,
205 + ) -> torch.Tensor:
206 + hidden_states = self.wte(input_ids)
207 + residual = None
208 + for i in range(len(self.h)):
209 + layer = self.h[i]
210 + hidden_states, residual = layer(
211 + positions,
212 + hidden_states,
213 + kv_caches[i],
214 + input_metadata,
215 + residual,
216 + )
217 + hidden_states, _ = self.ln_f(hidden_states, residual)
218 + return hidden_states
219 +
220 +
221 +class QWenLMHeadModel(nn.Module):
222 +
223 + def __init__(
224 + self,
225 + config: QWenConfig,
226 + linear_method: Optional[LinearMethodBase] = None,
227 + ):
228 + super().__init__()
229 + self.config = config
230 + self.linear_method = linear_method
231 + self.transformer = QWenModel(config, linear_method)
232 + self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
233 + self.sampler = Sampler(config.vocab_size)
234 +
235 + def forward(
236 + self,
237 + input_ids: torch.Tensor,
238 + positions: torch.Tensor,
239 + kv_caches: List[KVCache],
240 + input_metadata: InputMetadata,
241 + ) -> torch.Tensor:
242 + hidden_states = self.transformer(input_ids, positions, kv_caches,
243 + input_metadata)
244 + return hidden_states
245 +
246 + def sample(
247 + self,
248 + hidden_states: torch.Tensor,
249 + sampling_metadata: SamplingMetadata,
250 + ) -> Optional[SamplerOutput]:
251 + next_tokens = self.sampler(self.lm_head.weight, hidden_states,
252 + sampling_metadata)
253 + return next_tokens
254
255 def load_weights(self,
256 model_name_or_path: str,
@@ -42,24 +262,9 @@ class QWenLMHeadModel(LlamaForCausalLM):
262 ("gate_up_proj", "w2", 0),
263 ("gate_up_proj", "w1", 1),
264 ]
45 - param_weight_map = [
46 - ("model", "transformer"),
47 - (".self_attn.", ".attn."),
48 - (".layers.", ".h."),
49 - ("qkv_proj", "c_attn"),
50 - (".self_attn.o_proj", ".self_attn.c_proj"),
51 - ("norm", "ln_f"),
52 - ("mlp.down_proj", "mlp.c_proj"),
53 - ("input_layernorm", "ln_1"),
54 - ("post_attention_layernorm", "ln_2"),
55 - ("embed_tokens", "wte"),
56 - ]
265 params_dict = dict(self.named_parameters())
266 for name, loaded_weight in hf_model_weights_iterator(
267 model_name_or_path, cache_dir, load_format, revision):
60 - for (param_name, weight_name) in param_weight_map:
61 - name = name.replace(weight_name, param_name)
62 -
268 if "rotary_emb.inv_freq" in name:
269 continue
270 for (param_name, weight_name, shard_id) in stacked_params_mapping:
vllm/model_executor/models/stablelm.py
+270 -13
@@ -17,26 +17,283 @@
17 # https://huggingface.co/stabilityai/stablelm-3b-4e1t/blob/main/modeling_stablelm_epoch.py
18 # https://huggingface.co/stabilityai/stablelm-3b-4e1t/blob/main/config.json
19 """Inference-only StabeLM (https://github.com/Stability-AI/StableLM) model compatible with HuggingFace weights."""
20 -from typing import Optional
20 +from typing import List, Optional, Tuple
21
22 +import torch
23 +from torch import nn
24 from transformers import PretrainedConfig
25
24 -from vllm.model_executor.layers.linear import LinearMethodBase
25 -from vllm.model_executor.layers.layernorm import LayerNorm
26 -from vllm.model_executor.models.llama import LlamaForCausalLM
27 -from vllm.config import LoRAConfig
26 +from vllm.model_executor.input_metadata import InputMetadata
27 +from vllm.model_executor.layers.activation import SiluAndMul
28 +from vllm.model_executor.layers.attention import PagedAttention
29 +from vllm.model_executor.layers.linear import (LinearMethodBase,
30 + MergedColumnParallelLinear,
31 + QKVParallelLinear,
32 + RowParallelLinear)
33 +from vllm.model_executor.layers.rotary_embedding import get_rope
34 +from vllm.model_executor.layers.sampler import Sampler
35 +from vllm.model_executor.layers.vocab_parallel_embedding import (
36 + VocabParallelEmbedding, ParallelLMHead)
37 +from vllm.model_executor.parallel_utils.parallel_state import (
38 + get_tensor_model_parallel_world_size)
39 +from vllm.model_executor.sampling_metadata import SamplingMetadata
40 +from vllm.model_executor.weight_utils import (default_weight_loader,
41 + hf_model_weights_iterator)
42 +from vllm.sequence import SamplerOutput
43
44 +KVCache = Tuple[torch.Tensor, torch.Tensor]
45
30 -class StablelmForCausalLM(LlamaForCausalLM):
46 +
47 +class StablelmMLP(nn.Module):
48 +
49 + def __init__(self,
50 + config: PretrainedConfig,
51 + linear_method: Optional[LinearMethodBase] = None) -> None:
52 + super().__init__()
53 + self.config = config
54 + self.hidden_size = config.hidden_size
55 + self.intermediate_size = config.intermediate_size
56 + self.gate_up_proj = MergedColumnParallelLinear(
57 + config.hidden_size, [config.intermediate_size] * 2,
58 + bias=False,
59 + linear_method=linear_method)
60 + self.down_proj = RowParallelLinear(config.intermediate_size,
61 + config.hidden_size,
62 + bias=False)
63 + self.act_fn = SiluAndMul()
64 +
65 + def forward(self, x: torch.Tensor) -> torch.Tensor:
66 + gate_up, _ = self.gate_up_proj(x)
67 + x = self.act_fn(gate_up)
68 + x, _ = self.down_proj(x)
69 + return x
70 +
71 +
72 +class StablelmAttention(nn.Module):
73 +
74 + def __init__(self,
75 + config: PretrainedConfig,
76 + linear_method: Optional[LinearMethodBase] = None) -> None:
77 + super().__init__()
78 + self.config = config
79 + self.hidden_size = config.hidden_size
80 + tp_size = get_tensor_model_parallel_world_size()
81 + self.total_num_heads = config.num_attention_heads
82 + self.num_heads = self.total_num_heads // tp_size
83 +
84 + self.total_num_key_value_heads = config.num_key_value_heads
85 + if self.total_num_key_value_heads >= tp_size:
86 + # Number of KV heads is greater than TP size, so we partition
87 + # the KV heads across multiple tensor parallel GPUs.
88 + assert self.total_num_key_value_heads % tp_size == 0
89 + else:
90 + # Number of KV heads is less than TP size, so we replicate
91 + # the KV heads across multiple tensor parallel GPUs.
92 + assert tp_size % self.total_num_key_value_heads == 0
93 + self.num_key_value_heads = max(
94 + 1, self.total_num_key_value_heads // tp_size)
95 + self.head_dim = self.hidden_size // self.total_num_heads
96 + self.max_position_embeddings = config.max_position_embeddings
97 + self.rotary_ndims = int(self.head_dim * self.config.rope_pct)
98 + self.scaling = self.head_dim**-0.5
99 + self.q_size = self.num_heads * self.head_dim
100 + self.kv_size = self.num_key_value_heads * self.head_dim
101 + self.qkv_bias = getattr(config, "use_qkv_bias", False)
102 + if (self.head_dim * self.num_heads * tp_size) != self.hidden_size:
103 + raise ValueError(
104 + f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}"
105 + f" and `num_heads`: {self.num_heads}).")
106 +
107 + self.qkv_proj = QKVParallelLinear(self.hidden_size,
108 + self.head_dim,
109 + self.total_num_heads,
110 + self.total_num_key_value_heads,
111 + self.qkv_bias,
112 + linear_method=linear_method)
113 + self.o_proj = RowParallelLinear(self.total_num_heads * self.head_dim,
114 + self.hidden_size,
115 + bias=False,
116 + linear_method=linear_method)
117 + self.rotary_ndims = int(self.head_dim * self.config.rope_pct)
118 + self.rotary_emb = get_rope(
119 + self.head_dim,
120 + rotary_dim=self.rotary_ndims,
121 + max_position=self.config.max_position_embeddings,
122 + base=self.config.rope_theta,
123 + )
124 + self.attn = PagedAttention(self.num_heads,
125 + self.head_dim,
126 + self.scaling,
127 + num_kv_heads=self.num_key_value_heads)
128 +
129 + def forward(
130 + self,
131 + positions: torch.Tensor,
132 + hidden_states: torch.Tensor,
133 + kv_cache: KVCache,
134 + input_metadata: InputMetadata,
135 + ) -> torch.Tensor:
136 + qkv, _ = self.qkv_proj(hidden_states)
137 + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
138 + q, k = self.rotary_emb(positions, q, k)
139 + k_cache, v_cache = kv_cache
140 + attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
141 + output, _ = self.o_proj(attn_output)
142 + return output
143 +
144 +
145 +class StablelmDecoderLayer(nn.Module):
146
147 def __init__(
148 self,
34 - config: Optional[PretrainedConfig] = None,
149 + config: PretrainedConfig,
150 linear_method: Optional[LinearMethodBase] = None,
36 - lora_config: Optional[LoRAConfig] = None,
151 ) -> None:
38 - norm = LayerNorm(config.hidden_size, config.norm_eps)
39 - super().__init__(config=config,
40 - linear_method=linear_method,
41 - norm=norm,
42 - lora_config=lora_config)
152 + super().__init__()
153 + self.self_attn = StablelmAttention(config)
154 + self.mlp = StablelmMLP(config, linear_method)
155 + self.input_layernorm = nn.LayerNorm(config.hidden_size,
156 + eps=config.norm_eps)
157 + self.post_attention_layernorm = nn.LayerNorm(config.hidden_size,
158 + eps=config.norm_eps)
159 +
160 + def forward(
161 + self,
162 + positions: torch.Tensor,
163 + hidden_states: torch.Tensor,
164 + kv_cache: KVCache,
165 + input_metadata: InputMetadata,
166 + ) -> Tuple[torch.Tensor, torch.Tensor]:
167 + # Self Attention
168 + residual = hidden_states
169 + hidden_states = self.input_layernorm(hidden_states)
170 + hidden_states = self.self_attn(
171 + positions=positions,
172 + hidden_states=hidden_states,
173 + kv_cache=kv_cache,
174 + input_metadata=input_metadata,
175 + )
176 + hidden_states = residual + hidden_states
177 +
178 + # Fully Connected
179 + residual = hidden_states
180 + hidden_states = self.post_attention_layernorm(hidden_states)
181 + hidden_states = self.mlp(hidden_states)
182 + hidden_states = residual + hidden_states
183 +
184 + return hidden_states, residual
185 +
186 +
187 +class StableLMEpochModel(nn.Module):
188 +
189 + def __init__(self,
190 + config: PretrainedConfig,
191 + linear_method: Optional[LinearMethodBase] = None) -> None:
192 + super().__init__()
193 + # self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, config.pad_token_id)
194 + self.embed_tokens = VocabParallelEmbedding(
195 + config.vocab_size,
196 + config.hidden_size,
197 + )
198 + self.layers = nn.ModuleList([
199 + StablelmDecoderLayer(config, linear_method)
200 + for _ in range(config.num_hidden_layers)
201 + ])
202 + self.norm = nn.LayerNorm(config.hidden_size, eps=config.norm_eps)
203 +
204 + def forward(
205 + self,
206 + input_ids: torch.Tensor,
207 + positions: torch.Tensor,
208 + kv_caches: List[KVCache],
209 + input_metadata: InputMetadata,
210 + ) -> torch.Tensor:
211 + hidden_states = self.embed_tokens(input_ids)
212 + for i in range(len(self.layers)):
213 + layer = self.layers[i]
214 + hidden_states, residual = layer(
215 + positions,
216 + hidden_states,
217 + kv_caches[i],
218 + input_metadata,
219 + )
220 + hidden_states = self.norm(hidden_states)
221 + return hidden_states
222 +
223 +
224 +class StablelmForCausalLM(nn.Module):
225 +
226 + def __init__(
227 + self,
228 + config: PretrainedConfig,
229 + linear_method: Optional[LinearMethodBase] = None,
230 + ) -> None:
231 + super().__init__()
232 + self.config = config
233 + self.linear_method = linear_method
234 + self.model = StableLMEpochModel(config, linear_method)
235 + self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
236 + self.sampler = Sampler(config.vocab_size)
237 +
238 + def forward(
239 + self,
240 + input_ids: torch.Tensor,
241 + positions: torch.Tensor,
242 + kv_caches: List[KVCache],
243 + input_metadata: InputMetadata,
244 + ) -> torch.Tensor:
245 + hidden_states = self.model(input_ids, positions, kv_caches,
246 + input_metadata)
247 + return hidden_states
248 +
249 + def sample(
250 + self,
251 + hidden_states: torch.Tensor,
252 + sampling_metadata: SamplingMetadata,
253 + ) -> Optional[SamplerOutput]:
254 + next_tokens = self.sampler(self.lm_head.weight, hidden_states,
255 + sampling_metadata)
256 + return next_tokens
257 +
258 + def load_weights(self,
259 + model_name_or_path: str,
260 + cache_dir: Optional[str] = None,
261 + load_format: str = "auto",
262 + revision: Optional[str] = None):
263 + stacked_params_mapping = [
264 + # (param_name, shard_name, shard_id)
265 + ("qkv_proj", "q_proj", "q"),
266 + ("qkv_proj", "k_proj", "k"),
267 + ("qkv_proj", "v_proj", "v"),
268 + ("gate_up_proj", "gate_proj", 0),
269 + ("gate_up_proj", "up_proj", 1),
270 + ]
271 + params_dict = dict(self.named_parameters())
272 + for name, loaded_weight in hf_model_weights_iterator(
273 + model_name_or_path, cache_dir, load_format, revision):
274 + if "rotary_emb.inv_freq" in name:
275 + continue
276 + if ("rotary_emb.cos_cached" in name
277 + or "rotary_emb.sin_cached" in name):
278 + # Models trained using ColossalAI may include these tensors in
279 + # the checkpoint. Skip them.
280 + continue
281 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
282 + if weight_name not in name:
283 + continue
284 + name = name.replace(weight_name, param_name)
285 + # Skip loading extra bias for GPTQ models.
286 + if name.endswith(".bias") and name not in params_dict:
287 + continue
288 + param = params_dict[name]
289 + weight_loader = param.weight_loader
290 + weight_loader(param, loaded_weight, shard_id)
291 + break
292 + else:
293 + # Skip loading extra bias for GPTQ models.
294 + if name.endswith(".bias") and name not in params_dict:
295 + continue
296 + param = params_dict[name]
297 + weight_loader = getattr(param, "weight_loader",
298 + default_weight_loader)
299 + weight_loader(param, loaded_weight)
vllm/model_executor/models/yi.py new
+330
@@ -0,0 +1,330 @@
1 +# coding=utf-8
2 +# Adapted from
3 +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
4 +# Copyright 2023 The vLLM team.
5 +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
6 +#
7 +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
8 +# and OPT implementations in this library. It has been modified from its
9 +# original forms to accommodate minor architectural differences compared
10 +# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
11 +#
12 +# Licensed under the Apache License, Version 2.0 (the "License");
13 +# you may not use this file except in compliance with the License.
14 +# You may obtain a copy of the License at
15 +#
16 +# http://www.apache.org/licenses/LICENSE-2.0
17 +#
18 +# Unless required by applicable law or agreed to in writing, software
19 +# distributed under the License is distributed on an "AS IS" BASIS,
20 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21 +# See the License for the specific language governing permissions and
22 +# limitations under the License.
23 +"""Inference-only Yi model (https://01.ai) compatible with HuggingFace weights."""
24 +from typing import Any, Dict, List, Optional, Tuple
25 +
26 +import torch
27 +from torch import nn
28 +from vllm.transformers_utils.configs.yi import YiConfig
29 +
30 +from vllm.model_executor.input_metadata import InputMetadata
31 +from vllm.model_executor.layers.activation import SiluAndMul
32 +from vllm.model_executor.layers.attention import PagedAttention
33 +from vllm.model_executor.layers.layernorm import RMSNorm
34 +from vllm.model_executor.layers.linear import (LinearMethodBase,
35 + MergedColumnParallelLinear,
36 + QKVParallelLinear,
37 + RowParallelLinear)
38 +from vllm.model_executor.layers.rotary_embedding import get_rope
39 +from vllm.model_executor.layers.sampler import Sampler
40 +from vllm.model_executor.layers.vocab_parallel_embedding import (
41 + VocabParallelEmbedding, ParallelLMHead)
42 +from vllm.model_executor.parallel_utils.parallel_state import (
43 + get_tensor_model_parallel_world_size)
44 +from vllm.model_executor.sampling_metadata import SamplingMetadata
45 +from vllm.model_executor.weight_utils import (default_weight_loader,
46 + hf_model_weights_iterator)
47 +from vllm.sequence import SamplerOutput
48 +
49 +KVCache = Tuple[torch.Tensor, torch.Tensor]
50 +
51 +
52 +class YiMLP(nn.Module):
53 +
54 + def __init__(
55 + self,
56 + hidden_size: int,
57 + intermediate_size: int,
58 + hidden_act: str,
59 + linear_method: Optional[LinearMethodBase] = None,
60 + ) -> None:
61 + super().__init__()
62 + self.gate_up_proj = MergedColumnParallelLinear(
63 + hidden_size, [intermediate_size] * 2,
64 + bias=False,
65 + linear_method=linear_method)
66 + self.down_proj = RowParallelLinear(intermediate_size,
67 + hidden_size,
68 + bias=False,
69 + linear_method=linear_method)
70 + if hidden_act != "silu":
71 + raise ValueError(f"Unsupported activation: {hidden_act}. "
72 + "Only silu is supported for now.")
73 + self.act_fn = SiluAndMul()
74 +
75 + def forward(self, x):
76 + gate_up, _ = self.gate_up_proj(x)
77 + x = self.act_fn(gate_up)
78 + x, _ = self.down_proj(x)
79 + return x
80 +
81 +
82 +class YiAttention(nn.Module):
83 +
84 + def __init__(
85 + self,
86 + hidden_size: int,
87 + num_heads: int,
88 + num_kv_heads: int,
89 + rope_theta: float = 10000,
90 + rope_scaling: Optional[Dict[str, Any]] = None,
91 + max_position_embeddings: int = 8192,
92 + linear_method: Optional[LinearMethodBase] = None,
93 + ) -> None:
94 + super().__init__()
95 + self.hidden_size = hidden_size
96 + tp_size = get_tensor_model_parallel_world_size()
97 + self.total_num_heads = num_heads
98 + assert self.total_num_heads % tp_size == 0
99 + self.num_heads = self.total_num_heads // tp_size
100 + self.total_num_kv_heads = num_kv_heads
101 + if self.total_num_kv_heads >= tp_size:
102 + # Number of KV heads is greater than TP size, so we partition
103 + # the KV heads across multiple tensor parallel GPUs.
104 + assert self.total_num_kv_heads % tp_size == 0
105 + else:
106 + # Number of KV heads is less than TP size, so we replicate
107 + # the KV heads across multiple tensor parallel GPUs.
108 + assert tp_size % self.total_num_kv_heads == 0
109 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
110 + self.head_dim = hidden_size // self.total_num_heads
111 + self.q_size = self.num_heads * self.head_dim
112 + self.kv_size = self.num_kv_heads * self.head_dim
113 + self.scaling = self.head_dim**-0.5
114 + self.rope_theta = rope_theta
115 + self.max_position_embeddings = max_position_embeddings
116 +
117 + self.qkv_proj = QKVParallelLinear(
118 + hidden_size,
119 + self.head_dim,
120 + self.total_num_heads,
121 + self.total_num_kv_heads,
122 + bias=False,
123 + linear_method=linear_method,
124 + )
125 + self.o_proj = RowParallelLinear(
126 + self.total_num_heads * self.head_dim,
127 + hidden_size,
128 + bias=False,
129 + linear_method=linear_method,
130 + )
131 + self.rotary_emb = get_rope(
132 + self.head_dim,
133 + rotary_dim=self.head_dim,
134 + max_position=max_position_embeddings,
135 + base=self.rope_theta,
136 + rope_scaling=rope_scaling,
137 + )
138 + self.attn = PagedAttention(self.num_heads,
139 + self.head_dim,
140 + self.scaling,
141 + num_kv_heads=self.num_kv_heads)
142 +
143 + def forward(
144 + self,
145 + positions: torch.Tensor,
146 + hidden_states: torch.Tensor,
147 + kv_cache: KVCache,
148 + input_metadata: InputMetadata,
149 + ) -> torch.Tensor:
150 + qkv, _ = self.qkv_proj(hidden_states)
151 + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
152 + q, k = self.rotary_emb(positions, q, k)
153 + k_cache, v_cache = kv_cache
154 + attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
155 + output, _ = self.o_proj(attn_output)
156 + return output
157 +
158 +
159 +class YiDecoderLayer(nn.Module):
160 +
161 + def __init__(
162 + self,
163 + config: YiConfig,
164 + linear_method: Optional[LinearMethodBase] = None,
165 + ) -> None:
166 + super().__init__()
167 + self.hidden_size = config.hidden_size
168 + rope_theta = getattr(config, "rope_theta", 10000)
169 + rope_scaling = getattr(config, "rope_scaling", None)
170 + max_position_embeddings = getattr(config, "max_position_embeddings",
171 + 8192)
172 + self.self_attn = YiAttention(
173 + hidden_size=self.hidden_size,
174 + num_heads=config.num_attention_heads,
175 + num_kv_heads=config.num_key_value_heads,
176 + rope_theta=rope_theta,
177 + rope_scaling=rope_scaling,
178 + max_position_embeddings=max_position_embeddings,
179 + linear_method=linear_method,
180 + )
181 + self.mlp = YiMLP(
182 + hidden_size=self.hidden_size,
183 + intermediate_size=config.intermediate_size,
184 + hidden_act=config.hidden_act,
185 + linear_method=linear_method,
186 + )
187 + self.ln1 = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
188 + self.ln2 = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
189 +
190 + def forward(
191 + self,
192 + positions: torch.Tensor,
193 + hidden_states: torch.Tensor,
194 + kv_cache: KVCache,
195 + input_metadata: InputMetadata,
196 + residual: Optional[torch.Tensor],
197 + ) -> Tuple[torch.Tensor, torch.Tensor]:
198 + # Self Attention
199 + if residual is None:
200 + residual = hidden_states
201 + hidden_states = self.ln1(hidden_states)
202 + else:
203 + hidden_states, residual = self.ln1(hidden_states, residual)
204 + hidden_states = self.self_attn(
205 + positions=positions,
206 + hidden_states=hidden_states,
207 + kv_cache=kv_cache,
208 + input_metadata=input_metadata,
209 + )
210 +
211 + # Fully Connected
212 + hidden_states, residual = self.ln2(hidden_states, residual)
213 + hidden_states = self.mlp(hidden_states)
214 + return hidden_states, residual
215 +
216 +
217 +class YiModel(nn.Module):
218 +
219 + def __init__(
220 + self,
221 + config: YiConfig,
222 + linear_method: Optional[LinearMethodBase] = None,
223 + ) -> None:
224 + super().__init__()
225 + self.config = config
226 + self.padding_idx = config.pad_token_id
227 + self.vocab_size = config.vocab_size
228 + self.embed_tokens = VocabParallelEmbedding(
229 + config.vocab_size,
230 + config.hidden_size,
231 + )
232 + self.layers = nn.ModuleList([
233 + YiDecoderLayer(config, linear_method)
234 + for _ in range(config.num_hidden_layers)
235 + ])
236 + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
237 +
238 + def forward(
239 + self,
240 + input_ids: torch.Tensor,
241 + positions: torch.Tensor,
242 + kv_caches: List[KVCache],
243 + input_metadata: InputMetadata,
244 + ) -> torch.Tensor:
245 + hidden_states = self.embed_tokens(input_ids)
246 + residual = None
247 + for i in range(len(self.layers)):
248 + layer = self.layers[i]
249 + hidden_states, residual = layer(
250 + positions,
251 + hidden_states,
252 + kv_caches[i],
253 + input_metadata,
254 + residual,
255 + )
256 + hidden_states, _ = self.norm(hidden_states, residual)
257 + return hidden_states
258 +
259 +
260 +class YiForCausalLM(nn.Module):
261 +
262 + def __init__(
263 + self,
264 + config: YiConfig,
265 + linear_method: Optional[LinearMethodBase] = None,
266 + ) -> None:
267 + super().__init__()
268 + self.config = config
269 + self.linear_method = linear_method
270 + self.model = YiModel(config, linear_method)
271 + self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
272 + self.sampler = Sampler(config.vocab_size)
273 +
274 + def forward(
275 + self,
276 + input_ids: torch.Tensor,
277 + positions: torch.Tensor,
278 + kv_caches: List[KVCache],
279 + input_metadata: InputMetadata,
280 + ) -> torch.Tensor:
281 + hidden_states = self.model(input_ids, positions, kv_caches,
282 + input_metadata)
283 + return hidden_states
284 +
285 + def sample(
286 + self,
287 + hidden_states: torch.Tensor,
288 + sampling_metadata: SamplingMetadata,
289 + ) -> Optional[SamplerOutput]:
290 + next_tokens = self.sampler(self.lm_head.weight, hidden_states,
291 + sampling_metadata)
292 + return next_tokens
293 +
294 + def load_weights(self,
295 + model_name_or_path: str,
296 + cache_dir: Optional[str] = None,
297 + load_format: str = "auto",
298 + revision: Optional[str] = None):
299 + stacked_params_mapping = [
300 + # (param_name, shard_name, shard_id)
301 + ("qkv_proj", "q_proj", "q"),
302 + ("qkv_proj", "k_proj", "k"),
303 + ("qkv_proj", "v_proj", "v"),
304 + ("gate_up_proj", "gate_proj", 0),
305 + ("gate_up_proj", "up_proj", 1),
306 + ]
307 + params_dict = dict(self.named_parameters())
308 + for name, loaded_weight in hf_model_weights_iterator(
309 + model_name_or_path, cache_dir, load_format, revision):
310 + if "rotary_emb.inv_freq" in name:
311 + continue
312 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
313 + if weight_name not in name:
314 + continue
315 + name = name.replace(weight_name, param_name)
316 + # Skip loading extra bias for GPTQ models.
317 + if name.endswith(".bias") and name not in params_dict:
318 + continue
319 + param = params_dict[name]
320 + weight_loader = param.weight_loader
321 + weight_loader(param, loaded_weight, shard_id)
322 + break
323 + else:
324 + # Skip loading extra bias for GPTQ models.
325 + if name.endswith(".bias") and name not in params_dict:
326 + continue
327 + param = params_dict[name]
328 + weight_loader = getattr(param, "weight_loader",
329 + default_weight_loader)
330 + weight_loader(param, loaded_weight)
vllm/transformers_utils/config.py
+4
@@ -5,10 +5,14 @@ from transformers import AutoConfig, PretrainedConfig
5 from vllm.transformers_utils.configs import *
6
7 _CONFIG_REGISTRY = {
8 + "aquila": AquilaConfig,
9 + "baichuan": BaiChuanConfig,
10 "chatglm": ChatGLMConfig,
11 "mpt": MPTConfig,
12 + "qwen": QWenConfig,
13 "RefinedWeb": RWConfig, # For tiiuae/falcon-40b(-instruct)
14 "RefinedWebModel": RWConfig, # For tiiuae/falcon-7b(-instruct)
15 + "yi": YiConfig,
16 }
17
18
vllm/transformers_utils/configs/__init__.py
+8
@@ -1,12 +1,20 @@
1 +from vllm.transformers_utils.configs.aquila import AquilaConfig
2 +from vllm.transformers_utils.configs.baichuan import BaiChuanConfig
3 from vllm.transformers_utils.configs.chatglm import ChatGLMConfig
4 from vllm.transformers_utils.configs.mpt import MPTConfig
5 +from vllm.transformers_utils.configs.qwen import QWenConfig
6 # RWConfig is for the original tiiuae/falcon-40b(-instruct) and
7 # tiiuae/falcon-7b(-instruct) models. Newer Falcon models will use the
8 # `FalconConfig` class from the official HuggingFace transformers library.
9 from vllm.transformers_utils.configs.falcon import RWConfig
10 +from vllm.transformers_utils.configs.yi import YiConfig
11
12 __all__ = [
13 + "AquilaConfig",
14 + "BaiChuanConfig",
15 "ChatGLMConfig",
16 "MPTConfig",
17 + "QWenConfig",
18 "RWConfig",
19 + "YiConfig",
20 ]
vllm/transformers_utils/configs/aquila.py new
+69
@@ -0,0 +1,69 @@
1 +# coding=utf-8
2 +# Copyright 2023 EleutherAI and the HuggingFace Inc. team. All rights reserved.
3 +#
4 +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
5 +# and OPT implementations in this library. It has been modified from its
6 +# original forms to accommodate minor architectural differences compared
7 +# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
8 +#
9 +# Licensed under the Apache License, Version 2.0 (the "License");
10 +# you may not use this file except in compliance with the License.
11 +# You may obtain a copy of the License at
12 +#
13 +# http://www.apache.org/licenses/LICENSE-2.0
14 +#
15 +# Unless required by applicable law or agreed to in writing, software
16 +# distributed under the License is distributed on an "AS IS" BASIS,
17 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18 +# See the License for the specific language governing permissions and
19 +# limitations under the License.
20 +""" Aquila model configuration"""
21 +
22 +from transformers import PretrainedConfig
23 +
24 +
25 +class AquilaConfig(PretrainedConfig):
26 + model_type = "aquila"
27 + keys_to_ignore_at_inference = ["past_key_values"]
28 +
29 + def __init__(
30 + self,
31 + vocab_size=100008,
32 + hidden_size=4096,
33 + intermediate_size=11008,
34 + num_hidden_layers=32,
35 + num_attention_heads=32,
36 + num_key_value_heads=None,
37 + hidden_act="silu",
38 + max_position_embeddings=2048,
39 + initializer_range=0.006,
40 + rms_norm_eps=1e-5,
41 + use_cache=True,
42 + pad_token_id=0,
43 + bos_token_id=1,
44 + eos_token_id=2,
45 + tie_word_embeddings=False,
46 + **kwargs,
47 + ):
48 + self.vocab_size = vocab_size
49 + self.max_position_embeddings = max_position_embeddings
50 + self.hidden_size = hidden_size
51 + self.intermediate_size = intermediate_size
52 + self.num_hidden_layers = num_hidden_layers
53 + # for backward compatibility
54 + if num_key_value_heads is None:
55 + num_key_value_heads = num_attention_heads
56 +
57 + self.num_key_value_heads = num_key_value_heads
58 + self.num_attention_heads = num_attention_heads
59 + self.hidden_act = hidden_act
60 + self.initializer_range = initializer_range
61 + self.rms_norm_eps = rms_norm_eps
62 + self.use_cache = use_cache
63 + super().__init__(
64 + pad_token_id=pad_token_id,
65 + bos_token_id=bos_token_id,
66 + eos_token_id=eos_token_id,
67 + tie_word_embeddings=tie_word_embeddings,
68 + **kwargs,
69 + )
vllm/transformers_utils/configs/baichuan.py new
+62
@@ -0,0 +1,62 @@
1 +# coding=utf-8
2 +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
3 +#
4 +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
5 +# and OPT implementations in this library. It has been modified from its
6 +# original forms to accommodate minor architectural differences compared
7 +# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
8 +#
9 +# Licensed under the Apache License, Version 2.0 (the "License");
10 +# you may not use this file except in compliance with the License.
11 +# You may obtain a copy of the License at
12 +#
13 +# http://www.apache.org/licenses/LICENSE-2.0
14 +#
15 +# Unless required by applicable law or agreed to in writing, software
16 +# distributed under the License is distributed on an "AS IS" BASIS,
17 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18 +# See the License for the specific language governing permissions and
19 +# limitations under the License.
20 +
21 +from transformers.configuration_utils import PretrainedConfig
22 +
23 +
24 +class BaiChuanConfig(PretrainedConfig):
25 + model_type = "baichuan"
26 + keys_to_ignore_at_inference = ["past_key_values"]
27 +
28 + def __init__(
29 + self,
30 + vocab_size=64000,
31 + hidden_size=4096,
32 + intermediate_size=11008,
33 + num_hidden_layers=32,
34 + num_attention_heads=32,
35 + hidden_act="silu",
36 + max_position_embeddings=4096,
37 + initializer_range=0.02,
38 + rms_norm_eps=1e-6,
39 + use_cache=True,
40 + pad_token_id=0,
41 + bos_token_id=1,
42 + eos_token_id=2,
43 + tie_word_embeddings=False,
44 + **kwargs,
45 + ):
46 + self.vocab_size = vocab_size
47 + self.max_position_embeddings = max_position_embeddings
48 + self.hidden_size = hidden_size
49 + self.intermediate_size = intermediate_size
50 + self.num_hidden_layers = num_hidden_layers
51 + self.num_attention_heads = num_attention_heads
52 + self.hidden_act = hidden_act
53 + self.initializer_range = initializer_range
54 + self.rms_norm_eps = rms_norm_eps
55 + self.use_cache = use_cache
56 + super().__init__(
57 + pad_token_id=pad_token_id,
58 + bos_token_id=bos_token_id,
59 + eos_token_id=eos_token_id,
60 + tie_word_embeddings=tie_word_embeddings,
61 + **kwargs,
62 + )
vllm/transformers_utils/configs/qwen.py new
+60
@@ -0,0 +1,60 @@
1 +# Copyright (c) Alibaba Cloud.
2 +# LICENSE: https://huggingface.co/Qwen/Qwen-7B/blob/main/LICENSE
3 +
4 +from transformers import PretrainedConfig
5 +
6 +
7 +class QWenConfig(PretrainedConfig):
8 + model_type = "qwen"
9 + keys_to_ignore_at_inference = ["past_key_values"]
10 +
11 + def __init__(
12 + self,
13 + vocab_size=151936,
14 + hidden_size=4096,
15 + num_hidden_layers=32,
16 + num_attention_heads=32,
17 + emb_dropout_prob=0.0,
18 + attn_dropout_prob=0.0,
19 + layer_norm_epsilon=1e-6,
20 + initializer_range=0.02,
21 + max_position_embeddings=8192,
22 + scale_attn_weights=True,
23 + use_cache=True,
24 + bf16=False,
25 + fp16=False,
26 + fp32=False,
27 + kv_channels=128,
28 + rotary_pct=1.0,
29 + rotary_emb_base=10000,
30 + use_dynamic_ntk=True,
31 + use_logn_attn=True,
32 + use_flash_attn="auto",
33 + intermediate_size=22016,
34 + no_bias=True,
35 + tie_word_embeddings=False,
36 + **kwargs,
37 + ):
38 + self.vocab_size = vocab_size
39 + self.hidden_size = hidden_size
40 + self.intermediate_size = intermediate_size
41 + self.num_hidden_layers = num_hidden_layers
42 + self.num_attention_heads = num_attention_heads
43 + self.emb_dropout_prob = emb_dropout_prob
44 + self.attn_dropout_prob = attn_dropout_prob
45 + self.layer_norm_epsilon = layer_norm_epsilon
46 + self.initializer_range = initializer_range
47 + self.scale_attn_weights = scale_attn_weights
48 + self.use_cache = use_cache
49 + self.max_position_embeddings = max_position_embeddings
50 + self.bf16 = bf16
51 + self.fp16 = fp16
52 + self.fp32 = fp32
53 + self.kv_channels = kv_channels
54 + self.rotary_pct = rotary_pct
55 + self.rotary_emb_base = rotary_emb_base
56 + self.use_dynamic_ntk = use_dynamic_ntk
57 + self.use_logn_attn = use_logn_attn
58 + self.use_flash_attn = use_flash_attn
59 + self.no_bias = no_bias
60 + super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)
vllm/transformers_utils/configs/yi.py new
+64
@@ -0,0 +1,64 @@
1 +""" Yi model configuration"""
2 +from transformers.configuration_utils import PretrainedConfig
3 +from transformers.utils import logging
4 +
5 +logger = logging.get_logger(__name__)
6 +
7 +Yi_PRETRAINED_CONFIG_ARCHIVE_MAP = {}
8 +
9 +
10 +class YiConfig(PretrainedConfig):
11 + r"""
12 + Reference:
13 + https://huggingface.co/01-ai/Yi-6B/blob/main/configuration_yi.py
14 + """
15 + model_type = "Yi"
16 + keys_to_ignore_at_inference = ["past_key_values"]
17 +
18 + def __init__(
19 + self,
20 + vocab_size=64000,
21 + hidden_size=4096,
22 + intermediate_size=11008,
23 + num_hidden_layers=32,
24 + num_attention_heads=32,
25 + num_key_value_heads=4,
26 + hidden_act="silu",
27 + max_position_embeddings=4096,
28 + initializer_range=0.02,
29 + rms_norm_eps=1e-5,
30 + use_cache=True,
31 + pad_token_id=0,
32 + bos_token_id=1,
33 + eos_token_id=2,
34 + tie_word_embeddings=False,
35 + output_attentions=False,
36 + rope_theta=5000000.0,
37 + **kwargs,
38 + ):
39 + self.vocab_size = vocab_size
40 + self.max_position_embeddings = max_position_embeddings
41 + self.hidden_size = hidden_size
42 + self.intermediate_size = intermediate_size
43 + self.num_hidden_layers = num_hidden_layers
44 + self.num_attention_heads = num_attention_heads
45 +
46 + # for backward compatibility
47 + if num_key_value_heads is None:
48 + num_key_value_heads = num_attention_heads
49 +
50 + self.num_key_value_heads = num_key_value_heads
51 + self.hidden_act = hidden_act
52 + self.initializer_range = initializer_range
53 + self.rms_norm_eps = rms_norm_eps
54 + self.use_cache = use_cache
55 + self.output_attentions = output_attentions
56 + self.rope_theta = rope_theta
57 +
58 + super().__init__(
59 + pad_token_id=pad_token_id,
60 + bos_token_id=bos_token_id,
61 + eos_token_id=eos_token_id,
62 + tie_word_embeddings=tie_word_embeddings,
63 + **kwargs,
64 + )