@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5c976a7e1a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Refactor llama family models (#2637)

Roy committed Feb 13, 2024 at 16:09 UTC 5c976a7e1a1bec875bf6474824b7dff39e38de18
17 files changed +236 -2720
vllm/model_executor/layers/layernorm.py
+25
@@ -7,6 +7,31 @@ import torch.nn as nn
7 from vllm._C import ops
8
9
10 +class LayerNorm(nn.LayerNorm):
11 +
12 + def __init__(
13 + self,
14 + hidden_size: int,
15 + eps: float = 1e-6,
16 + ) -> None:
17 + super().__init__(hidden_size, eps=eps)
18 +
19 + def forward(
20 + self,
21 + x: torch.Tensor,
22 + residual: Optional[torch.Tensor] = None,
23 + ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
24 + """normalization."""
25 + if residual is not None:
26 + x = x + residual
27 + residual = x
28 + x = super().forward(x)
29 + if residual is None:
30 + return x
31 + else:
32 + return x, residual
33 +
34 +
35 class RMSNorm(nn.Module):
36 """Root mean square normalization.
37
vllm/model_executor/models/__init__.py
+4 -5
@@ -10,8 +10,8 @@ logger = init_logger(__name__)
10
11 # Architecture -> (module, class).
12 _MODELS = {
13 - "AquilaModel": ("aquila", "AquilaForCausalLM"),
14 - "AquilaForCausalLM": ("aquila", "AquilaForCausalLM"), # AquilaChat2
13 + "AquilaModel": ("llama", "LlamaForCausalLM"),
14 + "AquilaForCausalLM": ("llama", "LlamaForCausalLM"), # AquilaChat2
15 "BaiChuanForCausalLM": ("baichuan", "BaiChuanForCausalLM"), # baichuan-7b
16 "BaichuanForCausalLM": ("baichuan", "BaichuanForCausalLM"), # baichuan-13b
17 "BloomForCausalLM": ("bloom", "BloomForCausalLM"),
@@ -24,12 +24,12 @@ _MODELS = {
24 "GPTBigCodeForCausalLM": ("gpt_bigcode", "GPTBigCodeForCausalLM"),
25 "GPTJForCausalLM": ("gpt_j", "GPTJForCausalLM"),
26 "GPTNeoXForCausalLM": ("gpt_neox", "GPTNeoXForCausalLM"),
27 - "InternLMForCausalLM": ("internlm", "InternLMForCausalLM"),
27 + "InternLMForCausalLM": ("llama", "LlamaForCausalLM"),
28 "InternLM2ForCausalLM": ("internlm2", "InternLM2ForCausalLM"),
29 "LlamaForCausalLM": ("llama", "LlamaForCausalLM"),
30 # For decapoda-research/llama-*
31 "LLaMAForCausalLM": ("llama", "LlamaForCausalLM"),
32 - "MistralForCausalLM": ("mistral", "MistralForCausalLM"),
32 + "MistralForCausalLM": ("llama", "LlamaForCausalLM"),
33 "MixtralForCausalLM": ("mixtral", "MixtralForCausalLM"),
34 "QuantMixtralForCausalLM": ("mixtral_quant", "MixtralForCausalLM"),
35 # transformers's mpt class has lower case
@@ -41,7 +41,6 @@ _MODELS = {
41 "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
42 "RWForCausalLM": ("falcon", "FalconForCausalLM"),
43 "StableLMEpochForCausalLM": ("stablelm", "StablelmForCausalLM"),
44 - "YiForCausalLM": ("yi", "YiForCausalLM")
44 }
45
46 # Models not supported by ROCm.
vllm/model_executor/models/aquila.py deleted
-342
@@ -1,342 +0,0 @@
1 -# coding=utf-8
2 -# Adapted from
3 -# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
4 -# Copyright 2023 The vLLM team.
5 -# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
6 -#
7 -# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
8 -# and OPT implementations in this library. It has been modified from its
9 -# original forms to accommodate minor architectural differences compared
10 -# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
11 -#
12 -# Licensed under the Apache License, Version 2.0 (the "License");
13 -# you may not use this file except in compliance with the License.
14 -# You may obtain a copy of the License at
15 -#
16 -# http://www.apache.org/licenses/LICENSE-2.0
17 -#
18 -# Unless required by applicable law or agreed to in writing, software
19 -# distributed under the License is distributed on an "AS IS" BASIS,
20 -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21 -# See the License for the specific language governing permissions and
22 -# limitations under the License.
23 -"""Inference-only LLaMA model compatible with HuggingFace weights."""
24 -from typing import Any, Dict, List, Optional, Tuple
25 -
26 -import torch
27 -from torch import nn
28 -
29 -from vllm.model_executor.input_metadata import InputMetadata
30 -from vllm.model_executor.layers.activation import SiluAndMul
31 -from vllm.model_executor.layers.attention import PagedAttention
32 -from vllm.model_executor.layers.linear import (LinearMethodBase,
33 - MergedColumnParallelLinear,
34 - QKVParallelLinear,
35 - RowParallelLinear)
36 -from vllm.model_executor.layers.rotary_embedding import get_rope
37 -from vllm.model_executor.layers.sampler import Sampler
38 -from vllm.model_executor.layers.vocab_parallel_embedding import (
39 - VocabParallelEmbedding, ParallelLMHead)
40 -from vllm.model_executor.parallel_utils.parallel_state import (
41 - get_tensor_model_parallel_world_size)
42 -from vllm.model_executor.sampling_metadata import SamplingMetadata
43 -from vllm.model_executor.weight_utils import (default_weight_loader,
44 - hf_model_weights_iterator)
45 -from vllm.sequence import SamplerOutput
46 -from vllm.transformers_utils.configs.aquila import AquilaConfig
47 -
48 -KVCache = Tuple[torch.Tensor, torch.Tensor]
49 -
50 -
51 -class AquilaMLP(nn.Module):
52 -
53 - def __init__(
54 - self,
55 - hidden_size: int,
56 - intermediate_size: int,
57 - hidden_act: str,
58 - linear_method: Optional[LinearMethodBase] = None,
59 - ):
60 - super().__init__()
61 - self.gate_up_proj = MergedColumnParallelLinear(
62 - hidden_size, [intermediate_size] * 2,
63 - bias=False,
64 - linear_method=linear_method)
65 - self.down_proj = RowParallelLinear(intermediate_size,
66 - hidden_size,
67 - bias=False,
68 - linear_method=linear_method)
69 - if hidden_act != "silu":
70 - raise ValueError(f"Unsupported activation: {hidden_act}. "
71 - "Only silu is supported for now.")
72 - self.act_fn = SiluAndMul()
73 -
74 - def forward(self, x):
75 - gate_up, _ = self.gate_up_proj(x)
76 - x = self.act_fn(gate_up)
77 - x, _ = self.down_proj(x)
78 - return x
79 -
80 -
81 -class AquilaRMSNorm(nn.Module):
82 -
83 - def __init__(self, hidden_size, eps=1e-6):
84 - """
85 - AquilaRMSNorm is equivalent to T5LayerNorm
86 - """
87 - super().__init__()
88 - self.weight = nn.Parameter(torch.ones(hidden_size))
89 - self.variance_epsilon = eps
90 -
91 - def forward(self, hidden_states):
92 - input_dtype = hidden_states.dtype
93 - variance = hidden_states.to(torch.float32).pow(2).mean(-1,
94 - keepdim=True)
95 - hidden_states = hidden_states * torch.rsqrt(variance +
96 - self.variance_epsilon)
97 -
98 - return (self.weight * hidden_states).to(input_dtype)
99 -
100 -
101 -class AquilaAttention(nn.Module):
102 -
103 - def __init__(
104 - self,
105 - hidden_size: int,
106 - num_heads: int,
107 - num_kv_heads: int,
108 - rope_theta: float = 10000,
109 - max_position_embeddings: int = 8192,
110 - rope_scaling: Optional[Dict[str, Any]] = None,
111 - linear_method: Optional[LinearMethodBase] = None,
112 - ):
113 - super().__init__()
114 - self.hidden_size = hidden_size
115 - tp_size = get_tensor_model_parallel_world_size()
116 - self.total_num_heads = num_heads
117 - assert self.total_num_heads % tp_size == 0
118 - self.num_heads = self.total_num_heads // tp_size
119 - self.total_num_kv_heads = num_kv_heads
120 - assert self.total_num_kv_heads % tp_size == 0
121 - self.num_kv_heads = self.total_num_kv_heads // tp_size
122 - self.head_dim = hidden_size // self.total_num_heads
123 - self.q_size = self.num_heads * self.head_dim
124 - self.kv_size = self.num_kv_heads * self.head_dim
125 - self.scaling = self.head_dim**-0.5
126 - self.rope_theta = rope_theta
127 - self.max_position_embeddings = max_position_embeddings
128 -
129 - self.qkv_proj = QKVParallelLinear(
130 - hidden_size,
131 - self.head_dim,
132 - self.total_num_heads,
133 - self.total_num_kv_heads,
134 - bias=False,
135 - linear_method=linear_method,
136 - )
137 - self.o_proj = RowParallelLinear(
138 - self.total_num_heads * self.head_dim,
139 - hidden_size,
140 - bias=False,
141 - linear_method=linear_method,
142 - )
143 - self.rotary_emb = get_rope(
144 - self.head_dim,
145 - rotary_dim=self.head_dim,
146 - max_position=self.max_position_embeddings,
147 - base=self.rope_theta,
148 - rope_scaling=rope_scaling,
149 - )
150 - self.attn = PagedAttention(self.num_heads,
151 - self.head_dim,
152 - self.scaling,
153 - num_kv_heads=self.num_kv_heads)
154 -
155 - def forward(
156 - self,
157 - positions: torch.Tensor,
158 - hidden_states: torch.Tensor,
159 - kv_cache: KVCache,
160 - input_metadata: InputMetadata,
161 - ) -> torch.Tensor:
162 - qkv, _ = self.qkv_proj(hidden_states)
163 - q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
164 - q, k = self.rotary_emb(positions, q, k)
165 - k_cache, v_cache = kv_cache
166 - attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
167 - output, _ = self.o_proj(attn_output)
168 - return output
169 -
170 -
171 -class AquilaDecoderLayer(nn.Module):
172 -
173 - def __init__(
174 - self,
175 - config: AquilaConfig,
176 - linear_method: Optional[LinearMethodBase] = None,
177 - ):
178 - super().__init__()
179 - self.hidden_size = config.hidden_size
180 - rope_theta = getattr(config, "rope_theta", 10000)
181 - rope_scaling = getattr(config, "rope_scaling", None)
182 - max_position_embeddings = getattr(config, "max_position_embeddings",
183 - 8192)
184 - self.self_attn = AquilaAttention(
185 - hidden_size=self.hidden_size,
186 - num_heads=config.num_attention_heads,
187 - num_kv_heads=config.num_key_value_heads,
188 - rope_theta=rope_theta,
189 - max_position_embeddings=max_position_embeddings,
190 - rope_scaling=rope_scaling,
191 - linear_method=linear_method,
192 - )
193 - self.mlp = AquilaMLP(
194 - hidden_size=self.hidden_size,
195 - intermediate_size=config.intermediate_size,
196 - hidden_act=config.hidden_act,
197 - linear_method=linear_method,
198 - )
199 - self.input_layernorm = AquilaRMSNorm(config.hidden_size,
200 - eps=config.rms_norm_eps)
201 - self.post_attention_layernorm = AquilaRMSNorm(config.hidden_size,
202 - eps=config.rms_norm_eps)
203 -
204 - def forward(
205 - self,
206 - positions: torch.Tensor,
207 - hidden_states: torch.Tensor,
208 - kv_cache: KVCache,
209 - input_metadata: InputMetadata,
210 - ) -> torch.Tensor:
211 - # Self Attention
212 - residual = hidden_states
213 - hidden_states = self.input_layernorm(hidden_states)
214 - hidden_states = self.self_attn(
215 - positions=positions,
216 - hidden_states=hidden_states,
217 - kv_cache=kv_cache,
218 - input_metadata=input_metadata,
219 - )
220 - hidden_states = residual + hidden_states
221 -
222 - # Fully Connected
223 - residual = hidden_states
224 - hidden_states = self.post_attention_layernorm(hidden_states)
225 - hidden_states = self.mlp(hidden_states)
226 - hidden_states = residual + hidden_states
227 - return hidden_states
228 -
229 -
230 -class AquilaModel(nn.Module):
231 -
232 - def __init__(
233 - self,
234 - config: AquilaConfig,
235 - linear_method: Optional[LinearMethodBase] = None,
236 - ):
237 - super().__init__()
238 - self.config = config
239 - self.padding_idx = config.pad_token_id
240 - self.vocab_size = config.vocab_size
241 - self.embed_tokens = VocabParallelEmbedding(
242 - config.vocab_size,
243 - config.hidden_size,
244 - )
245 - self.layers = nn.ModuleList([
246 - AquilaDecoderLayer(config, linear_method)
247 - for _ in range(config.num_hidden_layers)
248 - ])
249 - self.norm = AquilaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
250 -
251 - def forward(
252 - self,
253 - input_ids: torch.Tensor,
254 - positions: torch.Tensor,
255 - kv_caches: List[KVCache],
256 - input_metadata: InputMetadata,
257 - ) -> torch.Tensor:
258 - hidden_states = self.embed_tokens(input_ids)
259 - for i in range(len(self.layers)):
260 - layer = self.layers[i]
261 - hidden_states = layer(
262 - positions,
263 - hidden_states,
264 - kv_caches[i],
265 - input_metadata,
266 - )
267 - hidden_states = self.norm(hidden_states)
268 -
269 - return hidden_states
270 -
271 -
272 -class AquilaForCausalLM(nn.Module):
273 -
274 - def __init__(
275 - self,
276 - config,
277 - linear_method: Optional[LinearMethodBase] = None,
278 - ):
279 - super().__init__()
280 - self.config = config
281 - self.linear_method = linear_method
282 - self.model = AquilaModel(config, linear_method)
283 - self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
284 - self.sampler = Sampler(config.vocab_size)
285 -
286 - def forward(
287 - self,
288 - input_ids: torch.Tensor,
289 - positions: torch.Tensor,
290 - kv_caches: List[KVCache],
291 - input_metadata: InputMetadata,
292 - ) -> torch.Tensor:
293 - hidden_states = self.model(input_ids, positions, kv_caches,
294 - input_metadata)
295 - return hidden_states
296 -
297 - def sample(
298 - self,
299 - hidden_states: torch.Tensor,
300 - sampling_metadata: SamplingMetadata,
301 - ) -> Optional[SamplerOutput]:
302 - next_tokens = self.sampler(self.lm_head.weight, hidden_states,
303 - sampling_metadata)
304 - return next_tokens
305 -
306 - def load_weights(self,
307 - model_name_or_path: str,
308 - cache_dir: Optional[str] = None,
309 - load_format: str = "auto",
310 - revision: Optional[str] = None):
311 - stacked_params_mapping = [
312 - # (param_name, shard_name, shard_id)
313 - ("qkv_proj", "q_proj", "q"),
314 - ("qkv_proj", "k_proj", "k"),
315 - ("qkv_proj", "v_proj", "v"),
316 - ("gate_up_proj", "gate_proj", 0),
317 - ("gate_up_proj", "up_proj", 1),
318 - ]
319 - params_dict = dict(self.named_parameters())
320 - for name, loaded_weight in hf_model_weights_iterator(
321 - model_name_or_path, cache_dir, load_format, revision):
322 - if "rotary_emb.inv_freq" in name:
323 - continue
324 - for (param_name, weight_name, shard_id) in stacked_params_mapping:
325 - if weight_name not in name:
326 - continue
327 - name = name.replace(weight_name, param_name)
328 - # Skip loading extra bias for GPTQ models.
329 - if name.endswith(".bias") and name not in params_dict:
330 - continue
331 - param = params_dict[name]
332 - weight_loader = param.weight_loader
333 - weight_loader(param, loaded_weight, shard_id)
334 - break
335 - else:
336 - # Skip loading extra bias for GPTQ models.
337 - if name.endswith(".bias") and name not in params_dict:
338 - continue
339 - param = params_dict[name]
340 - weight_loader = getattr(param, "weight_loader",
341 - default_weight_loader)
342 - weight_loader(param, loaded_weight)
vllm/model_executor/models/baichuan.py
+32 -303
@@ -18,305 +18,19 @@
18 # See the License for the specific language governing permissions and
19 # limitations under the License.
20 """Inference-only BaiChuan model compatible with HuggingFace weights."""
21 -import math
22 -from typing import List, Optional, Tuple
21 +from typing import Optional
22
23 import torch
25 -from torch import nn
24 +from transformers import PretrainedConfig
25 +from vllm.config import LoRAConfig
26
27 -from vllm.model_executor.input_metadata import InputMetadata
28 -from vllm.model_executor.layers.activation import SiluAndMul
29 -from vllm.model_executor.layers.attention import PagedAttention
30 -from vllm.model_executor.layers.layernorm import RMSNorm
31 -from vllm.model_executor.layers.linear import (LinearMethodBase,
32 - MergedColumnParallelLinear,
33 - QKVParallelLinear,
34 - RowParallelLinear)
35 -from vllm.model_executor.layers.rotary_embedding import get_rope
36 -from vllm.model_executor.layers.sampler import Sampler
37 -from vllm.model_executor.layers.vocab_parallel_embedding import (
38 - VocabParallelEmbedding, ParallelLMHead)
39 -from vllm.model_executor.parallel_utils.parallel_state import (
40 - get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
41 -from vllm.model_executor.sampling_metadata import SamplingMetadata
27 +from vllm.model_executor.layers.linear import LinearMethodBase
28 +from vllm.model_executor.models.llama import LlamaForCausalLM
29 from vllm.model_executor.weight_utils import (default_weight_loader,
30 hf_model_weights_iterator)
44 -from vllm.sequence import SamplerOutput
45 -from vllm.transformers_utils.configs.baichuan import BaiChuanConfig
31
47 -KVCache = Tuple[torch.Tensor, torch.Tensor]
32
49 -
50 -def _get_alibi_slopes(total_num_heads: int) -> torch.Tensor:
51 - closest_power_of_2 = 2**math.floor(math.log2(total_num_heads))
52 - base = torch.tensor(
53 - 2**(-(2**-(math.log2(closest_power_of_2) - 3))),
54 - dtype=torch.float32,
55 - )
56 - powers = torch.arange(1, 1 + closest_power_of_2, dtype=torch.int32)
57 - slopes = torch.pow(base, powers)
58 -
59 - if closest_power_of_2 != total_num_heads:
60 - extra_base = torch.tensor(
61 - 2**(-(2**-(math.log2(2 * closest_power_of_2) - 3))),
62 - dtype=torch.float32,
63 - )
64 - num_remaining_heads = min(closest_power_of_2,
65 - total_num_heads - closest_power_of_2)
66 - extra_powers = torch.arange(start=1,
67 - end=1 + 2 * num_remaining_heads,
68 - step=2,
69 - dtype=torch.int32)
70 - slopes = torch.cat(
71 - [slopes, torch.pow(extra_base, extra_powers)], dim=0)
72 - return slopes
73 -
74 -
75 -class BaiChuanMLP(nn.Module):
76 -
77 - def __init__(
78 - self,
79 - hidden_size: int,
80 - intermediate_size: int,
81 - hidden_act: str,
82 - linear_method: Optional[LinearMethodBase] = None,
83 - ):
84 - super().__init__()
85 - self.gate_up_proj = MergedColumnParallelLinear(
86 - hidden_size, [intermediate_size] * 2,
87 - bias=False,
88 - linear_method=linear_method)
89 - self.down_proj = RowParallelLinear(intermediate_size,
90 - hidden_size,
91 - bias=False,
92 - linear_method=linear_method)
93 - if hidden_act != "silu":
94 - raise ValueError(f"Unsupported activation: {hidden_act}. "
95 - "Only silu is supported for now.")
96 - self.act_fn = SiluAndMul()
97 -
98 - def forward(self, x):
99 - gate_up, _ = self.gate_up_proj(x)
100 - x = self.act_fn(gate_up)
101 - x, _ = self.down_proj(x)
102 - return x
103 -
104 -
105 -class BaiChuanAttention(nn.Module):
106 - """Multi-headed attention from 'Attention Is All You Need' paper"""
107 -
108 - def __init__(
109 - self,
110 - hidden_size: int,
111 - num_heads: int,
112 - position_embedding: str,
113 - rope_theta: float = 10000,
114 - max_position_embeddings: int = 8192,
115 - linear_method: Optional[LinearMethodBase] = None,
116 - ):
117 - super().__init__()
118 - self.hidden_size = hidden_size
119 - tensor_model_parallel_world_size = get_tensor_model_parallel_world_size(
120 - )
121 - self.total_num_heads = num_heads
122 - assert self.total_num_heads % tensor_model_parallel_world_size == 0
123 - self.num_heads = (self.total_num_heads //
124 - tensor_model_parallel_world_size)
125 - self.head_dim = hidden_size // self.total_num_heads
126 - self.postion_embedding = position_embedding
127 - self.rope_theta = rope_theta
128 - self.max_position_embeddings = max_position_embeddings
129 -
130 - # pylint: disable=invalid-name
131 - self.W_pack = QKVParallelLinear(
132 - hidden_size,
133 - self.head_dim,
134 - self.total_num_heads,
135 - self.total_num_heads,
136 - bias=False,
137 - linear_method=linear_method,
138 - )
139 - self.o_proj = RowParallelLinear(
140 - self.total_num_heads * self.head_dim,
141 - hidden_size,
142 - bias=False,
143 - linear_method=linear_method,
144 - )
145 - # Create the alibi slopes and slice them.
146 - if self.postion_embedding == "ALIBI":
147 - tp_rank = get_tensor_model_parallel_rank()
148 - head_start = tp_rank * self.num_heads
149 - head_end = (tp_rank + 1) * self.num_heads
150 - alibi_slopes = _get_alibi_slopes(self.total_num_heads)
151 - alibi_slopes = alibi_slopes[head_start:head_end].tolist()
152 -
153 - scaling = self.head_dim**-0.5
154 - self.attn = PagedAttention(self.num_heads,
155 - self.head_dim,
156 - scaling,
157 - alibi_slopes=alibi_slopes)
158 - else:
159 - self.rotary_emb = get_rope(
160 - self.head_dim,
161 - rotary_dim=self.head_dim,
162 - max_position=self.max_position_embeddings,
163 - base=self.rope_theta,
164 - )
165 - self.scaling = self.head_dim**-0.5
166 - self.attn = PagedAttention(self.num_heads, self.head_dim,
167 - self.scaling)
168 -
169 - def forward(
170 - self,
171 - positions: torch.Tensor,
172 - hidden_states: torch.Tensor,
173 - kv_cache: KVCache,
174 - input_metadata: InputMetadata,
175 - ) -> torch.Tensor:
176 - qkv, _ = self.W_pack(hidden_states)
177 - q, k, v = qkv.chunk(chunks=3, dim=-1)
178 - if self.postion_embedding != "ALIBI":
179 - q, k = self.rotary_emb(positions, q, k)
180 - k_cache, v_cache = kv_cache
181 - attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
182 - output, _ = self.o_proj(attn_output)
183 - return output
184 -
185 -
186 -class BaiChuanDecoderLayer(nn.Module):
187 -
188 - def __init__(self,
189 - config: BaiChuanConfig,
190 - position_embedding: str,
191 - linear_method: Optional[LinearMethodBase] = None):
192 - super().__init__()
193 - self.hidden_size = config.hidden_size
194 - rope_theta = getattr(config, "rope_theta", 10000)
195 - max_position_embeddings = getattr(config, "max_position_embeddings",
196 - 8192)
197 - self.self_attn = BaiChuanAttention(
198 - hidden_size=self.hidden_size,
199 - num_heads=config.num_attention_heads,
200 - position_embedding=position_embedding,
201 - rope_theta=rope_theta,
202 - max_position_embeddings=max_position_embeddings,
203 - linear_method=linear_method,
204 - )
205 - self.mlp = BaiChuanMLP(
206 - hidden_size=self.hidden_size,
207 - intermediate_size=config.intermediate_size,
208 - hidden_act=config.hidden_act,
209 - linear_method=linear_method,
210 - )
211 - self.input_layernorm = RMSNorm(config.hidden_size,
212 - eps=config.rms_norm_eps)
213 - self.post_attention_layernorm = RMSNorm(config.hidden_size,
214 - eps=config.rms_norm_eps)
215 -
216 - def forward(
217 - self,
218 - positions: torch.Tensor,
219 - hidden_states: torch.Tensor,
220 - kv_cache: KVCache,
221 - input_metadata: InputMetadata,
222 - residual: Optional[torch.Tensor],
223 - ) -> Tuple[torch.Tensor, torch.Tensor]:
224 - # Self Attention
225 - if residual is None:
226 - residual = hidden_states
227 - hidden_states = self.input_layernorm(hidden_states)
228 - else:
229 - hidden_states, residual = self.input_layernorm(
230 - hidden_states, residual)
231 - hidden_states = self.self_attn(
232 - positions=positions,
233 - hidden_states=hidden_states,
234 - kv_cache=kv_cache,
235 - input_metadata=input_metadata,
236 - )
237 -
238 - # Fully Connected
239 - hidden_states, residual = self.post_attention_layernorm(
240 - hidden_states, residual)
241 - hidden_states = self.mlp(hidden_states)
242 - return hidden_states, residual
243 -
244 -
245 -class BaiChuanModel(nn.Module):
246 -
247 - def __init__(self,
248 - config: BaiChuanConfig,
249 - position_embedding: str,
250 - linear_method: Optional[LinearMethodBase] = None):
251 - super().__init__()
252 - self.config = config
253 - self.padding_idx = config.pad_token_id
254 - self.vocab_size = config.vocab_size
255 -
256 - self.embed_tokens = VocabParallelEmbedding(
257 - config.vocab_size,
258 - config.hidden_size,
259 - )
260 - self.layers = nn.ModuleList([
261 - BaiChuanDecoderLayer(config, position_embedding, linear_method)
262 - for _ in range(config.num_hidden_layers)
263 - ])
264 - self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
265 -
266 - def forward(
267 - self,
268 - input_ids: torch.Tensor,
269 - positions: torch.Tensor,
270 - kv_caches: List[KVCache],
271 - input_metadata: InputMetadata,
272 - ) -> torch.Tensor:
273 - hidden_states = self.embed_tokens(input_ids)
274 - residual = None
275 - for i in range(len(self.layers)):
276 - layer = self.layers[i]
277 - hidden_states, residual = layer(
278 - positions,
279 - hidden_states,
280 - kv_caches[i],
281 - input_metadata,
282 - residual,
283 - )
284 - hidden_states, _ = self.norm(hidden_states, residual)
285 - return hidden_states
286 -
287 -
288 -class BaiChuanBaseForCausalLM(nn.Module):
289 -
290 - def __init__(self,
291 - config,
292 - position_embedding: str,
293 - linear_method: Optional[LinearMethodBase] = None):
294 - super().__init__()
295 - self.config = config
296 - self.linear_method = linear_method
297 - self.model = BaiChuanModel(config, position_embedding, linear_method)
298 - self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
299 - self.sampler = Sampler(config.vocab_size)
300 -
301 - def forward(
302 - self,
303 - input_ids: torch.Tensor,
304 - positions: torch.Tensor,
305 - kv_caches: List[KVCache],
306 - input_metadata: InputMetadata,
307 - ) -> torch.Tensor:
308 - hidden_states = self.model(input_ids, positions, kv_caches,
309 - input_metadata)
310 - return hidden_states
311 -
312 - def sample(
313 - self,
314 - hidden_states: torch.Tensor,
315 - sampling_metadata: SamplingMetadata,
316 - ) -> Optional[SamplerOutput]:
317 - next_tokens = self.sampler(self.lm_head.weight, hidden_states,
318 - sampling_metadata)
319 - return next_tokens
33 +class BaiChuanBaseForCausalLM(LlamaForCausalLM):
34
35 def load_weights(self,
36 model_name_or_path: str,
@@ -328,9 +42,15 @@ class BaiChuanBaseForCausalLM(nn.Module):
42 ("gate_up_proj", "gate_proj", 0),
43 ("gate_up_proj", "up_proj", 1),
44 ]
45 + param_weight_map = [
46 + ("qkv_proj", "W_pack"),
47 + ]
48 params_dict = dict(self.named_parameters())
49 for name, loaded_weight in hf_model_weights_iterator(
50 model_name_or_path, cache_dir, load_format, revision):
51 + for (param_name, weight_name) in param_weight_map:
52 + name = name.replace(weight_name, param_name)
53 +
54 if "rotary_emb.inv_freq" in name:
55 continue
56 if name == "lm_head.weight":
@@ -368,19 +88,28 @@ class BaiChuanBaseForCausalLM(nn.Module):
88 class BaichuanForCausalLM(BaiChuanBaseForCausalLM):
89 """Baichuan 13B and Baichuan2 7B/13B."""
90
371 - def __init__(self,
372 - config,
373 - linear_method: Optional[LinearMethodBase] = None):
374 - if config.hidden_size == 4096: # baichuan2 7b
375 - super().__init__(config, "ROPE", linear_method)
376 - else: # baichuan 13b, baichuan2 13b
377 - super().__init__(config, "ALIBI", linear_method)
91 + def __init__(
92 + self,
93 + config: Optional[PretrainedConfig] = None,
94 + linear_method: Optional[LinearMethodBase] = None,
95 + lora_config: Optional[LoRAConfig] = None,
96 + ) -> None:
97 + if config.hidden_size != 4096: # baichuan 13b, baichuan2 13b
98 + config.postion_embedding = "ALIBI"
99 + super().__init__(config=config,
100 + linear_method=linear_method,
101 + lora_config=lora_config)
102
103
104 class BaiChuanForCausalLM(BaiChuanBaseForCausalLM):
105 """Baichuan 7B."""
106
383 - def __init__(self,
384 - config,
385 - linear_method: Optional[LinearMethodBase] = None):
386 - super().__init__(config, "ROPE", linear_method)
107 + def __init__(
108 + self,
109 + config: Optional[PretrainedConfig] = None,
110 + linear_method: Optional[LinearMethodBase] = None,
111 + lora_config: Optional[LoRAConfig] = None,
112 + ) -> None:
113 + super().__init__(config=config,
114 + linear_method=linear_method,
115 + lora_config=lora_config)
vllm/model_executor/models/internlm.py deleted
-299
@@ -1,299 +0,0 @@
1 -# -*- coding: utf-8 -*-
2 -from typing import Any, Dict, List, Optional, Tuple
3 -
4 -import torch
5 -from torch import nn
6 -from transformers import LlamaConfig
7 -
8 -from vllm.model_executor.input_metadata import InputMetadata
9 -from vllm.model_executor.layers.activation import SiluAndMul
10 -from vllm.model_executor.layers.attention import PagedAttention
11 -from vllm.model_executor.layers.layernorm import RMSNorm
12 -from vllm.model_executor.layers.linear import (LinearMethodBase,
13 - MergedColumnParallelLinear,
14 - QKVParallelLinear,
15 - RowParallelLinear)
16 -from vllm.model_executor.layers.rotary_embedding import get_rope
17 -from vllm.model_executor.layers.sampler import Sampler
18 -from vllm.model_executor.layers.vocab_parallel_embedding import (
19 - VocabParallelEmbedding, ParallelLMHead)
20 -from vllm.model_executor.parallel_utils.parallel_state import (
21 - get_tensor_model_parallel_world_size)
22 -from vllm.model_executor.sampling_metadata import SamplingMetadata
23 -from vllm.model_executor.weight_utils import (default_weight_loader,
24 - hf_model_weights_iterator)
25 -from vllm.sequence import SamplerOutput
26 -
27 -KVCache = Tuple[torch.Tensor, torch.Tensor]
28 -
29 -
30 -class InternLMMLP(nn.Module):
31 -
32 - def __init__(
33 - self,
34 - hidden_size: int,
35 - intermediate_size: int,
36 - hidden_act: str,
37 - linear_method: Optional[LinearMethodBase] = None,
38 - ):
39 - super().__init__()
40 - self.gate_up_proj = MergedColumnParallelLinear(
41 - hidden_size, [intermediate_size] * 2,
42 - bias=False,
43 - linear_method=linear_method)
44 - self.down_proj = RowParallelLinear(intermediate_size,
45 - hidden_size,
46 - bias=False,
47 - linear_method=linear_method)
48 - if hidden_act != "silu":
49 - raise ValueError(f"Unsupported activation: {hidden_act}. "
50 - "Only silu is supported for now.")
51 - self.act_fn = SiluAndMul()
52 -
53 - def forward(self, x):
54 - gate_up, _ = self.gate_up_proj(x)
55 - x = self.act_fn(gate_up)
56 - x, _ = self.down_proj(x)
57 - return x
58 -
59 -
60 -class InternLMAttention(nn.Module):
61 -
62 - def __init__(
63 - self,
64 - hidden_size: int,
65 - num_heads: int,
66 - bias: bool,
67 - rope_theta: float = 10000,
68 - max_position_embeddings: int = 8192,
69 - linear_method: Optional[LinearMethodBase] = None,
70 - rope_scaling: Optional[Dict[str, Any]] = None,
71 - ):
72 - super().__init__()
73 - self.hidden_size = hidden_size
74 - tensor_model_parallel_world_size = (
75 - get_tensor_model_parallel_world_size())
76 - self.total_num_heads = num_heads
77 - assert self.total_num_heads % tensor_model_parallel_world_size == 0
78 - self.num_heads = (self.total_num_heads //
79 - tensor_model_parallel_world_size)
80 - self.head_dim = hidden_size // self.total_num_heads
81 - self.scaling = self.head_dim**-0.5
82 - self.rope_theta = rope_theta
83 - self.max_position_embeddings = max_position_embeddings
84 -
85 - self.qkv_proj = QKVParallelLinear(
86 - hidden_size,
87 - self.head_dim,
88 - self.total_num_heads,
89 - bias=bias,
90 - linear_method=linear_method,
91 - )
92 - self.o_proj = RowParallelLinear(
93 - self.total_num_heads * self.head_dim,
94 - hidden_size,
95 - bias=bias,
96 - linear_method=linear_method,
97 - )
98 - self.rotary_emb = get_rope(
99 - self.head_dim,
100 - rotary_dim=self.head_dim,
101 - max_position=self.max_position_embeddings,
102 - base=self.rope_theta,
103 - rope_scaling=rope_scaling,
104 - )
105 - self.attn = PagedAttention(self.num_heads, self.head_dim, self.scaling)
106 -
107 - def forward(
108 - self,
109 - positions: torch.Tensor,
110 - hidden_states: torch.Tensor,
111 - kv_cache: KVCache,
112 - input_metadata: InputMetadata,
113 - ) -> torch.Tensor:
114 - qkv, _ = self.qkv_proj(hidden_states)
115 - q, k, v = qkv.chunk(chunks=3, dim=-1)
116 - q, k = self.rotary_emb(positions, q, k)
117 - k_cache, v_cache = kv_cache
118 - attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
119 - output, _ = self.o_proj(attn_output)
120 - return output
121 -
122 -
123 -class InternLMDecoderLayer(nn.Module):
124 -
125 - def __init__(
126 - self,
127 - config: LlamaConfig,
128 - linear_method: Optional[LinearMethodBase] = None,
129 - ):
130 - super().__init__()
131 - self.hidden_size = config.hidden_size
132 - rope_theta = getattr(config, "rope_theta", 10000)
133 - max_position_embeddings = getattr(config, "max_position_embeddings",
134 - 8192)
135 - self.self_attn = InternLMAttention(
136 - hidden_size=self.hidden_size,
137 - num_heads=config.num_attention_heads,
138 - bias=config.bias,
139 - rope_theta=rope_theta,
140 - max_position_embeddings=max_position_embeddings,
141 - linear_method=linear_method,
142 - rope_scaling=getattr(config, "rope_scaling", None),
143 - )
144 - self.mlp = InternLMMLP(
145 - hidden_size=self.hidden_size,
146 - intermediate_size=config.intermediate_size,
147 - hidden_act=config.hidden_act,
148 - linear_method=linear_method,
149 - )
150 - self.input_layernorm = RMSNorm(config.hidden_size,
151 - eps=config.rms_norm_eps)
152 - self.post_attention_layernorm = RMSNorm(config.hidden_size,
153 - eps=config.rms_norm_eps)
154 -
155 - def forward(
156 - self,
157 - positions: torch.Tensor,
158 - hidden_states: torch.Tensor,
159 - kv_cache: KVCache,
160 - input_metadata: InputMetadata,
161 - residual: Optional[torch.Tensor],
162 - ) -> Tuple[torch.Tensor, torch.Tensor]:
163 - # Self Attention
164 - if residual is None:
165 - residual = hidden_states
166 - hidden_states = self.input_layernorm(hidden_states)
167 - else:
168 - hidden_states, residual = self.input_layernorm(
169 - hidden_states, residual)
170 - hidden_states = self.self_attn(
171 - positions=positions,
172 - hidden_states=hidden_states,
173 - kv_cache=kv_cache,
174 - input_metadata=input_metadata,
175 - )
176 -
177 - # Fully Connected
178 - hidden_states, residual = self.post_attention_layernorm(
179 - hidden_states, residual)
180 - hidden_states = self.mlp(hidden_states)
181 - return hidden_states, residual
182 -
183 -
184 -class InternLMModel(nn.Module):
185 -
186 - def __init__(
187 - self,
188 - config: LlamaConfig,
189 - linear_method: Optional[LinearMethodBase] = None,
190 - ):
191 - super().__init__()
192 - self.config = config
193 - self.padding_idx = config.pad_token_id
194 - self.vocab_size = config.vocab_size
195 -
196 - vocab_size = ((config.vocab_size + 63) // 64) * 64
197 - self.embed_tokens = VocabParallelEmbedding(
198 - vocab_size,
199 - config.hidden_size,
200 - )
201 - self.layers = nn.ModuleList([
202 - InternLMDecoderLayer(config, linear_method)
203 - for _ in range(config.num_hidden_layers)
204 - ])
205 - self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
206 -
207 - def forward(
208 - self,
209 - input_ids: torch.Tensor,
210 - positions: torch.Tensor,
211 - kv_caches: List[KVCache],
212 - input_metadata: InputMetadata,
213 - ) -> torch.Tensor:
214 - hidden_states = self.embed_tokens(input_ids)
215 - residual = None
216 - for i in range(len(self.layers)):
217 - layer = self.layers[i]
218 - hidden_states, residual = layer(
219 - positions,
220 - hidden_states,
221 - kv_caches[i],
222 - input_metadata,
223 - residual,
224 - )
225 - hidden_states, _ = self.norm(hidden_states, residual)
226 - return hidden_states
227 -
228 -
229 -class InternLMForCausalLM(nn.Module):
230 -
231 - def __init__(
232 - self,
233 - config,
234 - linear_method: Optional[LinearMethodBase] = None,
235 - ):
236 - super().__init__()
237 - self.config = config
238 - self.linear_method = linear_method
239 - self.model = InternLMModel(config, linear_method)
240 - self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
241 - self.sampler = Sampler(config.vocab_size)
242 -
243 - def forward(
244 - self,
245 - input_ids: torch.Tensor,
246 - positions: torch.Tensor,
247 - kv_caches: List[KVCache],
248 - input_metadata: InputMetadata,
249 - ) -> torch.Tensor:
250 - hidden_states = self.model(input_ids, positions, kv_caches,
251 - input_metadata)
252 - return hidden_states
253 -
254 - def sample(
255 - self,
256 - hidden_states: torch.Tensor,
257 - sampling_metadata: SamplingMetadata,
258 - ) -> Optional[SamplerOutput]:
259 - next_tokens = self.sampler(self.lm_head.weight, hidden_states,
260 - sampling_metadata)
261 - return next_tokens
262 -
263 - def load_weights(self,
264 - model_name_or_path: str,
265 - cache_dir: Optional[str] = None,
266 - load_format: str = "auto",
267 - revision: Optional[str] = None):
268 - stacked_params_mapping = [
269 - # (param_name, shard_name, shard_id)
270 - ("qkv_proj", "q_proj", "q"),
271 - ("qkv_proj", "k_proj", "k"),
272 - ("qkv_proj", "v_proj", "v"),
273 - ("gate_up_proj", "gate_proj", 0),
274 - ("gate_up_proj", "up_proj", 1),
275 - ]
276 - params_dict = dict(self.named_parameters())
277 - for name, loaded_weight in hf_model_weights_iterator(
278 - model_name_or_path, cache_dir, load_format, revision):
279 - if "rotary_emb.inv_freq" in name:
280 - continue
281 - for (param_name, weight_name, shard_id) in stacked_params_mapping:
282 - if weight_name not in name:
283 - continue
284 - name = name.replace(weight_name, param_name)
285 - # Skip loading extra bias for GPTQ models.
286 - if name.endswith(".bias") and name not in params_dict:
287 - continue
288 - param = params_dict[name]
289 - weight_loader = param.weight_loader
290 - weight_loader(param, loaded_weight, shard_id)
291 - break
292 - else:
293 - # Skip loading extra bias for GPTQ models.
294 - if name.endswith(".bias") and name not in params_dict:
295 - continue
296 - param = params_dict[name]
297 - weight_loader = getattr(param, "weight_loader",
298 - default_weight_loader)
299 - weight_loader(param, loaded_weight)
vllm/model_executor/models/internlm2.py
+25 -260
@@ -1,276 +1,27 @@
1 # -*- coding: utf-8 -*-
2 -from typing import Any, Dict, List, Optional, Tuple
2 +from typing import Optional
3
4 import torch
5 -from torch import nn
5 from transformers import PretrainedConfig
6 +from vllm.config import LoRAConfig
7
8 -from vllm.model_executor.input_metadata import InputMetadata
9 -from vllm.model_executor.layers.activation import SiluAndMul
10 -from vllm.model_executor.layers.attention import PagedAttention
11 -from vllm.model_executor.layers.layernorm import RMSNorm
12 -from vllm.model_executor.layers.linear import (LinearMethodBase,
13 - MergedColumnParallelLinear,
14 - QKVParallelLinear,
15 - RowParallelLinear)
16 -from vllm.model_executor.layers.rotary_embedding import get_rope
17 -from vllm.model_executor.layers.sampler import Sampler
18 -from vllm.model_executor.layers.vocab_parallel_embedding import (
19 - VocabParallelEmbedding, ParallelLMHead)
20 -from vllm.model_executor.parallel_utils.parallel_state import (
21 - get_tensor_model_parallel_world_size)
22 -from vllm.model_executor.sampling_metadata import SamplingMetadata
8 +from vllm.model_executor.layers.linear import LinearMethodBase
9 +from vllm.model_executor.models.llama import LlamaForCausalLM
10 from vllm.model_executor.weight_utils import (default_weight_loader,
11 hf_model_weights_iterator)
25 -from vllm.sequence import SamplerOutput
12
27 -KVCache = Tuple[torch.Tensor, torch.Tensor]
13
29 -
30 -class InternLM2MLP(nn.Module):
31 -
32 - def __init__(
33 - self,
34 - hidden_size: int,
35 - intermediate_size: int,
36 - hidden_act: str,
37 - linear_method: Optional[LinearMethodBase] = None,
38 - ) -> None:
39 - super().__init__()
40 - self.gate_up_proj = MergedColumnParallelLinear(
41 - hidden_size, [intermediate_size] * 2,
42 - bias=False,
43 - linear_method=linear_method)
44 - self.w2 = RowParallelLinear(intermediate_size,
45 - hidden_size,
46 - bias=False,
47 - linear_method=linear_method)
48 - if hidden_act != "silu":
49 - raise ValueError(f"Unsupported activation: {hidden_act}. "
50 - "Only silu is supported for now.")
51 - self.act_fn = SiluAndMul()
52 -
53 - def forward(self, x):
54 - gate_up, _ = self.gate_up_proj(x)
55 - x = self.act_fn(gate_up)
56 - x, _ = self.w2(x)
57 - return x
58 -
59 -
60 -class InternLM2Attention(nn.Module):
61 -
62 - def __init__(
63 - self,
64 - hidden_size: int,
65 - num_heads: int,
66 - num_kv_heads: int,
67 - rope_theta: float = 10000,
68 - rope_scaling: Optional[Dict[str, Any]] = None,
69 - max_position_embeddings: int = 8192,
70 - linear_method: Optional[LinearMethodBase] = None,
71 - ) -> None:
72 - super().__init__()
73 - self.hidden_size = hidden_size
74 - tp_size = get_tensor_model_parallel_world_size()
75 - self.total_num_heads = num_heads
76 - assert self.total_num_heads % tp_size == 0
77 - self.num_heads = self.total_num_heads // tp_size
78 - self.total_num_kv_heads = num_kv_heads
79 - if self.total_num_kv_heads >= tp_size:
80 - # Number of KV heads is greater than TP size, so we partition
81 - # the KV heads across multiple tensor parallel GPUs.
82 - assert self.total_num_kv_heads % tp_size == 0
83 - else:
84 - # Number of KV heads is less than TP size, so we replicate
85 - # the KV heads across multiple tensor parallel GPUs.
86 - assert tp_size % self.total_num_kv_heads == 0
87 - self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
88 - self.head_dim = hidden_size // self.total_num_heads
89 - self.q_size = self.num_heads * self.head_dim
90 - self.kv_size = self.num_kv_heads * self.head_dim
91 - self.scaling = self.head_dim**-0.5
92 - self.rope_theta = rope_theta
93 - self.max_position_embeddings = max_position_embeddings
94 -
95 - self.wqkv = QKVParallelLinear(
96 - hidden_size,
97 - self.head_dim,
98 - self.total_num_heads,
99 - self.total_num_kv_heads,
100 - bias=False,
101 - linear_method=linear_method,
102 - )
103 - self.wo = RowParallelLinear(
104 - self.total_num_heads * self.head_dim,
105 - hidden_size,
106 - bias=False,
107 - linear_method=linear_method,
108 - )
109 -
110 - self.rotary_emb = get_rope(
111 - self.head_dim,
112 - rotary_dim=self.head_dim,
113 - max_position=max_position_embeddings,
114 - base=rope_theta,
115 - rope_scaling=rope_scaling,
116 - )
117 - self.attn = PagedAttention(self.num_heads,
118 - self.head_dim,
119 - self.scaling,
120 - num_kv_heads=self.num_kv_heads)
121 -
122 - def forward(
123 - self,
124 - positions: torch.Tensor,
125 - hidden_states: torch.Tensor,
126 - kv_cache: KVCache,
127 - input_metadata: InputMetadata,
128 - ) -> torch.Tensor:
129 - qkv, _ = self.wqkv(hidden_states)
130 - q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
131 - q, k = self.rotary_emb(positions, q, k)
132 - k_cache, v_cache = kv_cache
133 - attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
134 - output, _ = self.wo(attn_output)
135 - return output
136 -
137 -
138 -class InternLMDecoderLayer(nn.Module):
139 -
140 - def __init__(
141 - self,
142 - config: PretrainedConfig,
143 - linear_method: Optional[LinearMethodBase] = None,
144 - ) -> None:
145 - super().__init__()
146 - self.hidden_size = config.hidden_size
147 - rope_theta = getattr(config, "rope_theta", 10000)
148 - rope_scaling = getattr(config, "rope_scaling", None)
149 - max_position_embeddings = getattr(config, "max_position_embeddings",
150 - 8192)
151 - self.attention = InternLM2Attention(
152 - hidden_size=self.hidden_size,
153 - num_heads=config.num_attention_heads,
154 - num_kv_heads=config.num_key_value_heads,
155 - rope_theta=rope_theta,
156 - rope_scaling=rope_scaling,
157 - max_position_embeddings=max_position_embeddings,
158 - linear_method=linear_method,
159 - )
160 - self.feed_forward = InternLM2MLP(
161 - hidden_size=self.hidden_size,
162 - intermediate_size=config.intermediate_size,
163 - hidden_act=config.hidden_act,
164 - linear_method=linear_method,
165 - )
166 - self.attention_norm = RMSNorm(config.hidden_size,
167 - eps=config.rms_norm_eps)
168 - self.ffn_norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
169 -
170 - def forward(
171 - self,
172 - positions: torch.Tensor,
173 - hidden_states: torch.Tensor,
174 - kv_cache: KVCache,
175 - input_metadata: InputMetadata,
176 - residual: Optional[torch.Tensor],
177 - ) -> Tuple[torch.Tensor, torch.Tensor]:
178 - # Self Attention
179 - if residual is None:
180 - residual = hidden_states
181 - hidden_states = self.attention_norm(hidden_states)
182 - else:
183 - hidden_states, residual = self.attention_norm(
184 - hidden_states, residual)
185 - hidden_states = self.attention(
186 - positions=positions,
187 - hidden_states=hidden_states,
188 - kv_cache=kv_cache,
189 - input_metadata=input_metadata,
190 - )
191 -
192 - # Fully Connected
193 - hidden_states, residual = self.ffn_norm(hidden_states, residual)
194 - hidden_states = self.feed_forward(hidden_states)
195 - return hidden_states, residual
196 -
197 -
198 -class InternLM2Model(nn.Module):
14 +class InternLM2ForCausalLM(LlamaForCausalLM):
15
16 def __init__(
17 self,
202 - config: PretrainedConfig,
18 + config: Optional[PretrainedConfig] = None,
19 linear_method: Optional[LinearMethodBase] = None,
20 + lora_config: Optional[LoRAConfig] = None,
21 ) -> None:
205 - super().__init__()
206 - self.config = config
207 - self.padding_idx = config.pad_token_id
208 - self.vocab_size = config.vocab_size
209 - self.tok_embeddings = VocabParallelEmbedding(
210 - config.vocab_size,
211 - config.hidden_size,
212 - )
213 - self.layers = nn.ModuleList([
214 - InternLMDecoderLayer(config, linear_method)
215 - for _ in range(config.num_hidden_layers)
216 - ])
217 - self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
218 -
219 - def forward(
220 - self,
221 - input_ids: torch.Tensor,
222 - positions: torch.Tensor,
223 - kv_caches: List[KVCache],
224 - input_metadata: InputMetadata,
225 - ) -> torch.Tensor:
226 - hidden_states = self.tok_embeddings(input_ids)
227 - residual = None
228 - for i in range(len(self.layers)):
229 - layer = self.layers[i]
230 - hidden_states, residual = layer(
231 - positions,
232 - hidden_states,
233 - kv_caches[i],
234 - input_metadata,
235 - residual,
236 - )
237 - hidden_states, _ = self.norm(hidden_states, residual)
238 - return hidden_states
239 -
240 -
241 -class InternLM2ForCausalLM(nn.Module):
242 -
243 - def __init__(
244 - self,
245 - config: PretrainedConfig,
246 - linear_method: Optional[LinearMethodBase] = None,
247 - ) -> None:
248 - super().__init__()
249 - self.config = config
250 - self.linear_method = linear_method
251 - self.model = InternLM2Model(config, linear_method)
252 - self.output = ParallelLMHead(config.vocab_size, config.hidden_size)
253 - self.sampler = Sampler(config.vocab_size)
254 -
255 - def forward(
256 - self,
257 - input_ids: torch.Tensor,
258 - positions: torch.Tensor,
259 - kv_caches: List[KVCache],
260 - input_metadata: InputMetadata,
261 - ) -> torch.Tensor:
262 - hidden_states = self.model(input_ids, positions, kv_caches,
263 - input_metadata)
264 - return hidden_states
265 -
266 - def sample(
267 - self,
268 - hidden_states: torch.Tensor,
269 - sampling_metadata: SamplingMetadata,
270 - ) -> Optional[SamplerOutput]:
271 - next_tokens = self.sampler(self.output.weight, hidden_states,
272 - sampling_metadata)
273 - return next_tokens
22 + super().__init__(config=config,
23 + linear_method=linear_method,
24 + lora_config=lora_config)
25
26 def load_weights(self,
27 model_name_or_path: str,
@@ -282,9 +33,23 @@ class InternLM2ForCausalLM(nn.Module):
33 ("gate_up_proj", "w1", 0),
34 ("gate_up_proj", "w3", 1),
35 ]
36 + param_weight_map = [
37 + ("qkv_proj", "wqkv"),
38 + ("o_proj", "wo"),
39 + ("down_proj", "w2"),
40 + ("input_layernorm", "attention_norm"),
41 + ("post_attention_layernorm", "ffn_norm"),
42 + ("embed_tokens", "tok_embeddings"),
43 + (".self_attn.", ".attention."),
44 + ("mlp", "feed_forward"),
45 + ("lm_head", "output"),
46 + ]
47 params_dict = dict(self.named_parameters())
48 for name, loaded_weight in hf_model_weights_iterator(
49 model_name_or_path, cache_dir, load_format, revision):
50 + for (param_name, weight_name) in param_weight_map:
51 + name = name.replace(weight_name, param_name)
52 +
53 if "rotary_emb.inv_freq" in name:
54 continue
55 for (param_name, weight_name, shard_id) in stacked_params_mapping:
@@ -303,7 +68,7 @@ class InternLM2ForCausalLM(nn.Module):
68 if name.endswith(".bias") and name not in params_dict:
69 continue
70 param = params_dict[name]
306 - if "wqkv" in name:
71 + if "qkv_proj" in name:
72 config = self.config
73 kv_groups = config.num_attention_heads // config.num_key_value_heads
74 head_dim = config.hidden_size // config.num_attention_heads
vllm/model_executor/models/llama.py
+106 -56
@@ -21,8 +21,9 @@
21 # See the License for the specific language governing permissions and
22 # limitations under the License.
23 """Inference-only LLaMA model compatible with HuggingFace weights."""
24 -from typing import Any, Dict, List, Optional, Tuple
24 +from typing import List, Optional, Tuple
25
26 +import math
27 import torch
28 from torch import nn
29 from transformers import LlamaConfig
@@ -40,34 +41,60 @@ from vllm.model_executor.layers.sampler import Sampler
41 from vllm.model_executor.layers.vocab_parallel_embedding import (
42 VocabParallelEmbedding, ParallelLMHead, DEFAULT_VOCAB_PADDING_SIZE)
43 from vllm.model_executor.parallel_utils.parallel_state import (
43 - get_tensor_model_parallel_world_size)
44 + get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
45 from vllm.model_executor.sampling_metadata import SamplingMetadata
46 from vllm.model_executor.weight_utils import (default_weight_loader,
47 hf_model_weights_iterator)
48 from vllm.sequence import SamplerOutput
49 from vllm.config import LoRAConfig
50
51 +from copy import deepcopy
52 +
53 KVCache = Tuple[torch.Tensor, torch.Tensor]
54
55
56 +def _get_alibi_slopes(total_num_heads: int) -> torch.Tensor:
57 + closest_power_of_2 = 2**math.floor(math.log2(total_num_heads))
58 + base = torch.tensor(
59 + 2**(-(2**-(math.log2(closest_power_of_2) - 3))),
60 + dtype=torch.float32,
61 + )
62 + powers = torch.arange(1, 1 + closest_power_of_2, dtype=torch.int32)
63 + slopes = torch.pow(base, powers)
64 +
65 + if closest_power_of_2 != total_num_heads:
66 + extra_base = torch.tensor(
67 + 2**(-(2**-(math.log2(2 * closest_power_of_2) - 3))),
68 + dtype=torch.float32,
69 + )
70 + num_remaining_heads = min(closest_power_of_2,
71 + total_num_heads - closest_power_of_2)
72 + extra_powers = torch.arange(start=1,
73 + end=1 + 2 * num_remaining_heads,
74 + step=2,
75 + dtype=torch.int32)
76 + slopes = torch.cat(
77 + [slopes, torch.pow(extra_base, extra_powers)], dim=0)
78 + return slopes
79 +
80 +
81 class LlamaMLP(nn.Module):
82
83 def __init__(
84 self,
57 - hidden_size: int,
58 - intermediate_size: int,
59 - hidden_act: str,
85 + config: LlamaConfig,
86 linear_method: Optional[LinearMethodBase] = None,
87 ) -> None:
88 super().__init__()
89 self.gate_up_proj = MergedColumnParallelLinear(
64 - hidden_size, [intermediate_size] * 2,
90 + config.hidden_size, [config.intermediate_size] * 2,
91 bias=False,
92 linear_method=linear_method)
67 - self.down_proj = RowParallelLinear(intermediate_size,
68 - hidden_size,
93 + self.down_proj = RowParallelLinear(config.intermediate_size,
94 + config.hidden_size,
95 bias=False,
96 linear_method=linear_method)
97 + hidden_act = getattr(config, "hidden_act", "silu")
98 if hidden_act != "silu":
99 raise ValueError(f"Unsupported activation: {hidden_act}. "
100 "Only silu is supported for now.")
@@ -84,21 +111,19 @@ class LlamaAttention(nn.Module):
111
112 def __init__(
113 self,
87 - hidden_size: int,
88 - num_heads: int,
89 - num_kv_heads: int,
90 - rope_theta: float = 10000,
91 - rope_scaling: Optional[Dict[str, Any]] = None,
92 - max_position_embeddings: int = 8192,
114 + config: LlamaConfig,
115 linear_method: Optional[LinearMethodBase] = None,
116 ) -> None:
117 super().__init__()
96 - self.hidden_size = hidden_size
118 + self.hidden_size = config.hidden_size
119 tp_size = get_tensor_model_parallel_world_size()
98 - self.total_num_heads = num_heads
120 + self.total_num_heads = getattr(config, "num_attention_heads", None)
121 assert self.total_num_heads % tp_size == 0
122 self.num_heads = self.total_num_heads // tp_size
101 - self.total_num_kv_heads = num_kv_heads
123 +
124 + # defaut to mha
125 + self.total_num_kv_heads = getattr(config, "num_key_value_heads",
126 + self.total_num_heads)
127 if self.total_num_kv_heads >= tp_size:
128 # Number of KV heads is greater than TP size, so we partition
129 # the KV heads across multiple tensor parallel GPUs.
@@ -108,39 +133,68 @@ class LlamaAttention(nn.Module):
133 # the KV heads across multiple tensor parallel GPUs.
134 assert tp_size % self.total_num_kv_heads == 0
135 self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
111 - self.head_dim = hidden_size // self.total_num_heads
136 + self.head_dim = self.hidden_size // self.total_num_heads
137 self.q_size = self.num_heads * self.head_dim
138 self.kv_size = self.num_kv_heads * self.head_dim
139 self.scaling = self.head_dim**-0.5
115 - self.rope_theta = rope_theta
116 - self.max_position_embeddings = max_position_embeddings
140 + max_position_embeddings = getattr(config, "max_position_embeddings",
141 + 8192)
142 + self.max_position_embeddings = config.max_position_embeddings
143
144 + # internlm
145 + bias = getattr(config, "bias", False)
146 +
147 + # stablelm
148 + qkv_bias = getattr(config, "use_qkv_bias", False)
149 self.qkv_proj = QKVParallelLinear(
119 - hidden_size,
150 + self.hidden_size,
151 self.head_dim,
152 self.total_num_heads,
153 self.total_num_kv_heads,
123 - bias=False,
154 + bias=bias or qkv_bias,
155 linear_method=linear_method,
156 )
157 self.o_proj = RowParallelLinear(
158 self.total_num_heads * self.head_dim,
128 - hidden_size,
129 - bias=False,
159 + self.hidden_size,
160 + bias=bias,
161 linear_method=linear_method,
162 )
163
133 - self.rotary_emb = get_rope(
134 - self.head_dim,
135 - rotary_dim=self.head_dim,
136 - max_position=max_position_embeddings,
137 - base=rope_theta,
138 - rope_scaling=rope_scaling,
139 - )
140 - self.attn = PagedAttention(self.num_heads,
141 - self.head_dim,
142 - self.scaling,
143 - num_kv_heads=self.num_kv_heads)
164 + # mistral
165 + sliding_window = getattr(config, "sliding_window", None)
166 +
167 + self.postion_embedding = getattr(config, "postion_embedding", "ROPE")
168 + # Create the alibi slopes and slice them.
169 + if self.postion_embedding == "ALIBI":
170 + tp_rank = get_tensor_model_parallel_rank()
171 + head_start = tp_rank * self.num_heads
172 + head_end = (tp_rank + 1) * self.num_heads
173 + alibi_slopes = _get_alibi_slopes(self.total_num_heads)
174 + alibi_slopes = alibi_slopes[head_start:head_end].tolist()
175 +
176 + self.attn = PagedAttention(self.num_heads,
177 + self.head_dim,
178 + self.scaling,
179 + alibi_slopes=alibi_slopes,
180 + sliding_window=sliding_window)
181 + else:
182 + rope_theta = getattr(config, "rope_theta", 10000)
183 + rope_scaling = getattr(config, "rope_scaling", None)
184 + # stablelm
185 + rope_pct = getattr(config, "rope_pct", 1)
186 + self.rotary_emb = get_rope(
187 + self.head_dim,
188 + rotary_dim=int(self.head_dim * rope_pct),
189 + max_position=max_position_embeddings,
190 + base=rope_theta,
191 + rope_scaling=rope_scaling,
192 + )
193 + self.attn = PagedAttention(self.num_heads,
194 + self.head_dim,
195 + self.scaling,
196 + num_kv_heads=self.num_kv_heads,
197 + sliding_window=sliding_window)
198
199 def forward(
200 self,
@@ -151,7 +205,8 @@ class LlamaAttention(nn.Module):
205 ) -> torch.Tensor:
206 qkv, _ = self.qkv_proj(hidden_states)
207 q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
154 - q, k = self.rotary_emb(positions, q, k)
208 + if self.postion_embedding != "ALIBI":
209 + q, k = self.rotary_emb(positions, q, k)
210 k_cache, v_cache = kv_cache
211 attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
212 output, _ = self.o_proj(attn_output)
@@ -164,32 +219,20 @@ class LlamaDecoderLayer(nn.Module):
219 self,
220 config: LlamaConfig,
221 linear_method: Optional[LinearMethodBase] = None,
222 + norm: Optional[torch.Tensor] = None,
223 ) -> None:
224 super().__init__()
225 self.hidden_size = config.hidden_size
170 - rope_theta = getattr(config, "rope_theta", 10000)
171 - rope_scaling = getattr(config, "rope_scaling", None)
172 - max_position_embeddings = getattr(config, "max_position_embeddings",
173 - 8192)
226 self.self_attn = LlamaAttention(
175 - hidden_size=self.hidden_size,
176 - num_heads=config.num_attention_heads,
177 - num_kv_heads=config.num_key_value_heads,
178 - rope_theta=rope_theta,
179 - rope_scaling=rope_scaling,
180 - max_position_embeddings=max_position_embeddings,
227 + config,
228 linear_method=linear_method,
229 )
230 self.mlp = LlamaMLP(
184 - hidden_size=self.hidden_size,
185 - intermediate_size=config.intermediate_size,
186 - hidden_act=config.hidden_act,
231 + config,
232 linear_method=linear_method,
233 )
189 - self.input_layernorm = RMSNorm(config.hidden_size,
190 - eps=config.rms_norm_eps)
191 - self.post_attention_layernorm = RMSNorm(config.hidden_size,
192 - eps=config.rms_norm_eps)
234 + self.input_layernorm = deepcopy(norm)
235 + self.post_attention_layernorm = deepcopy(norm)
236
237 def forward(
238 self,
@@ -226,6 +269,7 @@ class LlamaModel(nn.Module):
269 self,
270 config: LlamaConfig,
271 linear_method: Optional[LinearMethodBase] = None,
272 + norm: Optional[torch.Tensor] = None,
273 lora_config: Optional[LoRAConfig] = None,
274 ) -> None:
275 super().__init__()
@@ -241,10 +285,10 @@ class LlamaModel(nn.Module):
285 org_num_embeddings=config.vocab_size,
286 )
287 self.layers = nn.ModuleList([
244 - LlamaDecoderLayer(config, linear_method)
288 + LlamaDecoderLayer(config, linear_method, norm)
289 for _ in range(config.num_hidden_layers)
290 ])
247 - self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
291 + self.norm = norm
292
293 def forward(
294 self,
@@ -275,12 +319,18 @@ class LlamaForCausalLM(nn.Module):
319 self,
320 config: LlamaConfig,
321 linear_method: Optional[LinearMethodBase] = None,
322 + norm: Optional[torch.Tensor] = None,
323 lora_config: Optional[LoRAConfig] = None,
324 ) -> None:
325 super().__init__()
326 self.config = config
327 self.linear_method = linear_method
283 - self.model = LlamaModel(config, linear_method, lora_config=lora_config)
328 + if norm is None:
329 + norm = RMSNorm(config.hidden_size, config.rms_norm_eps)
330 + self.model = LlamaModel(config,
331 + linear_method,
332 + norm=norm,
333 + lora_config=lora_config)
334 unpadded_vocab_size = config.vocab_size
335 if lora_config:
336 unpadded_vocab_size += lora_config.lora_extra_vocab_size
vllm/model_executor/models/mistral.py deleted
-352
@@ -1,352 +0,0 @@
1 -# coding=utf-8
2 -# Adapted from
3 -# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
4 -# Copyright 2023 The vLLM team.
5 -# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
6 -#
7 -# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
8 -# and OPT implementations in this library. It has been modified from its
9 -# original forms to accommodate minor architectural differences compared
10 -# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
11 -#
12 -# Licensed under the Apache License, Version 2.0 (the "License");
13 -# you may not use this file except in compliance with the License.
14 -# You may obtain a copy of the License at
15 -#
16 -# http://www.apache.org/licenses/LICENSE-2.0
17 -#
18 -# Unless required by applicable law or agreed to in writing, software
19 -# distributed under the License is distributed on an "AS IS" BASIS,
20 -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21 -# See the License for the specific language governing permissions and
22 -# limitations under the License.
23 -"""Inference-only Mistral model compatible with HuggingFace weights."""
24 -from typing import List, Optional, Tuple
25 -
26 -import torch
27 -from torch import nn
28 -from transformers import MistralConfig
29 -
30 -from vllm.model_executor.input_metadata import InputMetadata
31 -from vllm.model_executor.layers.activation import SiluAndMul
32 -from vllm.model_executor.layers.attention import PagedAttention
33 -from vllm.model_executor.layers.layernorm import RMSNorm
34 -from vllm.model_executor.layers.linear import (LinearMethodBase,
35 - MergedColumnParallelLinear,
36 - QKVParallelLinear,
37 - RowParallelLinear)
38 -from vllm.model_executor.layers.rotary_embedding import get_rope
39 -from vllm.model_executor.layers.sampler import Sampler
40 -from vllm.model_executor.layers.vocab_parallel_embedding import (
41 - VocabParallelEmbedding, ParallelLMHead, DEFAULT_VOCAB_PADDING_SIZE)
42 -from vllm.model_executor.parallel_utils.parallel_state import (
43 - get_tensor_model_parallel_world_size)
44 -from vllm.model_executor.sampling_metadata import SamplingMetadata
45 -from vllm.model_executor.weight_utils import (default_weight_loader,
46 - hf_model_weights_iterator)
47 -from vllm.sequence import SamplerOutput
48 -from vllm.config import LoRAConfig
49 -
50 -KVCache = Tuple[torch.Tensor, torch.Tensor]
51 -
52 -
53 -class MistralMLP(nn.Module):
54 -
55 - def __init__(
56 - self,
57 - hidden_size: int,
58 - intermediate_size: int,
59 - hidden_act: str,
60 - linear_method: Optional[LinearMethodBase] = None,
61 - ) -> None:
62 - super().__init__()
63 - self.gate_up_proj = MergedColumnParallelLinear(
64 - hidden_size, [intermediate_size] * 2,
65 - bias=False,
66 - linear_method=linear_method)
67 - self.down_proj = RowParallelLinear(intermediate_size,
68 - hidden_size,
69 - bias=False,
70 - linear_method=linear_method)
71 - if hidden_act != "silu":
72 - raise ValueError(f"Unsupported activation: {hidden_act}. "
73 - "Only silu is supported for now.")
74 - self.act_fn = SiluAndMul()
75 -
76 - def forward(self, x):
77 - gate_up, _ = self.gate_up_proj(x)
78 - x = self.act_fn(gate_up)
79 - x, _ = self.down_proj(x)
80 - return x
81 -
82 -
83 -class MistralAttention(nn.Module):
84 -
85 - def __init__(self,
86 - hidden_size: int,
87 - num_heads: int,
88 - num_kv_heads: int,
89 - max_position: int = 4096 * 32,
90 - rope_theta: float = 10000,
91 - linear_method: Optional[LinearMethodBase] = None,
92 - sliding_window: Optional[int] = None) -> None:
93 - super().__init__()
94 - self.hidden_size = hidden_size
95 - tp_size = get_tensor_model_parallel_world_size()
96 - self.total_num_heads = num_heads
97 - assert self.total_num_heads % tp_size == 0
98 - self.num_heads = self.total_num_heads // tp_size
99 - self.total_num_kv_heads = num_kv_heads
100 - if self.total_num_kv_heads >= tp_size:
101 - # Number of KV heads is greater than TP size, so we partition
102 - # the KV heads across multiple tensor parallel GPUs.
103 - assert self.total_num_kv_heads % tp_size == 0
104 - else:
105 - # Number of KV heads is less than TP size, so we replicate
106 - # the KV heads across multiple tensor parallel GPUs.
107 - assert tp_size % self.total_num_kv_heads == 0
108 - self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
109 - self.head_dim = hidden_size // self.total_num_heads
110 - self.q_size = self.num_heads * self.head_dim
111 - self.kv_size = self.num_kv_heads * self.head_dim
112 - self.scaling = self.head_dim**-0.5
113 - self.rope_theta = rope_theta
114 - self.sliding_window = sliding_window
115 -
116 - self.qkv_proj = QKVParallelLinear(
117 - hidden_size,
118 - self.head_dim,
119 - self.total_num_heads,
120 - self.total_num_kv_heads,
121 - bias=False,
122 - linear_method=linear_method,
123 - )
124 - self.o_proj = RowParallelLinear(
125 - self.total_num_heads * self.head_dim,
126 - hidden_size,
127 - bias=False,
128 - linear_method=linear_method,
129 - )
130 -
131 - self.rotary_emb = get_rope(
132 - self.head_dim,
133 - rotary_dim=self.head_dim,
134 - max_position=max_position,
135 - base=self.rope_theta,
136 - )
137 - self.attn = PagedAttention(self.num_heads,
138 - self.head_dim,
139 - self.scaling,
140 - num_kv_heads=self.num_kv_heads,
141 - sliding_window=self.sliding_window)
142 -
143 - def forward(
144 - self,
145 - positions: torch.Tensor,
146 - hidden_states: torch.Tensor,
147 - kv_cache: KVCache,
148 - input_metadata: InputMetadata,
149 - ) -> torch.Tensor:
150 - qkv, _ = self.qkv_proj(hidden_states)
151 - q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
152 - q, k = self.rotary_emb(positions, q, k)
153 - k_cache, v_cache = kv_cache
154 - attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
155 - output, _ = self.o_proj(attn_output)
156 - return output
157 -
158 -
159 -class MistralDecoderLayer(nn.Module):
160 -
161 - def __init__(
162 - self,
163 - config: MistralConfig,
164 - linear_method: Optional[LinearMethodBase] = None,
165 - ) -> None:
166 - super().__init__()
167 - self.hidden_size = config.hidden_size
168 - # Requires transformers > 4.32.0
169 - rope_theta = getattr(config, "rope_theta", 10000)
170 - self.self_attn = MistralAttention(
171 - hidden_size=self.hidden_size,
172 - num_heads=config.num_attention_heads,
173 - max_position=config.max_position_embeddings,
174 - num_kv_heads=config.num_key_value_heads,
175 - rope_theta=rope_theta,
176 - linear_method=linear_method,
177 - sliding_window=config.sliding_window)
178 - self.mlp = MistralMLP(
179 - hidden_size=self.hidden_size,
180 - intermediate_size=config.intermediate_size,
181 - hidden_act=config.hidden_act,
182 - linear_method=linear_method,
183 - )
184 - self.input_layernorm = RMSNorm(config.hidden_size,
185 - eps=config.rms_norm_eps)
186 - self.post_attention_layernorm = RMSNorm(config.hidden_size,
187 - eps=config.rms_norm_eps)
188 -
189 - def forward(
190 - self,
191 - positions: torch.Tensor,
192 - hidden_states: torch.Tensor,
193 - kv_cache: KVCache,
194 - input_metadata: InputMetadata,
195 - residual: Optional[torch.Tensor],
196 - ) -> Tuple[torch.Tensor, torch.Tensor]:
197 - # Self Attention
198 - if residual is None:
199 - residual = hidden_states
200 - hidden_states = self.input_layernorm(hidden_states)
201 - else:
202 - hidden_states, residual = self.input_layernorm(
203 - hidden_states, residual)
204 - hidden_states = self.self_attn(
205 - positions=positions,
206 - hidden_states=hidden_states,
207 - kv_cache=kv_cache,
208 - input_metadata=input_metadata,
209 - )
210 -
211 - # Fully Connected
212 - hidden_states, residual = self.post_attention_layernorm(
213 - hidden_states, residual)
214 - hidden_states = self.mlp(hidden_states)
215 - return hidden_states, residual
216 -
217 -
218 -class MistralModel(nn.Module):
219 -
220 - def __init__(
221 - self,
222 - config: MistralConfig,
223 - linear_method: Optional[LinearMethodBase] = None,
224 - lora_config: Optional[LoRAConfig] = None,
225 - ) -> None:
226 - super().__init__()
227 - self.config = config
228 - self.padding_idx = config.pad_token_id
229 - lora_vocab = (lora_config.lora_extra_vocab_size *
230 - (lora_config.max_loras or 1)) if lora_config else 0
231 - self.vocab_size = config.vocab_size + lora_vocab
232 - self.org_vocab_size = config.vocab_size
233 -
234 - self.embed_tokens = VocabParallelEmbedding(
235 - self.vocab_size,
236 - config.hidden_size,
237 - org_num_embeddings=config.vocab_size,
238 - )
239 - self.layers = nn.ModuleList([
240 - MistralDecoderLayer(config, linear_method)
241 - for _ in range(config.num_hidden_layers)
242 - ])
243 - self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
244 -
245 - def forward(
246 - self,
247 - input_ids: torch.Tensor,
248 - positions: torch.Tensor,
249 - kv_caches: List[KVCache],
250 - input_metadata: InputMetadata,
251 - ) -> torch.Tensor:
252 - hidden_states = self.embed_tokens(input_ids)
253 - residual = None
254 - for i in range(len(self.layers)):
255 - layer = self.layers[i]
256 - hidden_states, residual = layer(
257 - positions,
258 - hidden_states,
259 - kv_caches[i],
260 - input_metadata,
261 - residual,
262 - )
263 - hidden_states, _ = self.norm(hidden_states, residual)
264 - return hidden_states
265 -
266 -
267 -class MistralForCausalLM(nn.Module):
268 - supports_lora = True
269 -
270 - def __init__(
271 - self,
272 - config: MistralConfig,
273 - linear_method: Optional[LinearMethodBase] = None,
274 - lora_config: Optional[LoRAConfig] = None,
275 - ) -> None:
276 - super().__init__()
277 - self.config = config
278 - self.linear_method = linear_method
279 - self.model = MistralModel(config,
280 - linear_method,
281 - lora_config=lora_config)
282 - unpadded_vocab_size = config.vocab_size
283 - if lora_config:
284 - unpadded_vocab_size += lora_config.lora_extra_vocab_size
285 - self.lm_head = ParallelLMHead(
286 - unpadded_vocab_size,
287 - config.hidden_size,
288 - org_num_embeddings=config.vocab_size,
289 - padding_size=DEFAULT_VOCAB_PADDING_SIZE
290 - # We need bigger padding if using lora for kernel
291 - # compatibility
292 - if not lora_config else lora_config.lora_vocab_padding_size,
293 - )
294 - self.sampler = Sampler(unpadded_vocab_size, config.vocab_size)
295 -
296 - def forward(
297 - self,
298 - input_ids: torch.Tensor,
299 - positions: torch.Tensor,
300 - kv_caches: List[KVCache],
301 - input_metadata: InputMetadata,
302 - ) -> torch.Tensor:
303 - hidden_states = self.model(input_ids, positions, kv_caches,
304 - input_metadata)
305 - return hidden_states
306 -
307 - def sample(
308 - self,
309 - hidden_states: torch.Tensor,
310 - sampling_metadata: SamplingMetadata,
311 - ) -> Optional[SamplerOutput]:
312 - next_tokens = self.sampler(self.lm_head.weight, hidden_states,
313 - sampling_metadata)
314 - return next_tokens
315 -
316 - def load_weights(self,
317 - model_name_or_path: str,
318 - cache_dir: Optional[str] = None,
319 - load_format: str = "auto",
320 - revision: Optional[str] = None):
321 - stacked_params_mapping = [
322 - # (param_name, shard_name, shard_id)
323 - ("qkv_proj", "q_proj", "q"),
324 - ("qkv_proj", "k_proj", "k"),
325 - ("qkv_proj", "v_proj", "v"),
326 - ("gate_up_proj", "gate_proj", 0),
327 - ("gate_up_proj", "up_proj", 1),
328 - ]
329 - params_dict = dict(self.named_parameters())
330 - for name, loaded_weight in hf_model_weights_iterator(
331 - model_name_or_path, cache_dir, load_format, revision):
332 - if "rotary_emb.inv_freq" in name:
333 - continue
334 - for (param_name, weight_name, shard_id) in stacked_params_mapping:
335 - if weight_name not in name:
336 - continue
337 - name = name.replace(weight_name, param_name)
338 - # Skip loading extra bias for GPTQ models.
339 - if name.endswith(".bias") and name not in params_dict:
340 - continue
341 - param = params_dict[name]
342 - weight_loader = param.weight_loader
343 - weight_loader(param, loaded_weight, shard_id)
344 - break
345 - else:
346 - # Skip loading extra bias for GPTQ models.
347 - if name.endswith(".bias") and name not in params_dict:
348 - continue
349 - param = params_dict[name]
350 - weight_loader = getattr(param, "weight_loader",
351 - default_weight_loader)
352 - weight_loader(param, loaded_weight)
vllm/model_executor/models/qwen.py
+31 -236
@@ -4,253 +4,33 @@
4 # Copyright (c) Alibaba Cloud.
5 # LICENSE: https://huggingface.co/Qwen/Qwen-7B/blob/main/LICENSE
6 """Inference-only QWen model compatible with HuggingFace weights."""
7 -from typing import Any, Dict, List, Optional, Tuple
7 +from typing import Optional
8
9 -import torch
10 -from torch import nn
9 +from transformers import PretrainedConfig
10 +from vllm.config import LoRAConfig
11
12 -from vllm.model_executor.input_metadata import InputMetadata
13 -from vllm.model_executor.layers.activation import SiluAndMul
14 -from vllm.model_executor.layers.attention import PagedAttention
12 +from vllm.model_executor.layers.linear import LinearMethodBase
13 from vllm.model_executor.layers.layernorm import RMSNorm
16 -from vllm.model_executor.layers.linear import (LinearMethodBase,
17 - MergedColumnParallelLinear,
18 - QKVParallelLinear,
19 - RowParallelLinear)
20 -from vllm.model_executor.layers.rotary_embedding import get_rope
21 -from vllm.model_executor.layers.sampler import Sampler
22 -from vllm.model_executor.layers.vocab_parallel_embedding import (
23 - VocabParallelEmbedding, ParallelLMHead)
24 -from vllm.model_executor.parallel_utils.parallel_state import (
25 - get_tensor_model_parallel_world_size)
26 -from vllm.model_executor.sampling_metadata import SamplingMetadata
14 +from vllm.model_executor.models.llama import LlamaForCausalLM
15 from vllm.model_executor.weight_utils import (default_weight_loader,
16 hf_model_weights_iterator)
29 -from vllm.sequence import SamplerOutput
30 -from vllm.transformers_utils.configs.qwen import QWenConfig
17
32 -KVCache = Tuple[torch.Tensor, torch.Tensor]
18
34 -
35 -class QWenMLP(nn.Module):
36 -
37 - def __init__(
38 - self,
39 - hidden_size: int,
40 - intermediate_size: int,
41 - hidden_act: str = "silu",
42 - linear_method: Optional[LinearMethodBase] = None,
43 - ):
44 - super().__init__()
45 - self.gate_up_proj = MergedColumnParallelLinear(
46 - hidden_size, [intermediate_size] * 2,
47 - bias=False,
48 - linear_method=linear_method)
49 - self.c_proj = RowParallelLinear(intermediate_size,
50 - hidden_size,
51 - bias=False,
52 - linear_method=linear_method)
53 - if hidden_act != "silu":
54 - raise ValueError(f"Unsupported activation: {hidden_act}. "
55 - "Only silu is supported for now.")
56 - self.act_fn = SiluAndMul()
57 -
58 - def forward(self, x):
59 - gate_up, _ = self.gate_up_proj(x)
60 - x = self.act_fn(gate_up)
61 - x, _ = self.c_proj(x)
62 - return x
63 -
64 -
65 -class QWenAttention(nn.Module):
66 -
67 - def __init__(
68 - self,
69 - hidden_size: int,
70 - num_heads: int,
71 - max_position_embeddings: int,
72 - rope_theta: float = 10000,
73 - rope_scaling: Optional[Dict[str, Any]] = None,
74 - linear_method: Optional[LinearMethodBase] = None,
75 - ):
76 - super().__init__()
77 - self.hidden_size = hidden_size
78 - tensor_model_parallel_world_size = get_tensor_model_parallel_world_size(
79 - )
80 - self.total_num_heads = num_heads
81 - assert self.total_num_heads % tensor_model_parallel_world_size == 0
82 - self.num_heads = (self.total_num_heads //
83 - tensor_model_parallel_world_size)
84 - self.head_dim = hidden_size // self.total_num_heads
85 - self.c_attn = QKVParallelLinear(
86 - hidden_size,
87 - self.head_dim,
88 - self.total_num_heads,
89 - bias=True,
90 - linear_method=linear_method,
91 - )
92 - self.c_proj = RowParallelLinear(
93 - self.total_num_heads * self.head_dim,
94 - hidden_size,
95 - bias=False,
96 - linear_method=linear_method,
97 - )
98 - self.scaling = self.head_dim**-0.5
99 -
100 - self.rotary_emb = get_rope(
101 - self.head_dim,
102 - rotary_dim=self.head_dim,
103 - max_position=max_position_embeddings,
104 - base=rope_theta,
105 - rope_scaling=rope_scaling,
106 - )
107 - self.attn = PagedAttention(self.num_heads, self.head_dim, self.scaling)
108 -
109 - def forward(
110 - self,
111 - positions: torch.Tensor,
112 - hidden_states: torch.Tensor,
113 - kv_cache: KVCache,
114 - input_metadata: InputMetadata,
115 - ) -> torch.Tensor:
116 - qkv, _ = self.c_attn(hidden_states)
117 - q, k, v = qkv.chunk(chunks=3, dim=-1)
118 - q, k = self.rotary_emb(positions, q, k)
119 - k_cache, v_cache = kv_cache
120 - attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
121 -
122 - output, _ = self.c_proj(attn_output)
123 - return output
124 -
125 -
126 -class QWenBlock(nn.Module):
127 -
128 - def __init__(
129 - self,
130 - config: QWenConfig,
131 - linear_method: Optional[LinearMethodBase] = None,
132 - ):
133 - super().__init__()
134 - self.ln_1 = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
135 -
136 - rope_theta = getattr(config, "rope_theta", 10000)
137 - rope_scaling = getattr(config, "rope_scaling", None)
138 - self.attn = QWenAttention(config.hidden_size,
139 - config.num_attention_heads,
140 - config.max_position_embeddings,
141 - rope_theta=rope_theta,
142 - rope_scaling=rope_scaling,
143 - linear_method=linear_method)
144 -
145 - self.ln_2 = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
146 -
147 - self.mlp = QWenMLP(config.hidden_size,
148 - config.intermediate_size // 2,
149 - linear_method=linear_method)
150 -
151 - def forward(
152 - self,
153 - positions: torch.Tensor,
154 - hidden_states: torch.Tensor,
155 - kv_cache: KVCache,
156 - input_metadata: InputMetadata,
157 - residual: Optional[torch.Tensor],
158 - ) -> Tuple[torch.Tensor, torch.Tensor]:
159 - # Self Attention
160 - if residual is None:
161 - residual = hidden_states
162 - hidden_states = self.ln_1(hidden_states)
163 - else:
164 - hidden_states, residual = self.ln_1(hidden_states, residual)
165 - hidden_states = self.attn(
166 - positions=positions,
167 - hidden_states=hidden_states,
168 - kv_cache=kv_cache,
169 - input_metadata=input_metadata,
170 - )
171 -
172 - # Fully Connected
173 - hidden_states, residual = self.ln_2(hidden_states, residual)
174 - hidden_states = self.mlp(hidden_states)
175 - return hidden_states, residual
176 -
177 -
178 -class QWenModel(nn.Module):
19 +class QWenLMHeadModel(LlamaForCausalLM):
20
21 def __init__(
22 self,
182 - config: QWenConfig,
23 + config: Optional[PretrainedConfig] = None,
24 linear_method: Optional[LinearMethodBase] = None,
184 - ):
185 - super().__init__()
186 - self.config = config
187 - self.vocab_size = config.vocab_size
188 -
189 - self.wte = VocabParallelEmbedding(
190 - config.vocab_size,
191 - config.hidden_size,
192 - )
193 - self.h = nn.ModuleList([
194 - QWenBlock(config, linear_method)
195 - for _ in range(config.num_hidden_layers)
196 - ])
197 - self.ln_f = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
198 -
199 - def forward(
200 - self,
201 - input_ids: torch.Tensor,
202 - positions: torch.Tensor,
203 - kv_caches: List[KVCache],
204 - input_metadata: InputMetadata,
205 - ) -> torch.Tensor:
206 - hidden_states = self.wte(input_ids)
207 - residual = None
208 - for i in range(len(self.h)):
209 - layer = self.h[i]
210 - hidden_states, residual = layer(
211 - positions,
212 - hidden_states,
213 - kv_caches[i],
214 - input_metadata,
215 - residual,
216 - )
217 - hidden_states, _ = self.ln_f(hidden_states, residual)
218 - return hidden_states
219 -
220 -
221 -class QWenLMHeadModel(nn.Module):
222 -
223 - def __init__(
224 - self,
225 - config: QWenConfig,
226 - linear_method: Optional[LinearMethodBase] = None,
227 - ):
228 - super().__init__()
229 - self.config = config
230 - self.linear_method = linear_method
231 - self.transformer = QWenModel(config, linear_method)
232 - self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
233 - self.sampler = Sampler(config.vocab_size)
234 -
235 - def forward(
236 - self,
237 - input_ids: torch.Tensor,
238 - positions: torch.Tensor,
239 - kv_caches: List[KVCache],
240 - input_metadata: InputMetadata,
241 - ) -> torch.Tensor:
242 - hidden_states = self.transformer(input_ids, positions, kv_caches,
243 - input_metadata)
244 - return hidden_states
245 -
246 - def sample(
247 - self,
248 - hidden_states: torch.Tensor,
249 - sampling_metadata: SamplingMetadata,
250 - ) -> Optional[SamplerOutput]:
251 - next_tokens = self.sampler(self.lm_head.weight, hidden_states,
252 - sampling_metadata)
253 - return next_tokens
25 + lora_config: Optional[LoRAConfig] = None,
26 + ) -> None:
27 + norm = RMSNorm(config.hidden_size, config.layer_norm_epsilon)
28 + config.use_qkv_bias = True
29 + config.intermediate_size = config.intermediate_size // 2
30 + super().__init__(config=config,
31 + linear_method=linear_method,
32 + norm=norm,
33 + lora_config=lora_config)
34
35 def load_weights(self,
36 model_name_or_path: str,
@@ -262,9 +42,24 @@ class QWenLMHeadModel(nn.Module):
42 ("gate_up_proj", "w2", 0),
43 ("gate_up_proj", "w1", 1),
44 ]
45 + param_weight_map = [
46 + ("model", "transformer"),
47 + (".self_attn.", ".attn."),
48 + (".layers.", ".h."),
49 + ("qkv_proj", "c_attn"),
50 + (".self_attn.o_proj", ".self_attn.c_proj"),
51 + ("norm", "ln_f"),
52 + ("mlp.down_proj", "mlp.c_proj"),
53 + ("input_layernorm", "ln_1"),
54 + ("post_attention_layernorm", "ln_2"),
55 + ("embed_tokens", "wte"),
56 + ]
57 params_dict = dict(self.named_parameters())
58 for name, loaded_weight in hf_model_weights_iterator(
59 model_name_or_path, cache_dir, load_format, revision):
60 + for (param_name, weight_name) in param_weight_map:
61 + name = name.replace(weight_name, param_name)
62 +
63 if "rotary_emb.inv_freq" in name:
64 continue
65 for (param_name, weight_name, shard_id) in stacked_params_mapping:
vllm/model_executor/models/stablelm.py
+13 -270
@@ -17,283 +17,26 @@
17 # https://huggingface.co/stabilityai/stablelm-3b-4e1t/blob/main/modeling_stablelm_epoch.py
18 # https://huggingface.co/stabilityai/stablelm-3b-4e1t/blob/main/config.json
19 """Inference-only StabeLM (https://github.com/Stability-AI/StableLM) model compatible with HuggingFace weights."""
20 -from typing import List, Optional, Tuple
20 +from typing import Optional
21
22 -import torch
23 -from torch import nn
22 from transformers import PretrainedConfig
23
26 -from vllm.model_executor.input_metadata import InputMetadata
27 -from vllm.model_executor.layers.activation import SiluAndMul
28 -from vllm.model_executor.layers.attention import PagedAttention
29 -from vllm.model_executor.layers.linear import (LinearMethodBase,
30 - MergedColumnParallelLinear,
31 - QKVParallelLinear,
32 - RowParallelLinear)
33 -from vllm.model_executor.layers.rotary_embedding import get_rope
34 -from vllm.model_executor.layers.sampler import Sampler
35 -from vllm.model_executor.layers.vocab_parallel_embedding import (
36 - VocabParallelEmbedding, ParallelLMHead)
37 -from vllm.model_executor.parallel_utils.parallel_state import (
38 - get_tensor_model_parallel_world_size)
39 -from vllm.model_executor.sampling_metadata import SamplingMetadata
40 -from vllm.model_executor.weight_utils import (default_weight_loader,
41 - hf_model_weights_iterator)
42 -from vllm.sequence import SamplerOutput
24 +from vllm.model_executor.layers.linear import LinearMethodBase
25 +from vllm.model_executor.layers.layernorm import LayerNorm
26 +from vllm.model_executor.models.llama import LlamaForCausalLM
27 +from vllm.config import LoRAConfig
28
44 -KVCache = Tuple[torch.Tensor, torch.Tensor]
29
46 -
47 -class StablelmMLP(nn.Module):
48 -
49 - def __init__(self,
50 - config: PretrainedConfig,
51 - linear_method: Optional[LinearMethodBase] = None) -> None:
52 - super().__init__()
53 - self.config = config
54 - self.hidden_size = config.hidden_size
55 - self.intermediate_size = config.intermediate_size
56 - self.gate_up_proj = MergedColumnParallelLinear(
57 - config.hidden_size, [config.intermediate_size] * 2,
58 - bias=False,
59 - linear_method=linear_method)
60 - self.down_proj = RowParallelLinear(config.intermediate_size,
61 - config.hidden_size,
62 - bias=False)
63 - self.act_fn = SiluAndMul()
64 -
65 - def forward(self, x: torch.Tensor) -> torch.Tensor:
66 - gate_up, _ = self.gate_up_proj(x)
67 - x = self.act_fn(gate_up)
68 - x, _ = self.down_proj(x)
69 - return x
70 -
71 -
72 -class StablelmAttention(nn.Module):
73 -
74 - def __init__(self,
75 - config: PretrainedConfig,
76 - linear_method: Optional[LinearMethodBase] = None) -> None:
77 - super().__init__()
78 - self.config = config
79 - self.hidden_size = config.hidden_size
80 - tp_size = get_tensor_model_parallel_world_size()
81 - self.total_num_heads = config.num_attention_heads
82 - self.num_heads = self.total_num_heads // tp_size
83 -
84 - self.total_num_key_value_heads = config.num_key_value_heads
85 - if self.total_num_key_value_heads >= tp_size:
86 - # Number of KV heads is greater than TP size, so we partition
87 - # the KV heads across multiple tensor parallel GPUs.
88 - assert self.total_num_key_value_heads % tp_size == 0
89 - else:
90 - # Number of KV heads is less than TP size, so we replicate
91 - # the KV heads across multiple tensor parallel GPUs.
92 - assert tp_size % self.total_num_key_value_heads == 0
93 - self.num_key_value_heads = max(
94 - 1, self.total_num_key_value_heads // tp_size)
95 - self.head_dim = self.hidden_size // self.total_num_heads
96 - self.max_position_embeddings = config.max_position_embeddings
97 - self.rotary_ndims = int(self.head_dim * self.config.rope_pct)
98 - self.scaling = self.head_dim**-0.5
99 - self.q_size = self.num_heads * self.head_dim
100 - self.kv_size = self.num_key_value_heads * self.head_dim
101 - self.qkv_bias = getattr(config, "use_qkv_bias", False)
102 - if (self.head_dim * self.num_heads * tp_size) != self.hidden_size:
103 - raise ValueError(
104 - f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}"
105 - f" and `num_heads`: {self.num_heads}).")
106 -
107 - self.qkv_proj = QKVParallelLinear(self.hidden_size,
108 - self.head_dim,
109 - self.total_num_heads,
110 - self.total_num_key_value_heads,
111 - self.qkv_bias,
112 - linear_method=linear_method)
113 - self.o_proj = RowParallelLinear(self.total_num_heads * self.head_dim,
114 - self.hidden_size,
115 - bias=False,
116 - linear_method=linear_method)
117 - self.rotary_ndims = int(self.head_dim * self.config.rope_pct)
118 - self.rotary_emb = get_rope(
119 - self.head_dim,
120 - rotary_dim=self.rotary_ndims,
121 - max_position=self.config.max_position_embeddings,
122 - base=self.config.rope_theta,
123 - )
124 - self.attn = PagedAttention(self.num_heads,
125 - self.head_dim,
126 - self.scaling,
127 - num_kv_heads=self.num_key_value_heads)
128 -
129 - def forward(
130 - self,
131 - positions: torch.Tensor,
132 - hidden_states: torch.Tensor,
133 - kv_cache: KVCache,
134 - input_metadata: InputMetadata,
135 - ) -> torch.Tensor:
136 - qkv, _ = self.qkv_proj(hidden_states)
137 - q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
138 - q, k = self.rotary_emb(positions, q, k)
139 - k_cache, v_cache = kv_cache
140 - attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
141 - output, _ = self.o_proj(attn_output)
142 - return output
143 -
144 -
145 -class StablelmDecoderLayer(nn.Module):
30 +class StablelmForCausalLM(LlamaForCausalLM):
31
32 def __init__(
33 self,
149 - config: PretrainedConfig,
34 + config: Optional[PretrainedConfig] = None,
35 linear_method: Optional[LinearMethodBase] = None,
36 + lora_config: Optional[LoRAConfig] = None,
37 ) -> None:
152 - super().__init__()
153 - self.self_attn = StablelmAttention(config)
154 - self.mlp = StablelmMLP(config, linear_method)
155 - self.input_layernorm = nn.LayerNorm(config.hidden_size,
156 - eps=config.norm_eps)
157 - self.post_attention_layernorm = nn.LayerNorm(config.hidden_size,
158 - eps=config.norm_eps)
159 -
160 - def forward(
161 - self,
162 - positions: torch.Tensor,
163 - hidden_states: torch.Tensor,
164 - kv_cache: KVCache,
165 - input_metadata: InputMetadata,
166 - ) -> Tuple[torch.Tensor, torch.Tensor]:
167 - # Self Attention
168 - residual = hidden_states
169 - hidden_states = self.input_layernorm(hidden_states)
170 - hidden_states = self.self_attn(
171 - positions=positions,
172 - hidden_states=hidden_states,
173 - kv_cache=kv_cache,
174 - input_metadata=input_metadata,
175 - )
176 - hidden_states = residual + hidden_states
177 -
178 - # Fully Connected
179 - residual = hidden_states
180 - hidden_states = self.post_attention_layernorm(hidden_states)
181 - hidden_states = self.mlp(hidden_states)
182 - hidden_states = residual + hidden_states
183 -
184 - return hidden_states, residual
185 -
186 -
187 -class StableLMEpochModel(nn.Module):
188 -
189 - def __init__(self,
190 - config: PretrainedConfig,
191 - linear_method: Optional[LinearMethodBase] = None) -> None:
192 - super().__init__()
193 - # self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, config.pad_token_id)
194 - self.embed_tokens = VocabParallelEmbedding(
195 - config.vocab_size,
196 - config.hidden_size,
197 - )
198 - self.layers = nn.ModuleList([
199 - StablelmDecoderLayer(config, linear_method)
200 - for _ in range(config.num_hidden_layers)
201 - ])
202 - self.norm = nn.LayerNorm(config.hidden_size, eps=config.norm_eps)
203 -
204 - def forward(
205 - self,
206 - input_ids: torch.Tensor,
207 - positions: torch.Tensor,
208 - kv_caches: List[KVCache],
209 - input_metadata: InputMetadata,
210 - ) -> torch.Tensor:
211 - hidden_states = self.embed_tokens(input_ids)
212 - for i in range(len(self.layers)):
213 - layer = self.layers[i]
214 - hidden_states, residual = layer(
215 - positions,
216 - hidden_states,
217 - kv_caches[i],
218 - input_metadata,
219 - )
220 - hidden_states = self.norm(hidden_states)
221 - return hidden_states
222 -
223 -
224 -class StablelmForCausalLM(nn.Module):
225 -
226 - def __init__(
227 - self,
228 - config: PretrainedConfig,
229 - linear_method: Optional[LinearMethodBase] = None,
230 - ) -> None:
231 - super().__init__()
232 - self.config = config
233 - self.linear_method = linear_method
234 - self.model = StableLMEpochModel(config, linear_method)
235 - self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
236 - self.sampler = Sampler(config.vocab_size)
237 -
238 - def forward(
239 - self,
240 - input_ids: torch.Tensor,
241 - positions: torch.Tensor,
242 - kv_caches: List[KVCache],
243 - input_metadata: InputMetadata,
244 - ) -> torch.Tensor:
245 - hidden_states = self.model(input_ids, positions, kv_caches,
246 - input_metadata)
247 - return hidden_states
248 -
249 - def sample(
250 - self,
251 - hidden_states: torch.Tensor,
252 - sampling_metadata: SamplingMetadata,
253 - ) -> Optional[SamplerOutput]:
254 - next_tokens = self.sampler(self.lm_head.weight, hidden_states,
255 - sampling_metadata)
256 - return next_tokens
257 -
258 - def load_weights(self,
259 - model_name_or_path: str,
260 - cache_dir: Optional[str] = None,
261 - load_format: str = "auto",
262 - revision: Optional[str] = None):
263 - stacked_params_mapping = [
264 - # (param_name, shard_name, shard_id)
265 - ("qkv_proj", "q_proj", "q"),
266 - ("qkv_proj", "k_proj", "k"),
267 - ("qkv_proj", "v_proj", "v"),
268 - ("gate_up_proj", "gate_proj", 0),
269 - ("gate_up_proj", "up_proj", 1),
270 - ]
271 - params_dict = dict(self.named_parameters())
272 - for name, loaded_weight in hf_model_weights_iterator(
273 - model_name_or_path, cache_dir, load_format, revision):
274 - if "rotary_emb.inv_freq" in name:
275 - continue
276 - if ("rotary_emb.cos_cached" in name
277 - or "rotary_emb.sin_cached" in name):
278 - # Models trained using ColossalAI may include these tensors in
279 - # the checkpoint. Skip them.
280 - continue
281 - for (param_name, weight_name, shard_id) in stacked_params_mapping:
282 - if weight_name not in name:
283 - continue
284 - name = name.replace(weight_name, param_name)
285 - # Skip loading extra bias for GPTQ models.
286 - if name.endswith(".bias") and name not in params_dict:
287 - continue
288 - param = params_dict[name]
289 - weight_loader = param.weight_loader
290 - weight_loader(param, loaded_weight, shard_id)
291 - break
292 - else:
293 - # Skip loading extra bias for GPTQ models.
294 - if name.endswith(".bias") and name not in params_dict:
295 - continue
296 - param = params_dict[name]
297 - weight_loader = getattr(param, "weight_loader",
298 - default_weight_loader)
299 - weight_loader(param, loaded_weight)
38 + norm = LayerNorm(config.hidden_size, config.norm_eps)
39 + super().__init__(config=config,
40 + linear_method=linear_method,
41 + norm=norm,
42 + lora_config=lora_config)
vllm/model_executor/models/yi.py deleted
-330
@@ -1,330 +0,0 @@
1 -# coding=utf-8
2 -# Adapted from
3 -# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
4 -# Copyright 2023 The vLLM team.
5 -# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
6 -#
7 -# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
8 -# and OPT implementations in this library. It has been modified from its
9 -# original forms to accommodate minor architectural differences compared
10 -# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
11 -#
12 -# Licensed under the Apache License, Version 2.0 (the "License");
13 -# you may not use this file except in compliance with the License.
14 -# You may obtain a copy of the License at
15 -#
16 -# http://www.apache.org/licenses/LICENSE-2.0
17 -#
18 -# Unless required by applicable law or agreed to in writing, software
19 -# distributed under the License is distributed on an "AS IS" BASIS,
20 -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21 -# See the License for the specific language governing permissions and
22 -# limitations under the License.
23 -"""Inference-only Yi model (https://01.ai) compatible with HuggingFace weights."""
24 -from typing import Any, Dict, List, Optional, Tuple
25 -
26 -import torch
27 -from torch import nn
28 -from vllm.transformers_utils.configs.yi import YiConfig
29 -
30 -from vllm.model_executor.input_metadata import InputMetadata
31 -from vllm.model_executor.layers.activation import SiluAndMul
32 -from vllm.model_executor.layers.attention import PagedAttention
33 -from vllm.model_executor.layers.layernorm import RMSNorm
34 -from vllm.model_executor.layers.linear import (LinearMethodBase,
35 - MergedColumnParallelLinear,
36 - QKVParallelLinear,
37 - RowParallelLinear)
38 -from vllm.model_executor.layers.rotary_embedding import get_rope
39 -from vllm.model_executor.layers.sampler import Sampler
40 -from vllm.model_executor.layers.vocab_parallel_embedding import (
41 - VocabParallelEmbedding, ParallelLMHead)
42 -from vllm.model_executor.parallel_utils.parallel_state import (
43 - get_tensor_model_parallel_world_size)
44 -from vllm.model_executor.sampling_metadata import SamplingMetadata
45 -from vllm.model_executor.weight_utils import (default_weight_loader,
46 - hf_model_weights_iterator)
47 -from vllm.sequence import SamplerOutput
48 -
49 -KVCache = Tuple[torch.Tensor, torch.Tensor]
50 -
51 -
52 -class YiMLP(nn.Module):
53 -
54 - def __init__(
55 - self,
56 - hidden_size: int,
57 - intermediate_size: int,
58 - hidden_act: str,
59 - linear_method: Optional[LinearMethodBase] = None,
60 - ) -> None:
61 - super().__init__()
62 - self.gate_up_proj = MergedColumnParallelLinear(
63 - hidden_size, [intermediate_size] * 2,
64 - bias=False,
65 - linear_method=linear_method)
66 - self.down_proj = RowParallelLinear(intermediate_size,
67 - hidden_size,
68 - bias=False,
69 - linear_method=linear_method)
70 - if hidden_act != "silu":
71 - raise ValueError(f"Unsupported activation: {hidden_act}. "
72 - "Only silu is supported for now.")
73 - self.act_fn = SiluAndMul()
74 -
75 - def forward(self, x):
76 - gate_up, _ = self.gate_up_proj(x)
77 - x = self.act_fn(gate_up)
78 - x, _ = self.down_proj(x)
79 - return x
80 -
81 -
82 -class YiAttention(nn.Module):
83 -
84 - def __init__(
85 - self,
86 - hidden_size: int,
87 - num_heads: int,
88 - num_kv_heads: int,
89 - rope_theta: float = 10000,
90 - rope_scaling: Optional[Dict[str, Any]] = None,
91 - max_position_embeddings: int = 8192,
92 - linear_method: Optional[LinearMethodBase] = None,
93 - ) -> None:
94 - super().__init__()
95 - self.hidden_size = hidden_size
96 - tp_size = get_tensor_model_parallel_world_size()
97 - self.total_num_heads = num_heads
98 - assert self.total_num_heads % tp_size == 0
99 - self.num_heads = self.total_num_heads // tp_size
100 - self.total_num_kv_heads = num_kv_heads
101 - if self.total_num_kv_heads >= tp_size:
102 - # Number of KV heads is greater than TP size, so we partition
103 - # the KV heads across multiple tensor parallel GPUs.
104 - assert self.total_num_kv_heads % tp_size == 0
105 - else:
106 - # Number of KV heads is less than TP size, so we replicate
107 - # the KV heads across multiple tensor parallel GPUs.
108 - assert tp_size % self.total_num_kv_heads == 0
109 - self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
110 - self.head_dim = hidden_size // self.total_num_heads
111 - self.q_size = self.num_heads * self.head_dim
112 - self.kv_size = self.num_kv_heads * self.head_dim
113 - self.scaling = self.head_dim**-0.5
114 - self.rope_theta = rope_theta
115 - self.max_position_embeddings = max_position_embeddings
116 -
117 - self.qkv_proj = QKVParallelLinear(
118 - hidden_size,
119 - self.head_dim,
120 - self.total_num_heads,
121 - self.total_num_kv_heads,
122 - bias=False,
123 - linear_method=linear_method,
124 - )
125 - self.o_proj = RowParallelLinear(
126 - self.total_num_heads * self.head_dim,
127 - hidden_size,
128 - bias=False,
129 - linear_method=linear_method,
130 - )
131 - self.rotary_emb = get_rope(
132 - self.head_dim,
133 - rotary_dim=self.head_dim,
134 - max_position=max_position_embeddings,
135 - base=self.rope_theta,
136 - rope_scaling=rope_scaling,
137 - )
138 - self.attn = PagedAttention(self.num_heads,
139 - self.head_dim,
140 - self.scaling,
141 - num_kv_heads=self.num_kv_heads)
142 -
143 - def forward(
144 - self,
145 - positions: torch.Tensor,
146 - hidden_states: torch.Tensor,
147 - kv_cache: KVCache,
148 - input_metadata: InputMetadata,
149 - ) -> torch.Tensor:
150 - qkv, _ = self.qkv_proj(hidden_states)
151 - q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
152 - q, k = self.rotary_emb(positions, q, k)
153 - k_cache, v_cache = kv_cache
154 - attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
155 - output, _ = self.o_proj(attn_output)
156 - return output
157 -
158 -
159 -class YiDecoderLayer(nn.Module):
160 -
161 - def __init__(
162 - self,
163 - config: YiConfig,
164 - linear_method: Optional[LinearMethodBase] = None,
165 - ) -> None:
166 - super().__init__()
167 - self.hidden_size = config.hidden_size
168 - rope_theta = getattr(config, "rope_theta", 10000)
169 - rope_scaling = getattr(config, "rope_scaling", None)
170 - max_position_embeddings = getattr(config, "max_position_embeddings",
171 - 8192)
172 - self.self_attn = YiAttention(
173 - hidden_size=self.hidden_size,
174 - num_heads=config.num_attention_heads,
175 - num_kv_heads=config.num_key_value_heads,
176 - rope_theta=rope_theta,
177 - rope_scaling=rope_scaling,
178 - max_position_embeddings=max_position_embeddings,
179 - linear_method=linear_method,
180 - )
181 - self.mlp = YiMLP(
182 - hidden_size=self.hidden_size,
183 - intermediate_size=config.intermediate_size,
184 - hidden_act=config.hidden_act,
185 - linear_method=linear_method,
186 - )
187 - self.ln1 = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
188 - self.ln2 = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
189 -
190 - def forward(
191 - self,
192 - positions: torch.Tensor,
193 - hidden_states: torch.Tensor,
194 - kv_cache: KVCache,
195 - input_metadata: InputMetadata,
196 - residual: Optional[torch.Tensor],
197 - ) -> Tuple[torch.Tensor, torch.Tensor]:
198 - # Self Attention
199 - if residual is None:
200 - residual = hidden_states
201 - hidden_states = self.ln1(hidden_states)
202 - else:
203 - hidden_states, residual = self.ln1(hidden_states, residual)
204 - hidden_states = self.self_attn(
205 - positions=positions,
206 - hidden_states=hidden_states,
207 - kv_cache=kv_cache,
208 - input_metadata=input_metadata,
209 - )
210 -
211 - # Fully Connected
212 - hidden_states, residual = self.ln2(hidden_states, residual)
213 - hidden_states = self.mlp(hidden_states)
214 - return hidden_states, residual
215 -
216 -
217 -class YiModel(nn.Module):
218 -
219 - def __init__(
220 - self,
221 - config: YiConfig,
222 - linear_method: Optional[LinearMethodBase] = None,
223 - ) -> None:
224 - super().__init__()
225 - self.config = config
226 - self.padding_idx = config.pad_token_id
227 - self.vocab_size = config.vocab_size
228 - self.embed_tokens = VocabParallelEmbedding(
229 - config.vocab_size,
230 - config.hidden_size,
231 - )
232 - self.layers = nn.ModuleList([
233 - YiDecoderLayer(config, linear_method)
234 - for _ in range(config.num_hidden_layers)
235 - ])
236 - self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
237 -
238 - def forward(
239 - self,
240 - input_ids: torch.Tensor,
241 - positions: torch.Tensor,
242 - kv_caches: List[KVCache],
243 - input_metadata: InputMetadata,
244 - ) -> torch.Tensor:
245 - hidden_states = self.embed_tokens(input_ids)
246 - residual = None
247 - for i in range(len(self.layers)):
248 - layer = self.layers[i]
249 - hidden_states, residual = layer(
250 - positions,
251 - hidden_states,
252 - kv_caches[i],
253 - input_metadata,
254 - residual,
255 - )
256 - hidden_states, _ = self.norm(hidden_states, residual)
257 - return hidden_states
258 -
259 -
260 -class YiForCausalLM(nn.Module):
261 -
262 - def __init__(
263 - self,
264 - config: YiConfig,
265 - linear_method: Optional[LinearMethodBase] = None,
266 - ) -> None:
267 - super().__init__()
268 - self.config = config
269 - self.linear_method = linear_method
270 - self.model = YiModel(config, linear_method)
271 - self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
272 - self.sampler = Sampler(config.vocab_size)
273 -
274 - def forward(
275 - self,
276 - input_ids: torch.Tensor,
277 - positions: torch.Tensor,
278 - kv_caches: List[KVCache],
279 - input_metadata: InputMetadata,
280 - ) -> torch.Tensor:
281 - hidden_states = self.model(input_ids, positions, kv_caches,
282 - input_metadata)
283 - return hidden_states
284 -
285 - def sample(
286 - self,
287 - hidden_states: torch.Tensor,
288 - sampling_metadata: SamplingMetadata,
289 - ) -> Optional[SamplerOutput]:
290 - next_tokens = self.sampler(self.lm_head.weight, hidden_states,
291 - sampling_metadata)
292 - return next_tokens
293 -
294 - def load_weights(self,
295 - model_name_or_path: str,
296 - cache_dir: Optional[str] = None,
297 - load_format: str = "auto",
298 - revision: Optional[str] = None):
299 - stacked_params_mapping = [
300 - # (param_name, shard_name, shard_id)
301 - ("qkv_proj", "q_proj", "q"),
302 - ("qkv_proj", "k_proj", "k"),
303 - ("qkv_proj", "v_proj", "v"),
304 - ("gate_up_proj", "gate_proj", 0),
305 - ("gate_up_proj", "up_proj", 1),
306 - ]
307 - params_dict = dict(self.named_parameters())
308 - for name, loaded_weight in hf_model_weights_iterator(
309 - model_name_or_path, cache_dir, load_format, revision):
310 - if "rotary_emb.inv_freq" in name:
311 - continue
312 - for (param_name, weight_name, shard_id) in stacked_params_mapping:
313 - if weight_name not in name:
314 - continue
315 - name = name.replace(weight_name, param_name)
316 - # Skip loading extra bias for GPTQ models.
317 - if name.endswith(".bias") and name not in params_dict:
318 - continue
319 - param = params_dict[name]
320 - weight_loader = param.weight_loader
321 - weight_loader(param, loaded_weight, shard_id)
322 - break
323 - else:
324 - # Skip loading extra bias for GPTQ models.
325 - if name.endswith(".bias") and name not in params_dict:
326 - continue
327 - param = params_dict[name]
328 - weight_loader = getattr(param, "weight_loader",
329 - default_weight_loader)
330 - weight_loader(param, loaded_weight)
vllm/transformers_utils/config.py
-4
@@ -5,14 +5,10 @@ from transformers import AutoConfig, PretrainedConfig
5 from vllm.transformers_utils.configs import *
6
7 _CONFIG_REGISTRY = {
8 - "aquila": AquilaConfig,
9 - "baichuan": BaiChuanConfig,
8 "chatglm": ChatGLMConfig,
9 "mpt": MPTConfig,
12 - "qwen": QWenConfig,
10 "RefinedWeb": RWConfig, # For tiiuae/falcon-40b(-instruct)
11 "RefinedWebModel": RWConfig, # For tiiuae/falcon-7b(-instruct)
15 - "yi": YiConfig,
12 }
13
14
vllm/transformers_utils/configs/__init__.py
-8
@@ -1,20 +1,12 @@
1 -from vllm.transformers_utils.configs.aquila import AquilaConfig
2 -from vllm.transformers_utils.configs.baichuan import BaiChuanConfig
1 from vllm.transformers_utils.configs.chatglm import ChatGLMConfig
2 from vllm.transformers_utils.configs.mpt import MPTConfig
5 -from vllm.transformers_utils.configs.qwen import QWenConfig
3 # RWConfig is for the original tiiuae/falcon-40b(-instruct) and
4 # tiiuae/falcon-7b(-instruct) models. Newer Falcon models will use the
5 # `FalconConfig` class from the official HuggingFace transformers library.
6 from vllm.transformers_utils.configs.falcon import RWConfig
10 -from vllm.transformers_utils.configs.yi import YiConfig
7
8 __all__ = [
13 - "AquilaConfig",
14 - "BaiChuanConfig",
9 "ChatGLMConfig",
10 "MPTConfig",
17 - "QWenConfig",
11 "RWConfig",
19 - "YiConfig",
12 ]
vllm/transformers_utils/configs/aquila.py deleted
-69
@@ -1,69 +0,0 @@
1 -# coding=utf-8
2 -# Copyright 2023 EleutherAI and the HuggingFace Inc. team. All rights reserved.
3 -#
4 -# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
5 -# and OPT implementations in this library. It has been modified from its
6 -# original forms to accommodate minor architectural differences compared
7 -# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
8 -#
9 -# Licensed under the Apache License, Version 2.0 (the "License");
10 -# you may not use this file except in compliance with the License.
11 -# You may obtain a copy of the License at
12 -#
13 -# http://www.apache.org/licenses/LICENSE-2.0
14 -#
15 -# Unless required by applicable law or agreed to in writing, software
16 -# distributed under the License is distributed on an "AS IS" BASIS,
17 -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18 -# See the License for the specific language governing permissions and
19 -# limitations under the License.
20 -""" Aquila model configuration"""
21 -
22 -from transformers import PretrainedConfig
23 -
24 -
25 -class AquilaConfig(PretrainedConfig):
26 - model_type = "aquila"
27 - keys_to_ignore_at_inference = ["past_key_values"]
28 -
29 - def __init__(
30 - self,
31 - vocab_size=100008,
32 - hidden_size=4096,
33 - intermediate_size=11008,
34 - num_hidden_layers=32,
35 - num_attention_heads=32,
36 - num_key_value_heads=None,
37 - hidden_act="silu",
38 - max_position_embeddings=2048,
39 - initializer_range=0.006,
40 - rms_norm_eps=1e-5,
41 - use_cache=True,
42 - pad_token_id=0,
43 - bos_token_id=1,
44 - eos_token_id=2,
45 - tie_word_embeddings=False,
46 - **kwargs,
47 - ):
48 - self.vocab_size = vocab_size
49 - self.max_position_embeddings = max_position_embeddings
50 - self.hidden_size = hidden_size
51 - self.intermediate_size = intermediate_size
52 - self.num_hidden_layers = num_hidden_layers
53 - # for backward compatibility
54 - if num_key_value_heads is None:
55 - num_key_value_heads = num_attention_heads
56 -
57 - self.num_key_value_heads = num_key_value_heads
58 - self.num_attention_heads = num_attention_heads
59 - self.hidden_act = hidden_act
60 - self.initializer_range = initializer_range
61 - self.rms_norm_eps = rms_norm_eps
62 - self.use_cache = use_cache
63 - super().__init__(
64 - pad_token_id=pad_token_id,
65 - bos_token_id=bos_token_id,
66 - eos_token_id=eos_token_id,
67 - tie_word_embeddings=tie_word_embeddings,
68 - **kwargs,
69 - )
vllm/transformers_utils/configs/baichuan.py deleted
-62
@@ -1,62 +0,0 @@
1 -# coding=utf-8
2 -# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
3 -#
4 -# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
5 -# and OPT implementations in this library. It has been modified from its
6 -# original forms to accommodate minor architectural differences compared
7 -# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
8 -#
9 -# Licensed under the Apache License, Version 2.0 (the "License");
10 -# you may not use this file except in compliance with the License.
11 -# You may obtain a copy of the License at
12 -#
13 -# http://www.apache.org/licenses/LICENSE-2.0
14 -#
15 -# Unless required by applicable law or agreed to in writing, software
16 -# distributed under the License is distributed on an "AS IS" BASIS,
17 -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18 -# See the License for the specific language governing permissions and
19 -# limitations under the License.
20 -
21 -from transformers.configuration_utils import PretrainedConfig
22 -
23 -
24 -class BaiChuanConfig(PretrainedConfig):
25 - model_type = "baichuan"
26 - keys_to_ignore_at_inference = ["past_key_values"]
27 -
28 - def __init__(
29 - self,
30 - vocab_size=64000,
31 - hidden_size=4096,
32 - intermediate_size=11008,
33 - num_hidden_layers=32,
34 - num_attention_heads=32,
35 - hidden_act="silu",
36 - max_position_embeddings=4096,
37 - initializer_range=0.02,
38 - rms_norm_eps=1e-6,
39 - use_cache=True,
40 - pad_token_id=0,
41 - bos_token_id=1,
42 - eos_token_id=2,
43 - tie_word_embeddings=False,
44 - **kwargs,
45 - ):
46 - self.vocab_size = vocab_size
47 - self.max_position_embeddings = max_position_embeddings
48 - self.hidden_size = hidden_size
49 - self.intermediate_size = intermediate_size
50 - self.num_hidden_layers = num_hidden_layers
51 - self.num_attention_heads = num_attention_heads
52 - self.hidden_act = hidden_act
53 - self.initializer_range = initializer_range
54 - self.rms_norm_eps = rms_norm_eps
55 - self.use_cache = use_cache
56 - super().__init__(
57 - pad_token_id=pad_token_id,
58 - bos_token_id=bos_token_id,
59 - eos_token_id=eos_token_id,
60 - tie_word_embeddings=tie_word_embeddings,
61 - **kwargs,
62 - )
vllm/transformers_utils/configs/qwen.py deleted
-60
@@ -1,60 +0,0 @@
1 -# Copyright (c) Alibaba Cloud.
2 -# LICENSE: https://huggingface.co/Qwen/Qwen-7B/blob/main/LICENSE
3 -
4 -from transformers import PretrainedConfig
5 -
6 -
7 -class QWenConfig(PretrainedConfig):
8 - model_type = "qwen"
9 - keys_to_ignore_at_inference = ["past_key_values"]
10 -
11 - def __init__(
12 - self,
13 - vocab_size=151936,
14 - hidden_size=4096,
15 - num_hidden_layers=32,
16 - num_attention_heads=32,
17 - emb_dropout_prob=0.0,
18 - attn_dropout_prob=0.0,
19 - layer_norm_epsilon=1e-6,
20 - initializer_range=0.02,
21 - max_position_embeddings=8192,
22 - scale_attn_weights=True,
23 - use_cache=True,
24 - bf16=False,
25 - fp16=False,
26 - fp32=False,
27 - kv_channels=128,
28 - rotary_pct=1.0,
29 - rotary_emb_base=10000,
30 - use_dynamic_ntk=True,
31 - use_logn_attn=True,
32 - use_flash_attn="auto",
33 - intermediate_size=22016,
34 - no_bias=True,
35 - tie_word_embeddings=False,
36 - **kwargs,
37 - ):
38 - self.vocab_size = vocab_size
39 - self.hidden_size = hidden_size
40 - self.intermediate_size = intermediate_size
41 - self.num_hidden_layers = num_hidden_layers
42 - self.num_attention_heads = num_attention_heads
43 - self.emb_dropout_prob = emb_dropout_prob
44 - self.attn_dropout_prob = attn_dropout_prob
45 - self.layer_norm_epsilon = layer_norm_epsilon
46 - self.initializer_range = initializer_range
47 - self.scale_attn_weights = scale_attn_weights
48 - self.use_cache = use_cache
49 - self.max_position_embeddings = max_position_embeddings
50 - self.bf16 = bf16
51 - self.fp16 = fp16
52 - self.fp32 = fp32
53 - self.kv_channels = kv_channels
54 - self.rotary_pct = rotary_pct
55 - self.rotary_emb_base = rotary_emb_base
56 - self.use_dynamic_ntk = use_dynamic_ntk
57 - self.use_logn_attn = use_logn_attn
58 - self.use_flash_attn = use_flash_attn
59 - self.no_bias = no_bias
60 - super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)
vllm/transformers_utils/configs/yi.py deleted
-64
@@ -1,64 +0,0 @@
1 -""" Yi model configuration"""
2 -from transformers.configuration_utils import PretrainedConfig
3 -from transformers.utils import logging
4 -
5 -logger = logging.get_logger(__name__)
6 -
7 -Yi_PRETRAINED_CONFIG_ARCHIVE_MAP = {}
8 -
9 -
10 -class YiConfig(PretrainedConfig):
11 - r"""
12 - Reference:
13 - https://huggingface.co/01-ai/Yi-6B/blob/main/configuration_yi.py
14 - """
15 - model_type = "Yi"
16 - keys_to_ignore_at_inference = ["past_key_values"]
17 -
18 - def __init__(
19 - self,
20 - vocab_size=64000,
21 - hidden_size=4096,
22 - intermediate_size=11008,
23 - num_hidden_layers=32,
24 - num_attention_heads=32,
25 - num_key_value_heads=4,
26 - hidden_act="silu",
27 - max_position_embeddings=4096,
28 - initializer_range=0.02,
29 - rms_norm_eps=1e-5,
30 - use_cache=True,
31 - pad_token_id=0,
32 - bos_token_id=1,
33 - eos_token_id=2,
34 - tie_word_embeddings=False,
35 - output_attentions=False,
36 - rope_theta=5000000.0,
37 - **kwargs,
38 - ):
39 - self.vocab_size = vocab_size
40 - self.max_position_embeddings = max_position_embeddings
41 - self.hidden_size = hidden_size
42 - self.intermediate_size = intermediate_size
43 - self.num_hidden_layers = num_hidden_layers
44 - self.num_attention_heads = num_attention_heads
45 -
46 - # for backward compatibility
47 - if num_key_value_heads is None:
48 - num_key_value_heads = num_attention_heads
49 -
50 - self.num_key_value_heads = num_key_value_heads
51 - self.hidden_act = hidden_act
52 - self.initializer_range = initializer_range
53 - self.rms_norm_eps = rms_norm_eps
54 - self.use_cache = use_cache
55 - self.output_attentions = output_attentions
56 - self.rope_theta = rope_theta
57 -
58 - super().__init__(
59 - pad_token_id=pad_token_id,
60 - bos_token_id=bos_token_id,
61 - eos_token_id=eos_token_id,
62 - tie_word_embeddings=tie_word_embeddings,
63 - **kwargs,
64 - )