A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Refactor llama family models (#2637)
Roy committed
Feb 13, 2024 at 16:09 UTC
5c976a7e1a1bec875bf6474824b7dff39e38de18
17 files changed
+236
-2720
vllm/model_executor/layers/layernorm.py
+25
@@ -7,6 +7,31 @@ import torch.nn as nn
7
from vllm._C import ops
8
9
10
+class LayerNorm(nn.LayerNorm):
11
+
12
+ def __init__(
13
+ self,
14
+ hidden_size: int,
15
+ eps: float = 1e-6,
16
+ ) -> None:
17
+ super().__init__(hidden_size, eps=eps)
18
+
19
+ def forward(
20
+ self,
21
+ x: torch.Tensor,
22
+ residual: Optional[torch.Tensor] = None,
23
+ ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
24
+ """normalization."""
25
+ if residual is not None:
26
+ x = x + residual
27
+ residual = x
28
+ x = super().forward(x)
29
+ if residual is None:
30
+ return x
31
+ else:
32
+ return x, residual
33
+
34
+
35
class RMSNorm(nn.Module):
36
"""Root mean square normalization.
37
vllm/model_executor/models/__init__.py
+4
-5
@@ -10,8 +10,8 @@ logger = init_logger(__name__)
10
11
# Architecture -> (module, class).
12
_MODELS = {
13
- "AquilaModel": ("aquila", "AquilaForCausalLM"),
14
- "AquilaForCausalLM": ("aquila", "AquilaForCausalLM"), # AquilaChat2
13
+ "AquilaModel": ("llama", "LlamaForCausalLM"),
14
+ "AquilaForCausalLM": ("llama", "LlamaForCausalLM"), # AquilaChat2
15
"BaiChuanForCausalLM": ("baichuan", "BaiChuanForCausalLM"), # baichuan-7b
16
"BaichuanForCausalLM": ("baichuan", "BaichuanForCausalLM"), # baichuan-13b
17
"BloomForCausalLM": ("bloom", "BloomForCausalLM"),
@@ -24,12 +24,12 @@ _MODELS = {
24
"GPTBigCodeForCausalLM": ("gpt_bigcode", "GPTBigCodeForCausalLM"),
25
"GPTJForCausalLM": ("gpt_j", "GPTJForCausalLM"),
26
"GPTNeoXForCausalLM": ("gpt_neox", "GPTNeoXForCausalLM"),
27
- "InternLMForCausalLM": ("internlm", "InternLMForCausalLM"),
27
+ "InternLMForCausalLM": ("llama", "LlamaForCausalLM"),
28
"InternLM2ForCausalLM": ("internlm2", "InternLM2ForCausalLM"),
29
"LlamaForCausalLM": ("llama", "LlamaForCausalLM"),
30
# For decapoda-research/llama-*
31
"LLaMAForCausalLM": ("llama", "LlamaForCausalLM"),
32
- "MistralForCausalLM": ("mistral", "MistralForCausalLM"),
32
+ "MistralForCausalLM": ("llama", "LlamaForCausalLM"),
33
"MixtralForCausalLM": ("mixtral", "MixtralForCausalLM"),
34
"QuantMixtralForCausalLM": ("mixtral_quant", "MixtralForCausalLM"),
35
# transformers's mpt class has lower case
@@ -41,7 +41,6 @@ _MODELS = {
41
"Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
42
"RWForCausalLM": ("falcon", "FalconForCausalLM"),
43
"StableLMEpochForCausalLM": ("stablelm", "StablelmForCausalLM"),
44
- "YiForCausalLM": ("yi", "YiForCausalLM")
44
}
45
46
# Models not supported by ROCm.
vllm/model_executor/models/aquila.py
deleted
-342
@@ -1,342 +0,0 @@
1
-# coding=utf-8
2
-# Adapted from
3
-# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
4
-# Copyright 2023 The vLLM team.
5
-# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
6
-#
7
-# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
8
-# and OPT implementations in this library. It has been modified from its
9
-# original forms to accommodate minor architectural differences compared
10
-# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
11
-#
12
-# Licensed under the Apache License, Version 2.0 (the "License");
13
-# you may not use this file except in compliance with the License.
14
-# You may obtain a copy of the License at
15
-#
16
-# http://www.apache.org/licenses/LICENSE-2.0
17
-#
18
-# Unless required by applicable law or agreed to in writing, software
19
-# distributed under the License is distributed on an "AS IS" BASIS,
20
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21
-# See the License for the specific language governing permissions and
22
-# limitations under the License.
23
-"""Inference-only LLaMA model compatible with HuggingFace weights."""
24
-from typing import Any, Dict, List, Optional, Tuple
25
-
26
-import torch
27
-from torch import nn
28
-
29
-from vllm.model_executor.input_metadata import InputMetadata
30
-from vllm.model_executor.layers.activation import SiluAndMul
31
-from vllm.model_executor.layers.attention import PagedAttention
32
-from vllm.model_executor.layers.linear import (LinearMethodBase,
33
- MergedColumnParallelLinear,
34
- QKVParallelLinear,
35
- RowParallelLinear)
36
-from vllm.model_executor.layers.rotary_embedding import get_rope
37
-from vllm.model_executor.layers.sampler import Sampler
38
-from vllm.model_executor.layers.vocab_parallel_embedding import (
39
- VocabParallelEmbedding, ParallelLMHead)
40
-from vllm.model_executor.parallel_utils.parallel_state import (
41
- get_tensor_model_parallel_world_size)
42
-from vllm.model_executor.sampling_metadata import SamplingMetadata
43
-from vllm.model_executor.weight_utils import (default_weight_loader,
44
- hf_model_weights_iterator)
45
-from vllm.sequence import SamplerOutput
46
-from vllm.transformers_utils.configs.aquila import AquilaConfig
47
-
48
-KVCache = Tuple[torch.Tensor, torch.Tensor]
49
-
50
-
51
-class AquilaMLP(nn.Module):
52
-
53
- def __init__(
54
- self,
55
- hidden_size: int,
56
- intermediate_size: int,
57
- hidden_act: str,
58
- linear_method: Optional[LinearMethodBase] = None,
59
- ):
60
- super().__init__()
61
- self.gate_up_proj = MergedColumnParallelLinear(
62
- hidden_size, [intermediate_size] * 2,
63
- bias=False,
64
- linear_method=linear_method)
65
- self.down_proj = RowParallelLinear(intermediate_size,
66
- hidden_size,
67
- bias=False,
68
- linear_method=linear_method)
69
- if hidden_act != "silu":
70
- raise ValueError(f"Unsupported activation: {hidden_act}. "
71
- "Only silu is supported for now.")
72
- self.act_fn = SiluAndMul()
73
-
74
- def forward(self, x):
75
- gate_up, _ = self.gate_up_proj(x)
76
- x = self.act_fn(gate_up)
77
- x, _ = self.down_proj(x)
78
- return x
79
-
80
-
81
-class AquilaRMSNorm(nn.Module):
82
-
83
- def __init__(self, hidden_size, eps=1e-6):
84
- """
85
- AquilaRMSNorm is equivalent to T5LayerNorm
86
- """
87
- super().__init__()
88
- self.weight = nn.Parameter(torch.ones(hidden_size))
89
- self.variance_epsilon = eps
90
-
91
- def forward(self, hidden_states):
92
- input_dtype = hidden_states.dtype
93
- variance = hidden_states.to(torch.float32).pow(2).mean(-1,
94
- keepdim=True)
95
- hidden_states = hidden_states * torch.rsqrt(variance +
96
- self.variance_epsilon)
97
-
98
- return (self.weight * hidden_states).to(input_dtype)
99
-
100
-
101
-class AquilaAttention(nn.Module):
102
-
103
- def __init__(
104
- self,
105
- hidden_size: int,
106
- num_heads: int,
107
- num_kv_heads: int,
108
- rope_theta: float = 10000,
109
- max_position_embeddings: int = 8192,
110
- rope_scaling: Optional[Dict[str, Any]] = None,
111
- linear_method: Optional[LinearMethodBase] = None,
112
- ):
113
- super().__init__()
114
- self.hidden_size = hidden_size
115
- tp_size = get_tensor_model_parallel_world_size()
116
- self.total_num_heads = num_heads
117
- assert self.total_num_heads % tp_size == 0
118
- self.num_heads = self.total_num_heads // tp_size
119
- self.total_num_kv_heads = num_kv_heads
120
- assert self.total_num_kv_heads % tp_size == 0
121
- self.num_kv_heads = self.total_num_kv_heads // tp_size
122
- self.head_dim = hidden_size // self.total_num_heads
123
- self.q_size = self.num_heads * self.head_dim
124
- self.kv_size = self.num_kv_heads * self.head_dim
125
- self.scaling = self.head_dim**-0.5
126
- self.rope_theta = rope_theta
127
- self.max_position_embeddings = max_position_embeddings
128
-
129
- self.qkv_proj = QKVParallelLinear(
130
- hidden_size,
131
- self.head_dim,
132
- self.total_num_heads,
133
- self.total_num_kv_heads,
134
- bias=False,
135
- linear_method=linear_method,
136
- )
137
- self.o_proj = RowParallelLinear(
138
- self.total_num_heads * self.head_dim,
139
- hidden_size,
140
- bias=False,
141
- linear_method=linear_method,
142
- )
143
- self.rotary_emb = get_rope(
144
- self.head_dim,
145
- rotary_dim=self.head_dim,
146
- max_position=self.max_position_embeddings,
147
- base=self.rope_theta,
148
- rope_scaling=rope_scaling,
149
- )
150
- self.attn = PagedAttention(self.num_heads,
151
- self.head_dim,
152
- self.scaling,
153
- num_kv_heads=self.num_kv_heads)
154
-
155
- def forward(
156
- self,
157
- positions: torch.Tensor,
158
- hidden_states: torch.Tensor,
159
- kv_cache: KVCache,
160
- input_metadata: InputMetadata,
161
- ) -> torch.Tensor:
162
- qkv, _ = self.qkv_proj(hidden_states)
163
- q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
164
- q, k = self.rotary_emb(positions, q, k)
165
- k_cache, v_cache = kv_cache
166
- attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
167
- output, _ = self.o_proj(attn_output)
168
- return output
169
-
170
-
171
-class AquilaDecoderLayer(nn.Module):
172
-
173
- def __init__(
174
- self,
175
- config: AquilaConfig,
176
- linear_method: Optional[LinearMethodBase] = None,
177
- ):
178
- super().__init__()
179
- self.hidden_size = config.hidden_size
180
- rope_theta = getattr(config, "rope_theta", 10000)
181
- rope_scaling = getattr(config, "rope_scaling", None)
182
- max_position_embeddings = getattr(config, "max_position_embeddings",
183
- 8192)
184
- self.self_attn = AquilaAttention(
185
- hidden_size=self.hidden_size,
186
- num_heads=config.num_attention_heads,
187
- num_kv_heads=config.num_key_value_heads,
188
- rope_theta=rope_theta,
189
- max_position_embeddings=max_position_embeddings,
190
- rope_scaling=rope_scaling,
191
- linear_method=linear_method,
192
- )
193
- self.mlp = AquilaMLP(
194
- hidden_size=self.hidden_size,
195
- intermediate_size=config.intermediate_size,
196
- hidden_act=config.hidden_act,
197
- linear_method=linear_method,
198
- )
199
- self.input_layernorm = AquilaRMSNorm(config.hidden_size,
200
- eps=config.rms_norm_eps)
201
- self.post_attention_layernorm = AquilaRMSNorm(config.hidden_size,
202
- eps=config.rms_norm_eps)
203
-
204
- def forward(
205
- self,
206
- positions: torch.Tensor,
207
- hidden_states: torch.Tensor,
208
- kv_cache: KVCache,
209
- input_metadata: InputMetadata,
210
- ) -> torch.Tensor:
211
- # Self Attention
212
- residual = hidden_states
213
- hidden_states = self.input_layernorm(hidden_states)
214
- hidden_states = self.self_attn(
215
- positions=positions,
216
- hidden_states=hidden_states,
217
- kv_cache=kv_cache,
218
- input_metadata=input_metadata,
219
- )
220
- hidden_states = residual + hidden_states
221
-
222
- # Fully Connected
223
- residual = hidden_states
224
- hidden_states = self.post_attention_layernorm(hidden_states)
225
- hidden_states = self.mlp(hidden_states)
226
- hidden_states = residual + hidden_states
227
- return hidden_states
228
-
229
-
230
-class AquilaModel(nn.Module):
231
-
232
- def __init__(
233
- self,
234
- config: AquilaConfig,
235
- linear_method: Optional[LinearMethodBase] = None,
236
- ):
237
- super().__init__()
238
- self.config = config
239
- self.padding_idx = config.pad_token_id
240
- self.vocab_size = config.vocab_size
241
- self.embed_tokens = VocabParallelEmbedding(
242
- config.vocab_size,
243
- config.hidden_size,
244
- )
245
- self.layers = nn.ModuleList([
246
- AquilaDecoderLayer(config, linear_method)
247
- for _ in range(config.num_hidden_layers)
248
- ])
249
- self.norm = AquilaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
250
-
251
- def forward(
252
- self,
253
- input_ids: torch.Tensor,
254
- positions: torch.Tensor,
255
- kv_caches: List[KVCache],
256
- input_metadata: InputMetadata,
257
- ) -> torch.Tensor:
258
- hidden_states = self.embed_tokens(input_ids)
259
- for i in range(len(self.layers)):
260
- layer = self.layers[i]
261
- hidden_states = layer(
262
- positions,
263
- hidden_states,
264
- kv_caches[i],
265
- input_metadata,
266
- )
267
- hidden_states = self.norm(hidden_states)
268
-
269
- return hidden_states
270
-
271
-
272
-class AquilaForCausalLM(nn.Module):
273
-
274
- def __init__(
275
- self,
276
- config,
277
- linear_method: Optional[LinearMethodBase] = None,
278
- ):
279
- super().__init__()
280
- self.config = config
281
- self.linear_method = linear_method
282
- self.model = AquilaModel(config, linear_method)
283
- self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
284
- self.sampler = Sampler(config.vocab_size)
285
-
286
- def forward(
287
- self,
288
- input_ids: torch.Tensor,
289
- positions: torch.Tensor,
290
- kv_caches: List[KVCache],
291
- input_metadata: InputMetadata,
292
- ) -> torch.Tensor:
293
- hidden_states = self.model(input_ids, positions, kv_caches,
294
- input_metadata)
295
- return hidden_states
296
-
297
- def sample(
298
- self,
299
- hidden_states: torch.Tensor,
300
- sampling_metadata: SamplingMetadata,
301
- ) -> Optional[SamplerOutput]:
302
- next_tokens = self.sampler(self.lm_head.weight, hidden_states,
303
- sampling_metadata)
304
- return next_tokens
305
-
306
- def load_weights(self,
307
- model_name_or_path: str,
308
- cache_dir: Optional[str] = None,
309
- load_format: str = "auto",
310
- revision: Optional[str] = None):
311
- stacked_params_mapping = [
312
- # (param_name, shard_name, shard_id)
313
- ("qkv_proj", "q_proj", "q"),
314
- ("qkv_proj", "k_proj", "k"),
315
- ("qkv_proj", "v_proj", "v"),
316
- ("gate_up_proj", "gate_proj", 0),
317
- ("gate_up_proj", "up_proj", 1),
318
- ]
319
- params_dict = dict(self.named_parameters())
320
- for name, loaded_weight in hf_model_weights_iterator(
321
- model_name_or_path, cache_dir, load_format, revision):
322
- if "rotary_emb.inv_freq" in name:
323
- continue
324
- for (param_name, weight_name, shard_id) in stacked_params_mapping:
325
- if weight_name not in name:
326
- continue
327
- name = name.replace(weight_name, param_name)
328
- # Skip loading extra bias for GPTQ models.
329
- if name.endswith(".bias") and name not in params_dict:
330
- continue
331
- param = params_dict[name]
332
- weight_loader = param.weight_loader
333
- weight_loader(param, loaded_weight, shard_id)
334
- break
335
- else:
336
- # Skip loading extra bias for GPTQ models.
337
- if name.endswith(".bias") and name not in params_dict:
338
- continue
339
- param = params_dict[name]
340
- weight_loader = getattr(param, "weight_loader",
341
- default_weight_loader)
342
- weight_loader(param, loaded_weight)
vllm/model_executor/models/baichuan.py
+32
-303
@@ -18,305 +18,19 @@
18
# See the License for the specific language governing permissions and
19
# limitations under the License.
20
"""Inference-only BaiChuan model compatible with HuggingFace weights."""
21
-import math
22
-from typing import List, Optional, Tuple
21
+from typing import Optional
22
23
import torch
25
-from torch import nn
24
+from transformers import PretrainedConfig
25
+from vllm.config import LoRAConfig
26
27
-from vllm.model_executor.input_metadata import InputMetadata
28
-from vllm.model_executor.layers.activation import SiluAndMul
29
-from vllm.model_executor.layers.attention import PagedAttention
30
-from vllm.model_executor.layers.layernorm import RMSNorm
31
-from vllm.model_executor.layers.linear import (LinearMethodBase,
32
- MergedColumnParallelLinear,
33
- QKVParallelLinear,
34
- RowParallelLinear)
35
-from vllm.model_executor.layers.rotary_embedding import get_rope
36
-from vllm.model_executor.layers.sampler import Sampler
37
-from vllm.model_executor.layers.vocab_parallel_embedding import (
38
- VocabParallelEmbedding, ParallelLMHead)
39
-from vllm.model_executor.parallel_utils.parallel_state import (
40
- get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
41
-from vllm.model_executor.sampling_metadata import SamplingMetadata
27
+from vllm.model_executor.layers.linear import LinearMethodBase
28
+from vllm.model_executor.models.llama import LlamaForCausalLM
29
from vllm.model_executor.weight_utils import (default_weight_loader,
30
hf_model_weights_iterator)
44
-from vllm.sequence import SamplerOutput
45
-from vllm.transformers_utils.configs.baichuan import BaiChuanConfig
31
47
-KVCache = Tuple[torch.Tensor, torch.Tensor]
32
49
-
50
-def _get_alibi_slopes(total_num_heads: int) -> torch.Tensor:
51
- closest_power_of_2 = 2**math.floor(math.log2(total_num_heads))
52
- base = torch.tensor(
53
- 2**(-(2**-(math.log2(closest_power_of_2) - 3))),
54
- dtype=torch.float32,
55
- )
56
- powers = torch.arange(1, 1 + closest_power_of_2, dtype=torch.int32)
57
- slopes = torch.pow(base, powers)
58
-
59
- if closest_power_of_2 != total_num_heads:
60
- extra_base = torch.tensor(
61
- 2**(-(2**-(math.log2(2 * closest_power_of_2) - 3))),
62
- dtype=torch.float32,
63
- )
64
- num_remaining_heads = min(closest_power_of_2,
65
- total_num_heads - closest_power_of_2)
66
- extra_powers = torch.arange(start=1,
67
- end=1 + 2 * num_remaining_heads,
68
- step=2,
69
- dtype=torch.int32)
70
- slopes = torch.cat(
71
- [slopes, torch.pow(extra_base, extra_powers)], dim=0)
72
- return slopes
73
-
74
-
75
-class BaiChuanMLP(nn.Module):
76
-
77
- def __init__(
78
- self,
79
- hidden_size: int,
80
- intermediate_size: int,
81
- hidden_act: str,
82
- linear_method: Optional[LinearMethodBase] = None,
83
- ):
84
- super().__init__()
85
- self.gate_up_proj = MergedColumnParallelLinear(
86
- hidden_size, [intermediate_size] * 2,
87
- bias=False,
88
- linear_method=linear_method)
89
- self.down_proj = RowParallelLinear(intermediate_size,
90
- hidden_size,
91
- bias=False,
92
- linear_method=linear_method)
93
- if hidden_act != "silu":
94
- raise ValueError(f"Unsupported activation: {hidden_act}. "
95
- "Only silu is supported for now.")
96
- self.act_fn = SiluAndMul()
97
-
98
- def forward(self, x):
99
- gate_up, _ = self.gate_up_proj(x)
100
- x = self.act_fn(gate_up)
101
- x, _ = self.down_proj(x)
102
- return x
103
-
104
-
105
-class BaiChuanAttention(nn.Module):
106
- """Multi-headed attention from 'Attention Is All You Need' paper"""
107
-
108
- def __init__(
109
- self,
110
- hidden_size: int,
111
- num_heads: int,
112
- position_embedding: str,
113
- rope_theta: float = 10000,
114
- max_position_embeddings: int = 8192,
115
- linear_method: Optional[LinearMethodBase] = None,
116
- ):
117
- super().__init__()
118
- self.hidden_size = hidden_size
119
- tensor_model_parallel_world_size = get_tensor_model_parallel_world_size(
120
- )
121
- self.total_num_heads = num_heads
122
- assert self.total_num_heads % tensor_model_parallel_world_size == 0
123
- self.num_heads = (self.total_num_heads //
124
- tensor_model_parallel_world_size)
125
- self.head_dim = hidden_size // self.total_num_heads
126
- self.postion_embedding = position_embedding
127
- self.rope_theta = rope_theta
128
- self.max_position_embeddings = max_position_embeddings
129
-
130
- # pylint: disable=invalid-name
131
- self.W_pack = QKVParallelLinear(
132
- hidden_size,
133
- self.head_dim,
134
- self.total_num_heads,
135
- self.total_num_heads,
136
- bias=False,
137
- linear_method=linear_method,
138
- )
139
- self.o_proj = RowParallelLinear(
140
- self.total_num_heads * self.head_dim,
141
- hidden_size,
142
- bias=False,
143
- linear_method=linear_method,
144
- )
145
- # Create the alibi slopes and slice them.
146
- if self.postion_embedding == "ALIBI":
147
- tp_rank = get_tensor_model_parallel_rank()
148
- head_start = tp_rank * self.num_heads
149
- head_end = (tp_rank + 1) * self.num_heads
150
- alibi_slopes = _get_alibi_slopes(self.total_num_heads)
151
- alibi_slopes = alibi_slopes[head_start:head_end].tolist()
152
-
153
- scaling = self.head_dim**-0.5
154
- self.attn = PagedAttention(self.num_heads,
155
- self.head_dim,
156
- scaling,
157
- alibi_slopes=alibi_slopes)
158
- else:
159
- self.rotary_emb = get_rope(
160
- self.head_dim,
161
- rotary_dim=self.head_dim,
162
- max_position=self.max_position_embeddings,
163
- base=self.rope_theta,
164
- )
165
- self.scaling = self.head_dim**-0.5
166
- self.attn = PagedAttention(self.num_heads, self.head_dim,
167
- self.scaling)
168
-
169
- def forward(
170
- self,
171
- positions: torch.Tensor,
172
- hidden_states: torch.Tensor,
173
- kv_cache: KVCache,
174
- input_metadata: InputMetadata,
175
- ) -> torch.Tensor:
176
- qkv, _ = self.W_pack(hidden_states)
177
- q, k, v = qkv.chunk(chunks=3, dim=-1)
178
- if self.postion_embedding != "ALIBI":
179
- q, k = self.rotary_emb(positions, q, k)
180
- k_cache, v_cache = kv_cache
181
- attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
182
- output, _ = self.o_proj(attn_output)
183
- return output
184
-
185
-
186
-class BaiChuanDecoderLayer(nn.Module):
187
-
188
- def __init__(self,
189
- config: BaiChuanConfig,
190
- position_embedding: str,
191
- linear_method: Optional[LinearMethodBase] = None):
192
- super().__init__()
193
- self.hidden_size = config.hidden_size
194
- rope_theta = getattr(config, "rope_theta", 10000)
195
- max_position_embeddings = getattr(config, "max_position_embeddings",
196
- 8192)
197
- self.self_attn = BaiChuanAttention(
198
- hidden_size=self.hidden_size,
199
- num_heads=config.num_attention_heads,
200
- position_embedding=position_embedding,
201
- rope_theta=rope_theta,
202
- max_position_embeddings=max_position_embeddings,
203
- linear_method=linear_method,
204
- )
205
- self.mlp = BaiChuanMLP(
206
- hidden_size=self.hidden_size,
207
- intermediate_size=config.intermediate_size,
208
- hidden_act=config.hidden_act,
209
- linear_method=linear_method,
210
- )
211
- self.input_layernorm = RMSNorm(config.hidden_size,
212
- eps=config.rms_norm_eps)
213
- self.post_attention_layernorm = RMSNorm(config.hidden_size,
214
- eps=config.rms_norm_eps)
215
-
216
- def forward(
217
- self,
218
- positions: torch.Tensor,
219
- hidden_states: torch.Tensor,
220
- kv_cache: KVCache,
221
- input_metadata: InputMetadata,
222
- residual: Optional[torch.Tensor],
223
- ) -> Tuple[torch.Tensor, torch.Tensor]:
224
- # Self Attention
225
- if residual is None:
226
- residual = hidden_states
227
- hidden_states = self.input_layernorm(hidden_states)
228
- else:
229
- hidden_states, residual = self.input_layernorm(
230
- hidden_states, residual)
231
- hidden_states = self.self_attn(
232
- positions=positions,
233
- hidden_states=hidden_states,
234
- kv_cache=kv_cache,
235
- input_metadata=input_metadata,
236
- )
237
-
238
- # Fully Connected
239
- hidden_states, residual = self.post_attention_layernorm(
240
- hidden_states, residual)
241
- hidden_states = self.mlp(hidden_states)
242
- return hidden_states, residual
243
-
244
-
245
-class BaiChuanModel(nn.Module):
246
-
247
- def __init__(self,
248
- config: BaiChuanConfig,
249
- position_embedding: str,
250
- linear_method: Optional[LinearMethodBase] = None):
251
- super().__init__()
252
- self.config = config
253
- self.padding_idx = config.pad_token_id
254
- self.vocab_size = config.vocab_size
255
-
256
- self.embed_tokens = VocabParallelEmbedding(
257
- config.vocab_size,
258
- config.hidden_size,
259
- )
260
- self.layers = nn.ModuleList([
261
- BaiChuanDecoderLayer(config, position_embedding, linear_method)
262
- for _ in range(config.num_hidden_layers)
263
- ])
264
- self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
265
-
266
- def forward(
267
- self,
268
- input_ids: torch.Tensor,
269
- positions: torch.Tensor,
270
- kv_caches: List[KVCache],
271
- input_metadata: InputMetadata,
272
- ) -> torch.Tensor:
273
- hidden_states = self.embed_tokens(input_ids)
274
- residual = None
275
- for i in range(len(self.layers)):
276
- layer = self.layers[i]
277
- hidden_states, residual = layer(
278
- positions,
279
- hidden_states,
280
- kv_caches[i],
281
- input_metadata,
282
- residual,
283
- )
284
- hidden_states, _ = self.norm(hidden_states, residual)
285
- return hidden_states
286
-
287
-
288
-class BaiChuanBaseForCausalLM(nn.Module):
289
-
290
- def __init__(self,
291
- config,
292
- position_embedding: str,
293
- linear_method: Optional[LinearMethodBase] = None):
294
- super().__init__()
295
- self.config = config
296
- self.linear_method = linear_method
297
- self.model = BaiChuanModel(config, position_embedding, linear_method)
298
- self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
299
- self.sampler = Sampler(config.vocab_size)
300
-
301
- def forward(
302
- self,
303
- input_ids: torch.Tensor,
304
- positions: torch.Tensor,
305
- kv_caches: List[KVCache],
306
- input_metadata: InputMetadata,
307
- ) -> torch.Tensor:
308
- hidden_states = self.model(input_ids, positions, kv_caches,
309
- input_metadata)
310
- return hidden_states
311
-
312
- def sample(
313
- self,
314
- hidden_states: torch.Tensor,
315
- sampling_metadata: SamplingMetadata,
316
- ) -> Optional[SamplerOutput]:
317
- next_tokens = self.sampler(self.lm_head.weight, hidden_states,
318
- sampling_metadata)
319
- return next_tokens
33
+class BaiChuanBaseForCausalLM(LlamaForCausalLM):
34
35
def load_weights(self,
36
model_name_or_path: str,
@@ -328,9 +42,15 @@ class BaiChuanBaseForCausalLM(nn.Module):
42
("gate_up_proj", "gate_proj", 0),
43
("gate_up_proj", "up_proj", 1),
44
]
45
+ param_weight_map = [
46
+ ("qkv_proj", "W_pack"),
47
+ ]
48
params_dict = dict(self.named_parameters())
49
for name, loaded_weight in hf_model_weights_iterator(
50
model_name_or_path, cache_dir, load_format, revision):
51
+ for (param_name, weight_name) in param_weight_map:
52
+ name = name.replace(weight_name, param_name)
53
+
54
if "rotary_emb.inv_freq" in name:
55
continue
56
if name == "lm_head.weight":
@@ -368,19 +88,28 @@ class BaiChuanBaseForCausalLM(nn.Module):
88
class BaichuanForCausalLM(BaiChuanBaseForCausalLM):
89
"""Baichuan 13B and Baichuan2 7B/13B."""
90
371
- def __init__(self,
372
- config,
373
- linear_method: Optional[LinearMethodBase] = None):
374
- if config.hidden_size == 4096: # baichuan2 7b
375
- super().__init__(config, "ROPE", linear_method)
376
- else: # baichuan 13b, baichuan2 13b
377
- super().__init__(config, "ALIBI", linear_method)
91
+ def __init__(
92
+ self,
93
+ config: Optional[PretrainedConfig] = None,
94
+ linear_method: Optional[LinearMethodBase] = None,
95
+ lora_config: Optional[LoRAConfig] = None,
96
+ ) -> None:
97
+ if config.hidden_size != 4096: # baichuan 13b, baichuan2 13b
98
+ config.postion_embedding = "ALIBI"
99
+ super().__init__(config=config,
100
+ linear_method=linear_method,
101
+ lora_config=lora_config)
102
103
104
class BaiChuanForCausalLM(BaiChuanBaseForCausalLM):
105
"""Baichuan 7B."""
106
383
- def __init__(self,
384
- config,
385
- linear_method: Optional[LinearMethodBase] = None):
386
- super().__init__(config, "ROPE", linear_method)
107
+ def __init__(
108
+ self,
109
+ config: Optional[PretrainedConfig] = None,
110
+ linear_method: Optional[LinearMethodBase] = None,
111
+ lora_config: Optional[LoRAConfig] = None,
112
+ ) -> None:
113
+ super().__init__(config=config,
114
+ linear_method=linear_method,
115
+ lora_config=lora_config)
vllm/model_executor/models/internlm.py
deleted
-299
@@ -1,299 +0,0 @@
1
-# -*- coding: utf-8 -*-
2
-from typing import Any, Dict, List, Optional, Tuple
3
-
4
-import torch
5
-from torch import nn
6
-from transformers import LlamaConfig
7
-
8
-from vllm.model_executor.input_metadata import InputMetadata
9
-from vllm.model_executor.layers.activation import SiluAndMul
10
-from vllm.model_executor.layers.attention import PagedAttention
11
-from vllm.model_executor.layers.layernorm import RMSNorm
12
-from vllm.model_executor.layers.linear import (LinearMethodBase,
13
- MergedColumnParallelLinear,
14
- QKVParallelLinear,
15
- RowParallelLinear)
16
-from vllm.model_executor.layers.rotary_embedding import get_rope
17
-from vllm.model_executor.layers.sampler import Sampler
18
-from vllm.model_executor.layers.vocab_parallel_embedding import (
19
- VocabParallelEmbedding, ParallelLMHead)
20
-from vllm.model_executor.parallel_utils.parallel_state import (
21
- get_tensor_model_parallel_world_size)
22
-from vllm.model_executor.sampling_metadata import SamplingMetadata
23
-from vllm.model_executor.weight_utils import (default_weight_loader,
24
- hf_model_weights_iterator)
25
-from vllm.sequence import SamplerOutput
26
-
27
-KVCache = Tuple[torch.Tensor, torch.Tensor]
28
-
29
-
30
-class InternLMMLP(nn.Module):
31
-
32
- def __init__(
33
- self,
34
- hidden_size: int,
35
- intermediate_size: int,
36
- hidden_act: str,
37
- linear_method: Optional[LinearMethodBase] = None,
38
- ):
39
- super().__init__()
40
- self.gate_up_proj = MergedColumnParallelLinear(
41
- hidden_size, [intermediate_size] * 2,
42
- bias=False,
43
- linear_method=linear_method)
44
- self.down_proj = RowParallelLinear(intermediate_size,
45
- hidden_size,
46
- bias=False,
47
- linear_method=linear_method)
48
- if hidden_act != "silu":
49
- raise ValueError(f"Unsupported activation: {hidden_act}. "
50
- "Only silu is supported for now.")
51
- self.act_fn = SiluAndMul()
52
-
53
- def forward(self, x):
54
- gate_up, _ = self.gate_up_proj(x)
55
- x = self.act_fn(gate_up)
56
- x, _ = self.down_proj(x)
57
- return x
58
-
59
-
60
-class InternLMAttention(nn.Module):
61
-
62
- def __init__(
63
- self,
64
- hidden_size: int,
65
- num_heads: int,
66
- bias: bool,
67
- rope_theta: float = 10000,
68
- max_position_embeddings: int = 8192,
69
- linear_method: Optional[LinearMethodBase] = None,
70
- rope_scaling: Optional[Dict[str, Any]] = None,
71
- ):
72
- super().__init__()
73
- self.hidden_size = hidden_size
74
- tensor_model_parallel_world_size = (
75
- get_tensor_model_parallel_world_size())
76
- self.total_num_heads = num_heads
77
- assert self.total_num_heads % tensor_model_parallel_world_size == 0
78
- self.num_heads = (self.total_num_heads //
79
- tensor_model_parallel_world_size)
80
- self.head_dim = hidden_size // self.total_num_heads
81
- self.scaling = self.head_dim**-0.5
82
- self.rope_theta = rope_theta
83
- self.max_position_embeddings = max_position_embeddings
84
-
85
- self.qkv_proj = QKVParallelLinear(
86
- hidden_size,
87
- self.head_dim,
88
- self.total_num_heads,
89
- bias=bias,
90
- linear_method=linear_method,
91
- )
92
- self.o_proj = RowParallelLinear(
93
- self.total_num_heads * self.head_dim,
94
- hidden_size,
95
- bias=bias,
96
- linear_method=linear_method,
97
- )
98
- self.rotary_emb = get_rope(
99
- self.head_dim,
100
- rotary_dim=self.head_dim,
101
- max_position=self.max_position_embeddings,
102
- base=self.rope_theta,
103
- rope_scaling=rope_scaling,
104
- )
105
- self.attn = PagedAttention(self.num_heads, self.head_dim, self.scaling)
106
-
107
- def forward(
108
- self,
109
- positions: torch.Tensor,
110
- hidden_states: torch.Tensor,
111
- kv_cache: KVCache,
112
- input_metadata: InputMetadata,
113
- ) -> torch.Tensor:
114
- qkv, _ = self.qkv_proj(hidden_states)
115
- q, k, v = qkv.chunk(chunks=3, dim=-1)
116
- q, k = self.rotary_emb(positions, q, k)
117
- k_cache, v_cache = kv_cache
118
- attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
119
- output, _ = self.o_proj(attn_output)
120
- return output
121
-
122
-
123
-class InternLMDecoderLayer(nn.Module):
124
-
125
- def __init__(
126
- self,
127
- config: LlamaConfig,
128
- linear_method: Optional[LinearMethodBase] = None,
129
- ):
130
- super().__init__()
131
- self.hidden_size = config.hidden_size
132
- rope_theta = getattr(config, "rope_theta", 10000)
133
- max_position_embeddings = getattr(config, "max_position_embeddings",
134
- 8192)
135
- self.self_attn = InternLMAttention(
136
- hidden_size=self.hidden_size,
137
- num_heads=config.num_attention_heads,
138
- bias=config.bias,
139
- rope_theta=rope_theta,
140
- max_position_embeddings=max_position_embeddings,
141
- linear_method=linear_method,
142
- rope_scaling=getattr(config, "rope_scaling", None),
143
- )
144
- self.mlp = InternLMMLP(
145
- hidden_size=self.hidden_size,
146
- intermediate_size=config.intermediate_size,
147
- hidden_act=config.hidden_act,
148
- linear_method=linear_method,
149
- )
150
- self.input_layernorm = RMSNorm(config.hidden_size,
151
- eps=config.rms_norm_eps)
152
- self.post_attention_layernorm = RMSNorm(config.hidden_size,
153
- eps=config.rms_norm_eps)
154
-
155
- def forward(
156
- self,
157
- positions: torch.Tensor,
158
- hidden_states: torch.Tensor,
159
- kv_cache: KVCache,
160
- input_metadata: InputMetadata,
161
- residual: Optional[torch.Tensor],
162
- ) -> Tuple[torch.Tensor, torch.Tensor]:
163
- # Self Attention
164
- if residual is None:
165
- residual = hidden_states
166
- hidden_states = self.input_layernorm(hidden_states)
167
- else:
168
- hidden_states, residual = self.input_layernorm(
169
- hidden_states, residual)
170
- hidden_states = self.self_attn(
171
- positions=positions,
172
- hidden_states=hidden_states,
173
- kv_cache=kv_cache,
174
- input_metadata=input_metadata,
175
- )
176
-
177
- # Fully Connected
178
- hidden_states, residual = self.post_attention_layernorm(
179
- hidden_states, residual)
180
- hidden_states = self.mlp(hidden_states)
181
- return hidden_states, residual
182
-
183
-
184
-class InternLMModel(nn.Module):
185
-
186
- def __init__(
187
- self,
188
- config: LlamaConfig,
189
- linear_method: Optional[LinearMethodBase] = None,
190
- ):
191
- super().__init__()
192
- self.config = config
193
- self.padding_idx = config.pad_token_id
194
- self.vocab_size = config.vocab_size
195
-
196
- vocab_size = ((config.vocab_size + 63) // 64) * 64
197
- self.embed_tokens = VocabParallelEmbedding(
198
- vocab_size,
199
- config.hidden_size,
200
- )
201
- self.layers = nn.ModuleList([
202
- InternLMDecoderLayer(config, linear_method)
203
- for _ in range(config.num_hidden_layers)
204
- ])
205
- self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
206
-
207
- def forward(
208
- self,
209
- input_ids: torch.Tensor,
210
- positions: torch.Tensor,
211
- kv_caches: List[KVCache],
212
- input_metadata: InputMetadata,
213
- ) -> torch.Tensor:
214
- hidden_states = self.embed_tokens(input_ids)
215
- residual = None
216
- for i in range(len(self.layers)):
217
- layer = self.layers[i]
218
- hidden_states, residual = layer(
219
- positions,
220
- hidden_states,
221
- kv_caches[i],
222
- input_metadata,
223
- residual,
224
- )
225
- hidden_states, _ = self.norm(hidden_states, residual)
226
- return hidden_states
227
-
228
-
229
-class InternLMForCausalLM(nn.Module):
230
-
231
- def __init__(
232
- self,
233
- config,
234
- linear_method: Optional[LinearMethodBase] = None,
235
- ):
236
- super().__init__()
237
- self.config = config
238
- self.linear_method = linear_method
239
- self.model = InternLMModel(config, linear_method)
240
- self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
241
- self.sampler = Sampler(config.vocab_size)
242
-
243
- def forward(
244
- self,
245
- input_ids: torch.Tensor,
246
- positions: torch.Tensor,
247
- kv_caches: List[KVCache],
248
- input_metadata: InputMetadata,
249
- ) -> torch.Tensor:
250
- hidden_states = self.model(input_ids, positions, kv_caches,
251
- input_metadata)
252
- return hidden_states
253
-
254
- def sample(
255
- self,
256
- hidden_states: torch.Tensor,
257
- sampling_metadata: SamplingMetadata,
258
- ) -> Optional[SamplerOutput]:
259
- next_tokens = self.sampler(self.lm_head.weight, hidden_states,
260
- sampling_metadata)
261
- return next_tokens
262
-
263
- def load_weights(self,
264
- model_name_or_path: str,
265
- cache_dir: Optional[str] = None,
266
- load_format: str = "auto",
267
- revision: Optional[str] = None):
268
- stacked_params_mapping = [
269
- # (param_name, shard_name, shard_id)
270
- ("qkv_proj", "q_proj", "q"),
271
- ("qkv_proj", "k_proj", "k"),
272
- ("qkv_proj", "v_proj", "v"),
273
- ("gate_up_proj", "gate_proj", 0),
274
- ("gate_up_proj", "up_proj", 1),
275
- ]
276
- params_dict = dict(self.named_parameters())
277
- for name, loaded_weight in hf_model_weights_iterator(
278
- model_name_or_path, cache_dir, load_format, revision):
279
- if "rotary_emb.inv_freq" in name:
280
- continue
281
- for (param_name, weight_name, shard_id) in stacked_params_mapping:
282
- if weight_name not in name:
283
- continue
284
- name = name.replace(weight_name, param_name)
285
- # Skip loading extra bias for GPTQ models.
286
- if name.endswith(".bias") and name not in params_dict:
287
- continue
288
- param = params_dict[name]
289
- weight_loader = param.weight_loader
290
- weight_loader(param, loaded_weight, shard_id)
291
- break
292
- else:
293
- # Skip loading extra bias for GPTQ models.
294
- if name.endswith(".bias") and name not in params_dict:
295
- continue
296
- param = params_dict[name]
297
- weight_loader = getattr(param, "weight_loader",
298
- default_weight_loader)
299
- weight_loader(param, loaded_weight)
vllm/model_executor/models/internlm2.py
+25
-260
@@ -1,276 +1,27 @@
1
# -*- coding: utf-8 -*-
2
-from typing import Any, Dict, List, Optional, Tuple
2
+from typing import Optional
3
4
import torch
5
-from torch import nn
5
from transformers import PretrainedConfig
6
+from vllm.config import LoRAConfig
7
8
-from vllm.model_executor.input_metadata import InputMetadata
9
-from vllm.model_executor.layers.activation import SiluAndMul
10
-from vllm.model_executor.layers.attention import PagedAttention
11
-from vllm.model_executor.layers.layernorm import RMSNorm
12
-from vllm.model_executor.layers.linear import (LinearMethodBase,
13
- MergedColumnParallelLinear,
14
- QKVParallelLinear,
15
- RowParallelLinear)
16
-from vllm.model_executor.layers.rotary_embedding import get_rope
17
-from vllm.model_executor.layers.sampler import Sampler
18
-from vllm.model_executor.layers.vocab_parallel_embedding import (
19
- VocabParallelEmbedding, ParallelLMHead)
20
-from vllm.model_executor.parallel_utils.parallel_state import (
21
- get_tensor_model_parallel_world_size)
22
-from vllm.model_executor.sampling_metadata import SamplingMetadata
8
+from vllm.model_executor.layers.linear import LinearMethodBase
9
+from vllm.model_executor.models.llama import LlamaForCausalLM
10
from vllm.model_executor.weight_utils import (default_weight_loader,
11
hf_model_weights_iterator)
25
-from vllm.sequence import SamplerOutput
12
27
-KVCache = Tuple[torch.Tensor, torch.Tensor]
13
29
-
30
-class InternLM2MLP(nn.Module):
31
-
32
- def __init__(
33
- self,
34
- hidden_size: int,
35
- intermediate_size: int,
36
- hidden_act: str,
37
- linear_method: Optional[LinearMethodBase] = None,
38
- ) -> None:
39
- super().__init__()
40
- self.gate_up_proj = MergedColumnParallelLinear(
41
- hidden_size, [intermediate_size] * 2,
42
- bias=False,
43
- linear_method=linear_method)
44
- self.w2 = RowParallelLinear(intermediate_size,
45
- hidden_size,
46
- bias=False,
47
- linear_method=linear_method)
48
- if hidden_act != "silu":
49
- raise ValueError(f"Unsupported activation: {hidden_act}. "
50
- "Only silu is supported for now.")
51
- self.act_fn = SiluAndMul()
52
-
53
- def forward(self, x):
54
- gate_up, _ = self.gate_up_proj(x)
55
- x = self.act_fn(gate_up)
56
- x, _ = self.w2(x)
57
- return x
58
-
59
-
60
-class InternLM2Attention(nn.Module):
61
-
62
- def __init__(
63
- self,
64
- hidden_size: int,
65
- num_heads: int,
66
- num_kv_heads: int,
67
- rope_theta: float = 10000,
68
- rope_scaling: Optional[Dict[str, Any]] = None,
69
- max_position_embeddings: int = 8192,
70
- linear_method: Optional[LinearMethodBase] = None,
71
- ) -> None:
72
- super().__init__()
73
- self.hidden_size = hidden_size
74
- tp_size = get_tensor_model_parallel_world_size()
75
- self.total_num_heads = num_heads
76
- assert self.total_num_heads % tp_size == 0
77
- self.num_heads = self.total_num_heads // tp_size
78
- self.total_num_kv_heads = num_kv_heads
79
- if self.total_num_kv_heads >= tp_size:
80
- # Number of KV heads is greater than TP size, so we partition
81
- # the KV heads across multiple tensor parallel GPUs.
82
- assert self.total_num_kv_heads % tp_size == 0
83
- else:
84
- # Number of KV heads is less than TP size, so we replicate
85
- # the KV heads across multiple tensor parallel GPUs.
86
- assert tp_size % self.total_num_kv_heads == 0
87
- self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
88
- self.head_dim = hidden_size // self.total_num_heads
89
- self.q_size = self.num_heads * self.head_dim
90
- self.kv_size = self.num_kv_heads * self.head_dim
91
- self.scaling = self.head_dim**-0.5
92
- self.rope_theta = rope_theta
93
- self.max_position_embeddings = max_position_embeddings
94
-
95
- self.wqkv = QKVParallelLinear(
96
- hidden_size,
97
- self.head_dim,
98
- self.total_num_heads,
99
- self.total_num_kv_heads,
100
- bias=False,
101
- linear_method=linear_method,
102
- )
103
- self.wo = RowParallelLinear(
104
- self.total_num_heads * self.head_dim,
105
- hidden_size,
106
- bias=False,
107
- linear_method=linear_method,
108
- )
109
-
110
- self.rotary_emb = get_rope(
111
- self.head_dim,
112
- rotary_dim=self.head_dim,
113
- max_position=max_position_embeddings,
114
- base=rope_theta,
115
- rope_scaling=rope_scaling,
116
- )
117
- self.attn = PagedAttention(self.num_heads,
118
- self.head_dim,
119
- self.scaling,
120
- num_kv_heads=self.num_kv_heads)
121
-
122
- def forward(
123
- self,
124
- positions: torch.Tensor,
125
- hidden_states: torch.Tensor,
126
- kv_cache: KVCache,
127
- input_metadata: InputMetadata,
128
- ) -> torch.Tensor:
129
- qkv, _ = self.wqkv(hidden_states)
130
- q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
131
- q, k = self.rotary_emb(positions, q, k)
132
- k_cache, v_cache = kv_cache
133
- attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
134
- output, _ = self.wo(attn_output)
135
- return output
136
-
137
-
138
-class InternLMDecoderLayer(nn.Module):
139
-
140
- def __init__(
141
- self,
142
- config: PretrainedConfig,
143
- linear_method: Optional[LinearMethodBase] = None,
144
- ) -> None:
145
- super().__init__()
146
- self.hidden_size = config.hidden_size
147
- rope_theta = getattr(config, "rope_theta", 10000)
148
- rope_scaling = getattr(config, "rope_scaling", None)
149
- max_position_embeddings = getattr(config, "max_position_embeddings",
150
- 8192)
151
- self.attention = InternLM2Attention(
152
- hidden_size=self.hidden_size,
153
- num_heads=config.num_attention_heads,
154
- num_kv_heads=config.num_key_value_heads,
155
- rope_theta=rope_theta,
156
- rope_scaling=rope_scaling,
157
- max_position_embeddings=max_position_embeddings,
158
- linear_method=linear_method,
159
- )
160
- self.feed_forward = InternLM2MLP(
161
- hidden_size=self.hidden_size,
162
- intermediate_size=config.intermediate_size,
163
- hidden_act=config.hidden_act,
164
- linear_method=linear_method,
165
- )
166
- self.attention_norm = RMSNorm(config.hidden_size,
167
- eps=config.rms_norm_eps)
168
- self.ffn_norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
169
-
170
- def forward(
171
- self,
172
- positions: torch.Tensor,
173
- hidden_states: torch.Tensor,
174
- kv_cache: KVCache,
175
- input_metadata: InputMetadata,
176
- residual: Optional[torch.Tensor],
177
- ) -> Tuple[torch.Tensor, torch.Tensor]:
178
- # Self Attention
179
- if residual is None:
180
- residual = hidden_states
181
- hidden_states = self.attention_norm(hidden_states)
182
- else:
183
- hidden_states, residual = self.attention_norm(
184
- hidden_states, residual)
185
- hidden_states = self.attention(
186
- positions=positions,
187
- hidden_states=hidden_states,
188
- kv_cache=kv_cache,
189
- input_metadata=input_metadata,
190
- )
191
-
192
- # Fully Connected
193
- hidden_states, residual = self.ffn_norm(hidden_states, residual)
194
- hidden_states = self.feed_forward(hidden_states)
195
- return hidden_states, residual
196
-
197
-
198
-class InternLM2Model(nn.Module):
14
+class InternLM2ForCausalLM(LlamaForCausalLM):
15
16
def __init__(
17
self,
202
- config: PretrainedConfig,
18
+ config: Optional[PretrainedConfig] = None,
19
linear_method: Optional[LinearMethodBase] = None,
20
+ lora_config: Optional[LoRAConfig] = None,
21
) -> None:
205
- super().__init__()
206
- self.config = config
207
- self.padding_idx = config.pad_token_id
208
- self.vocab_size = config.vocab_size
209
- self.tok_embeddings = VocabParallelEmbedding(
210
- config.vocab_size,
211
- config.hidden_size,
212
- )
213
- self.layers = nn.ModuleList([
214
- InternLMDecoderLayer(config, linear_method)
215
- for _ in range(config.num_hidden_layers)
216
- ])
217
- self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
218
-
219
- def forward(
220
- self,
221
- input_ids: torch.Tensor,
222
- positions: torch.Tensor,
223
- kv_caches: List[KVCache],
224
- input_metadata: InputMetadata,
225
- ) -> torch.Tensor:
226
- hidden_states = self.tok_embeddings(input_ids)
227
- residual = None
228
- for i in range(len(self.layers)):
229
- layer = self.layers[i]
230
- hidden_states, residual = layer(
231
- positions,
232
- hidden_states,
233
- kv_caches[i],
234
- input_metadata,
235
- residual,
236
- )
237
- hidden_states, _ = self.norm(hidden_states, residual)
238
- return hidden_states
239
-
240
-
241
-class InternLM2ForCausalLM(nn.Module):
242
-
243
- def __init__(
244
- self,
245
- config: PretrainedConfig,
246
- linear_method: Optional[LinearMethodBase] = None,
247
- ) -> None:
248
- super().__init__()
249
- self.config = config
250
- self.linear_method = linear_method
251
- self.model = InternLM2Model(config, linear_method)
252
- self.output = ParallelLMHead(config.vocab_size, config.hidden_size)
253
- self.sampler = Sampler(config.vocab_size)
254
-
255
- def forward(
256
- self,
257
- input_ids: torch.Tensor,
258
- positions: torch.Tensor,
259
- kv_caches: List[KVCache],
260
- input_metadata: InputMetadata,
261
- ) -> torch.Tensor:
262
- hidden_states = self.model(input_ids, positions, kv_caches,
263
- input_metadata)
264
- return hidden_states
265
-
266
- def sample(
267
- self,
268
- hidden_states: torch.Tensor,
269
- sampling_metadata: SamplingMetadata,
270
- ) -> Optional[SamplerOutput]:
271
- next_tokens = self.sampler(self.output.weight, hidden_states,
272
- sampling_metadata)
273
- return next_tokens
22
+ super().__init__(config=config,
23
+ linear_method=linear_method,
24
+ lora_config=lora_config)
25
26
def load_weights(self,
27
model_name_or_path: str,
@@ -282,9 +33,23 @@ class InternLM2ForCausalLM(nn.Module):
33
("gate_up_proj", "w1", 0),
34
("gate_up_proj", "w3", 1),
35
]
36
+ param_weight_map = [
37
+ ("qkv_proj", "wqkv"),
38
+ ("o_proj", "wo"),
39
+ ("down_proj", "w2"),
40
+ ("input_layernorm", "attention_norm"),
41
+ ("post_attention_layernorm", "ffn_norm"),
42
+ ("embed_tokens", "tok_embeddings"),
43
+ (".self_attn.", ".attention."),
44
+ ("mlp", "feed_forward"),
45
+ ("lm_head", "output"),
46
+ ]
47
params_dict = dict(self.named_parameters())
48
for name, loaded_weight in hf_model_weights_iterator(
49
model_name_or_path, cache_dir, load_format, revision):
50
+ for (param_name, weight_name) in param_weight_map:
51
+ name = name.replace(weight_name, param_name)
52
+
53
if "rotary_emb.inv_freq" in name:
54
continue
55
for (param_name, weight_name, shard_id) in stacked_params_mapping:
@@ -303,7 +68,7 @@ class InternLM2ForCausalLM(nn.Module):
68
if name.endswith(".bias") and name not in params_dict:
69
continue
70
param = params_dict[name]
306
- if "wqkv" in name:
71
+ if "qkv_proj" in name:
72
config = self.config
73
kv_groups = config.num_attention_heads // config.num_key_value_heads
74
head_dim = config.hidden_size // config.num_attention_heads
vllm/model_executor/models/llama.py
+106
-56
@@ -21,8 +21,9 @@
21
# See the License for the specific language governing permissions and
22
# limitations under the License.
23
"""Inference-only LLaMA model compatible with HuggingFace weights."""
24
-from typing import Any, Dict, List, Optional, Tuple
24
+from typing import List, Optional, Tuple
25
26
+import math
27
import torch
28
from torch import nn
29
from transformers import LlamaConfig
@@ -40,34 +41,60 @@ from vllm.model_executor.layers.sampler import Sampler
41
from vllm.model_executor.layers.vocab_parallel_embedding import (
42
VocabParallelEmbedding, ParallelLMHead, DEFAULT_VOCAB_PADDING_SIZE)
43
from vllm.model_executor.parallel_utils.parallel_state import (
43
- get_tensor_model_parallel_world_size)
44
+ get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
45
from vllm.model_executor.sampling_metadata import SamplingMetadata
46
from vllm.model_executor.weight_utils import (default_weight_loader,
47
hf_model_weights_iterator)
48
from vllm.sequence import SamplerOutput
49
from vllm.config import LoRAConfig
50
51
+from copy import deepcopy
52
+
53
KVCache = Tuple[torch.Tensor, torch.Tensor]
54
55
56
+def _get_alibi_slopes(total_num_heads: int) -> torch.Tensor:
57
+ closest_power_of_2 = 2**math.floor(math.log2(total_num_heads))
58
+ base = torch.tensor(
59
+ 2**(-(2**-(math.log2(closest_power_of_2) - 3))),
60
+ dtype=torch.float32,
61
+ )
62
+ powers = torch.arange(1, 1 + closest_power_of_2, dtype=torch.int32)
63
+ slopes = torch.pow(base, powers)
64
+
65
+ if closest_power_of_2 != total_num_heads:
66
+ extra_base = torch.tensor(
67
+ 2**(-(2**-(math.log2(2 * closest_power_of_2) - 3))),
68
+ dtype=torch.float32,
69
+ )
70
+ num_remaining_heads = min(closest_power_of_2,
71
+ total_num_heads - closest_power_of_2)
72
+ extra_powers = torch.arange(start=1,
73
+ end=1 + 2 * num_remaining_heads,
74
+ step=2,
75
+ dtype=torch.int32)
76
+ slopes = torch.cat(
77
+ [slopes, torch.pow(extra_base, extra_powers)], dim=0)
78
+ return slopes
79
+
80
+
81
class LlamaMLP(nn.Module):
82
83
def __init__(
84
self,
57
- hidden_size: int,
58
- intermediate_size: int,
59
- hidden_act: str,
85
+ config: LlamaConfig,
86
linear_method: Optional[LinearMethodBase] = None,
87
) -> None:
88
super().__init__()
89
self.gate_up_proj = MergedColumnParallelLinear(
64
- hidden_size, [intermediate_size] * 2,
90
+ config.hidden_size, [config.intermediate_size] * 2,
91
bias=False,
92
linear_method=linear_method)
67
- self.down_proj = RowParallelLinear(intermediate_size,
68
- hidden_size,
93
+ self.down_proj = RowParallelLinear(config.intermediate_size,
94
+ config.hidden_size,
95
bias=False,
96
linear_method=linear_method)
97
+ hidden_act = getattr(config, "hidden_act", "silu")
98
if hidden_act != "silu":
99
raise ValueError(f"Unsupported activation: {hidden_act}. "
100
"Only silu is supported for now.")
@@ -84,21 +111,19 @@ class LlamaAttention(nn.Module):
111
112
def __init__(
113
self,
87
- hidden_size: int,
88
- num_heads: int,
89
- num_kv_heads: int,
90
- rope_theta: float = 10000,
91
- rope_scaling: Optional[Dict[str, Any]] = None,
92
- max_position_embeddings: int = 8192,
114
+ config: LlamaConfig,
115
linear_method: Optional[LinearMethodBase] = None,
116
) -> None:
117
super().__init__()
96
- self.hidden_size = hidden_size
118
+ self.hidden_size = config.hidden_size
119
tp_size = get_tensor_model_parallel_world_size()
98
- self.total_num_heads = num_heads
120
+ self.total_num_heads = getattr(config, "num_attention_heads", None)
121
assert self.total_num_heads % tp_size == 0
122
self.num_heads = self.total_num_heads // tp_size
101
- self.total_num_kv_heads = num_kv_heads
123
+
124
+ # defaut to mha
125
+ self.total_num_kv_heads = getattr(config, "num_key_value_heads",
126
+ self.total_num_heads)
127
if self.total_num_kv_heads >= tp_size:
128
# Number of KV heads is greater than TP size, so we partition
129
# the KV heads across multiple tensor parallel GPUs.
@@ -108,39 +133,68 @@ class LlamaAttention(nn.Module):
133
# the KV heads across multiple tensor parallel GPUs.
134
assert tp_size % self.total_num_kv_heads == 0
135
self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
111
- self.head_dim = hidden_size // self.total_num_heads
136
+ self.head_dim = self.hidden_size // self.total_num_heads
137
self.q_size = self.num_heads * self.head_dim
138
self.kv_size = self.num_kv_heads * self.head_dim
139
self.scaling = self.head_dim**-0.5
115
- self.rope_theta = rope_theta
116
- self.max_position_embeddings = max_position_embeddings
140
+ max_position_embeddings = getattr(config, "max_position_embeddings",
141
+ 8192)
142
+ self.max_position_embeddings = config.max_position_embeddings
143
144
+ # internlm
145
+ bias = getattr(config, "bias", False)
146
+
147
+ # stablelm
148
+ qkv_bias = getattr(config, "use_qkv_bias", False)
149
self.qkv_proj = QKVParallelLinear(
119
- hidden_size,
150
+ self.hidden_size,
151
self.head_dim,
152
self.total_num_heads,
153
self.total_num_kv_heads,
123
- bias=False,
154
+ bias=bias or qkv_bias,
155
linear_method=linear_method,
156
)
157
self.o_proj = RowParallelLinear(
158
self.total_num_heads * self.head_dim,
128
- hidden_size,
129
- bias=False,
159
+ self.hidden_size,
160
+ bias=bias,
161
linear_method=linear_method,
162
)
163
133
- self.rotary_emb = get_rope(
134
- self.head_dim,
135
- rotary_dim=self.head_dim,
136
- max_position=max_position_embeddings,
137
- base=rope_theta,
138
- rope_scaling=rope_scaling,
139
- )
140
- self.attn = PagedAttention(self.num_heads,
141
- self.head_dim,
142
- self.scaling,
143
- num_kv_heads=self.num_kv_heads)
164
+ # mistral
165
+ sliding_window = getattr(config, "sliding_window", None)
166
+
167
+ self.postion_embedding = getattr(config, "postion_embedding", "ROPE")
168
+ # Create the alibi slopes and slice them.
169
+ if self.postion_embedding == "ALIBI":
170
+ tp_rank = get_tensor_model_parallel_rank()
171
+ head_start = tp_rank * self.num_heads
172
+ head_end = (tp_rank + 1) * self.num_heads
173
+ alibi_slopes = _get_alibi_slopes(self.total_num_heads)
174
+ alibi_slopes = alibi_slopes[head_start:head_end].tolist()
175
+
176
+ self.attn = PagedAttention(self.num_heads,
177
+ self.head_dim,
178
+ self.scaling,
179
+ alibi_slopes=alibi_slopes,
180
+ sliding_window=sliding_window)
181
+ else:
182
+ rope_theta = getattr(config, "rope_theta", 10000)
183
+ rope_scaling = getattr(config, "rope_scaling", None)
184
+ # stablelm
185
+ rope_pct = getattr(config, "rope_pct", 1)
186
+ self.rotary_emb = get_rope(
187
+ self.head_dim,
188
+ rotary_dim=int(self.head_dim * rope_pct),
189
+ max_position=max_position_embeddings,
190
+ base=rope_theta,
191
+ rope_scaling=rope_scaling,
192
+ )
193
+ self.attn = PagedAttention(self.num_heads,
194
+ self.head_dim,
195
+ self.scaling,
196
+ num_kv_heads=self.num_kv_heads,
197
+ sliding_window=sliding_window)
198
199
def forward(
200
self,
@@ -151,7 +205,8 @@ class LlamaAttention(nn.Module):
205
) -> torch.Tensor:
206
qkv, _ = self.qkv_proj(hidden_states)
207
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
154
- q, k = self.rotary_emb(positions, q, k)
208
+ if self.postion_embedding != "ALIBI":
209
+ q, k = self.rotary_emb(positions, q, k)
210
k_cache, v_cache = kv_cache
211
attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
212
output, _ = self.o_proj(attn_output)
@@ -164,32 +219,20 @@ class LlamaDecoderLayer(nn.Module):
219
self,
220
config: LlamaConfig,
221
linear_method: Optional[LinearMethodBase] = None,
222
+ norm: Optional[torch.Tensor] = None,
223
) -> None:
224
super().__init__()
225
self.hidden_size = config.hidden_size
170
- rope_theta = getattr(config, "rope_theta", 10000)
171
- rope_scaling = getattr(config, "rope_scaling", None)
172
- max_position_embeddings = getattr(config, "max_position_embeddings",
173
- 8192)
226
self.self_attn = LlamaAttention(
175
- hidden_size=self.hidden_size,
176
- num_heads=config.num_attention_heads,
177
- num_kv_heads=config.num_key_value_heads,
178
- rope_theta=rope_theta,
179
- rope_scaling=rope_scaling,
180
- max_position_embeddings=max_position_embeddings,
227
+ config,
228
linear_method=linear_method,
229
)
230
self.mlp = LlamaMLP(
184
- hidden_size=self.hidden_size,
185
- intermediate_size=config.intermediate_size,
186
- hidden_act=config.hidden_act,
231
+ config,
232
linear_method=linear_method,
233
)
189
- self.input_layernorm = RMSNorm(config.hidden_size,
190
- eps=config.rms_norm_eps)
191
- self.post_attention_layernorm = RMSNorm(config.hidden_size,
192
- eps=config.rms_norm_eps)
234
+ self.input_layernorm = deepcopy(norm)
235
+ self.post_attention_layernorm = deepcopy(norm)
236
237
def forward(
238
self,
@@ -226,6 +269,7 @@ class LlamaModel(nn.Module):
269
self,
270
config: LlamaConfig,
271
linear_method: Optional[LinearMethodBase] = None,
272
+ norm: Optional[torch.Tensor] = None,
273
lora_config: Optional[LoRAConfig] = None,
274
) -> None:
275
super().__init__()
@@ -241,10 +285,10 @@ class LlamaModel(nn.Module):
285
org_num_embeddings=config.vocab_size,
286
)
287
self.layers = nn.ModuleList([
244
- LlamaDecoderLayer(config, linear_method)
288
+ LlamaDecoderLayer(config, linear_method, norm)
289
for _ in range(config.num_hidden_layers)
290
])
247
- self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
291
+ self.norm = norm
292
293
def forward(
294
self,
@@ -275,12 +319,18 @@ class LlamaForCausalLM(nn.Module):
319
self,
320
config: LlamaConfig,
321
linear_method: Optional[LinearMethodBase] = None,
322
+ norm: Optional[torch.Tensor] = None,
323
lora_config: Optional[LoRAConfig] = None,
324
) -> None:
325
super().__init__()
326
self.config = config
327
self.linear_method = linear_method
283
- self.model = LlamaModel(config, linear_method, lora_config=lora_config)
328
+ if norm is None:
329
+ norm = RMSNorm(config.hidden_size, config.rms_norm_eps)
330
+ self.model = LlamaModel(config,
331
+ linear_method,
332
+ norm=norm,
333
+ lora_config=lora_config)
334
unpadded_vocab_size = config.vocab_size
335
if lora_config:
336
unpadded_vocab_size += lora_config.lora_extra_vocab_size
vllm/model_executor/models/mistral.py
deleted
-352
@@ -1,352 +0,0 @@
1
-# coding=utf-8
2
-# Adapted from
3
-# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
4
-# Copyright 2023 The vLLM team.
5
-# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
6
-#
7
-# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
8
-# and OPT implementations in this library. It has been modified from its
9
-# original forms to accommodate minor architectural differences compared
10
-# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
11
-#
12
-# Licensed under the Apache License, Version 2.0 (the "License");
13
-# you may not use this file except in compliance with the License.
14
-# You may obtain a copy of the License at
15
-#
16
-# http://www.apache.org/licenses/LICENSE-2.0
17
-#
18
-# Unless required by applicable law or agreed to in writing, software
19
-# distributed under the License is distributed on an "AS IS" BASIS,
20
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21
-# See the License for the specific language governing permissions and
22
-# limitations under the License.
23
-"""Inference-only Mistral model compatible with HuggingFace weights."""
24
-from typing import List, Optional, Tuple
25
-
26
-import torch
27
-from torch import nn
28
-from transformers import MistralConfig
29
-
30
-from vllm.model_executor.input_metadata import InputMetadata
31
-from vllm.model_executor.layers.activation import SiluAndMul
32
-from vllm.model_executor.layers.attention import PagedAttention
33
-from vllm.model_executor.layers.layernorm import RMSNorm
34
-from vllm.model_executor.layers.linear import (LinearMethodBase,
35
- MergedColumnParallelLinear,
36
- QKVParallelLinear,
37
- RowParallelLinear)
38
-from vllm.model_executor.layers.rotary_embedding import get_rope
39
-from vllm.model_executor.layers.sampler import Sampler
40
-from vllm.model_executor.layers.vocab_parallel_embedding import (
41
- VocabParallelEmbedding, ParallelLMHead, DEFAULT_VOCAB_PADDING_SIZE)
42
-from vllm.model_executor.parallel_utils.parallel_state import (
43
- get_tensor_model_parallel_world_size)
44
-from vllm.model_executor.sampling_metadata import SamplingMetadata
45
-from vllm.model_executor.weight_utils import (default_weight_loader,
46
- hf_model_weights_iterator)
47
-from vllm.sequence import SamplerOutput
48
-from vllm.config import LoRAConfig
49
-
50
-KVCache = Tuple[torch.Tensor, torch.Tensor]
51
-
52
-
53
-class MistralMLP(nn.Module):
54
-
55
- def __init__(
56
- self,
57
- hidden_size: int,
58
- intermediate_size: int,
59
- hidden_act: str,
60
- linear_method: Optional[LinearMethodBase] = None,
61
- ) -> None:
62
- super().__init__()
63
- self.gate_up_proj = MergedColumnParallelLinear(
64
- hidden_size, [intermediate_size] * 2,
65
- bias=False,
66
- linear_method=linear_method)
67
- self.down_proj = RowParallelLinear(intermediate_size,
68
- hidden_size,
69
- bias=False,
70
- linear_method=linear_method)
71
- if hidden_act != "silu":
72
- raise ValueError(f"Unsupported activation: {hidden_act}. "
73
- "Only silu is supported for now.")
74
- self.act_fn = SiluAndMul()
75
-
76
- def forward(self, x):
77
- gate_up, _ = self.gate_up_proj(x)
78
- x = self.act_fn(gate_up)
79
- x, _ = self.down_proj(x)
80
- return x
81
-
82
-
83
-class MistralAttention(nn.Module):
84
-
85
- def __init__(self,
86
- hidden_size: int,
87
- num_heads: int,
88
- num_kv_heads: int,
89
- max_position: int = 4096 * 32,
90
- rope_theta: float = 10000,
91
- linear_method: Optional[LinearMethodBase] = None,
92
- sliding_window: Optional[int] = None) -> None:
93
- super().__init__()
94
- self.hidden_size = hidden_size
95
- tp_size = get_tensor_model_parallel_world_size()
96
- self.total_num_heads = num_heads
97
- assert self.total_num_heads % tp_size == 0
98
- self.num_heads = self.total_num_heads // tp_size
99
- self.total_num_kv_heads = num_kv_heads
100
- if self.total_num_kv_heads >= tp_size:
101
- # Number of KV heads is greater than TP size, so we partition
102
- # the KV heads across multiple tensor parallel GPUs.
103
- assert self.total_num_kv_heads % tp_size == 0
104
- else:
105
- # Number of KV heads is less than TP size, so we replicate
106
- # the KV heads across multiple tensor parallel GPUs.
107
- assert tp_size % self.total_num_kv_heads == 0
108
- self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
109
- self.head_dim = hidden_size // self.total_num_heads
110
- self.q_size = self.num_heads * self.head_dim
111
- self.kv_size = self.num_kv_heads * self.head_dim
112
- self.scaling = self.head_dim**-0.5
113
- self.rope_theta = rope_theta
114
- self.sliding_window = sliding_window
115
-
116
- self.qkv_proj = QKVParallelLinear(
117
- hidden_size,
118
- self.head_dim,
119
- self.total_num_heads,
120
- self.total_num_kv_heads,
121
- bias=False,
122
- linear_method=linear_method,
123
- )
124
- self.o_proj = RowParallelLinear(
125
- self.total_num_heads * self.head_dim,
126
- hidden_size,
127
- bias=False,
128
- linear_method=linear_method,
129
- )
130
-
131
- self.rotary_emb = get_rope(
132
- self.head_dim,
133
- rotary_dim=self.head_dim,
134
- max_position=max_position,
135
- base=self.rope_theta,
136
- )
137
- self.attn = PagedAttention(self.num_heads,
138
- self.head_dim,
139
- self.scaling,
140
- num_kv_heads=self.num_kv_heads,
141
- sliding_window=self.sliding_window)
142
-
143
- def forward(
144
- self,
145
- positions: torch.Tensor,
146
- hidden_states: torch.Tensor,
147
- kv_cache: KVCache,
148
- input_metadata: InputMetadata,
149
- ) -> torch.Tensor:
150
- qkv, _ = self.qkv_proj(hidden_states)
151
- q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
152
- q, k = self.rotary_emb(positions, q, k)
153
- k_cache, v_cache = kv_cache
154
- attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
155
- output, _ = self.o_proj(attn_output)
156
- return output
157
-
158
-
159
-class MistralDecoderLayer(nn.Module):
160
-
161
- def __init__(
162
- self,
163
- config: MistralConfig,
164
- linear_method: Optional[LinearMethodBase] = None,
165
- ) -> None:
166
- super().__init__()
167
- self.hidden_size = config.hidden_size
168
- # Requires transformers > 4.32.0
169
- rope_theta = getattr(config, "rope_theta", 10000)
170
- self.self_attn = MistralAttention(
171
- hidden_size=self.hidden_size,
172
- num_heads=config.num_attention_heads,
173
- max_position=config.max_position_embeddings,
174
- num_kv_heads=config.num_key_value_heads,
175
- rope_theta=rope_theta,
176
- linear_method=linear_method,
177
- sliding_window=config.sliding_window)
178
- self.mlp = MistralMLP(
179
- hidden_size=self.hidden_size,
180
- intermediate_size=config.intermediate_size,
181
- hidden_act=config.hidden_act,
182
- linear_method=linear_method,
183
- )
184
- self.input_layernorm = RMSNorm(config.hidden_size,
185
- eps=config.rms_norm_eps)
186
- self.post_attention_layernorm = RMSNorm(config.hidden_size,
187
- eps=config.rms_norm_eps)
188
-
189
- def forward(
190
- self,
191
- positions: torch.Tensor,
192
- hidden_states: torch.Tensor,
193
- kv_cache: KVCache,
194
- input_metadata: InputMetadata,
195
- residual: Optional[torch.Tensor],
196
- ) -> Tuple[torch.Tensor, torch.Tensor]:
197
- # Self Attention
198
- if residual is None:
199
- residual = hidden_states
200
- hidden_states = self.input_layernorm(hidden_states)
201
- else:
202
- hidden_states, residual = self.input_layernorm(
203
- hidden_states, residual)
204
- hidden_states = self.self_attn(
205
- positions=positions,
206
- hidden_states=hidden_states,
207
- kv_cache=kv_cache,
208
- input_metadata=input_metadata,
209
- )
210
-
211
- # Fully Connected
212
- hidden_states, residual = self.post_attention_layernorm(
213
- hidden_states, residual)
214
- hidden_states = self.mlp(hidden_states)
215
- return hidden_states, residual
216
-
217
-
218
-class MistralModel(nn.Module):
219
-
220
- def __init__(
221
- self,
222
- config: MistralConfig,
223
- linear_method: Optional[LinearMethodBase] = None,
224
- lora_config: Optional[LoRAConfig] = None,
225
- ) -> None:
226
- super().__init__()
227
- self.config = config
228
- self.padding_idx = config.pad_token_id
229
- lora_vocab = (lora_config.lora_extra_vocab_size *
230
- (lora_config.max_loras or 1)) if lora_config else 0
231
- self.vocab_size = config.vocab_size + lora_vocab
232
- self.org_vocab_size = config.vocab_size
233
-
234
- self.embed_tokens = VocabParallelEmbedding(
235
- self.vocab_size,
236
- config.hidden_size,
237
- org_num_embeddings=config.vocab_size,
238
- )
239
- self.layers = nn.ModuleList([
240
- MistralDecoderLayer(config, linear_method)
241
- for _ in range(config.num_hidden_layers)
242
- ])
243
- self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
244
-
245
- def forward(
246
- self,
247
- input_ids: torch.Tensor,
248
- positions: torch.Tensor,
249
- kv_caches: List[KVCache],
250
- input_metadata: InputMetadata,
251
- ) -> torch.Tensor:
252
- hidden_states = self.embed_tokens(input_ids)
253
- residual = None
254
- for i in range(len(self.layers)):
255
- layer = self.layers[i]
256
- hidden_states, residual = layer(
257
- positions,
258
- hidden_states,
259
- kv_caches[i],
260
- input_metadata,
261
- residual,
262
- )
263
- hidden_states, _ = self.norm(hidden_states, residual)
264
- return hidden_states
265
-
266
-
267
-class MistralForCausalLM(nn.Module):
268
- supports_lora = True
269
-
270
- def __init__(
271
- self,
272
- config: MistralConfig,
273
- linear_method: Optional[LinearMethodBase] = None,
274
- lora_config: Optional[LoRAConfig] = None,
275
- ) -> None:
276
- super().__init__()
277
- self.config = config
278
- self.linear_method = linear_method
279
- self.model = MistralModel(config,
280
- linear_method,
281
- lora_config=lora_config)
282
- unpadded_vocab_size = config.vocab_size
283
- if lora_config:
284
- unpadded_vocab_size += lora_config.lora_extra_vocab_size
285
- self.lm_head = ParallelLMHead(
286
- unpadded_vocab_size,
287
- config.hidden_size,
288
- org_num_embeddings=config.vocab_size,
289
- padding_size=DEFAULT_VOCAB_PADDING_SIZE
290
- # We need bigger padding if using lora for kernel
291
- # compatibility
292
- if not lora_config else lora_config.lora_vocab_padding_size,
293
- )
294
- self.sampler = Sampler(unpadded_vocab_size, config.vocab_size)
295
-
296
- def forward(
297
- self,
298
- input_ids: torch.Tensor,
299
- positions: torch.Tensor,
300
- kv_caches: List[KVCache],
301
- input_metadata: InputMetadata,
302
- ) -> torch.Tensor:
303
- hidden_states = self.model(input_ids, positions, kv_caches,
304
- input_metadata)
305
- return hidden_states
306
-
307
- def sample(
308
- self,
309
- hidden_states: torch.Tensor,
310
- sampling_metadata: SamplingMetadata,
311
- ) -> Optional[SamplerOutput]:
312
- next_tokens = self.sampler(self.lm_head.weight, hidden_states,
313
- sampling_metadata)
314
- return next_tokens
315
-
316
- def load_weights(self,
317
- model_name_or_path: str,
318
- cache_dir: Optional[str] = None,
319
- load_format: str = "auto",
320
- revision: Optional[str] = None):
321
- stacked_params_mapping = [
322
- # (param_name, shard_name, shard_id)
323
- ("qkv_proj", "q_proj", "q"),
324
- ("qkv_proj", "k_proj", "k"),
325
- ("qkv_proj", "v_proj", "v"),
326
- ("gate_up_proj", "gate_proj", 0),
327
- ("gate_up_proj", "up_proj", 1),
328
- ]
329
- params_dict = dict(self.named_parameters())
330
- for name, loaded_weight in hf_model_weights_iterator(
331
- model_name_or_path, cache_dir, load_format, revision):
332
- if "rotary_emb.inv_freq" in name:
333
- continue
334
- for (param_name, weight_name, shard_id) in stacked_params_mapping:
335
- if weight_name not in name:
336
- continue
337
- name = name.replace(weight_name, param_name)
338
- # Skip loading extra bias for GPTQ models.
339
- if name.endswith(".bias") and name not in params_dict:
340
- continue
341
- param = params_dict[name]
342
- weight_loader = param.weight_loader
343
- weight_loader(param, loaded_weight, shard_id)
344
- break
345
- else:
346
- # Skip loading extra bias for GPTQ models.
347
- if name.endswith(".bias") and name not in params_dict:
348
- continue
349
- param = params_dict[name]
350
- weight_loader = getattr(param, "weight_loader",
351
- default_weight_loader)
352
- weight_loader(param, loaded_weight)
vllm/model_executor/models/qwen.py
+31
-236
@@ -4,253 +4,33 @@
4
# Copyright (c) Alibaba Cloud.
5
# LICENSE: https://huggingface.co/Qwen/Qwen-7B/blob/main/LICENSE
6
"""Inference-only QWen model compatible with HuggingFace weights."""
7
-from typing import Any, Dict, List, Optional, Tuple
7
+from typing import Optional
8
9
-import torch
10
-from torch import nn
9
+from transformers import PretrainedConfig
10
+from vllm.config import LoRAConfig
11
12
-from vllm.model_executor.input_metadata import InputMetadata
13
-from vllm.model_executor.layers.activation import SiluAndMul
14
-from vllm.model_executor.layers.attention import PagedAttention
12
+from vllm.model_executor.layers.linear import LinearMethodBase
13
from vllm.model_executor.layers.layernorm import RMSNorm
16
-from vllm.model_executor.layers.linear import (LinearMethodBase,
17
- MergedColumnParallelLinear,
18
- QKVParallelLinear,
19
- RowParallelLinear)
20
-from vllm.model_executor.layers.rotary_embedding import get_rope
21
-from vllm.model_executor.layers.sampler import Sampler
22
-from vllm.model_executor.layers.vocab_parallel_embedding import (
23
- VocabParallelEmbedding, ParallelLMHead)
24
-from vllm.model_executor.parallel_utils.parallel_state import (
25
- get_tensor_model_parallel_world_size)
26
-from vllm.model_executor.sampling_metadata import SamplingMetadata
14
+from vllm.model_executor.models.llama import LlamaForCausalLM
15
from vllm.model_executor.weight_utils import (default_weight_loader,
16
hf_model_weights_iterator)
29
-from vllm.sequence import SamplerOutput
30
-from vllm.transformers_utils.configs.qwen import QWenConfig
17
32
-KVCache = Tuple[torch.Tensor, torch.Tensor]
18
34
-
35
-class QWenMLP(nn.Module):
36
-
37
- def __init__(
38
- self,
39
- hidden_size: int,
40
- intermediate_size: int,
41
- hidden_act: str = "silu",
42
- linear_method: Optional[LinearMethodBase] = None,
43
- ):
44
- super().__init__()
45
- self.gate_up_proj = MergedColumnParallelLinear(
46
- hidden_size, [intermediate_size] * 2,
47
- bias=False,
48
- linear_method=linear_method)
49
- self.c_proj = RowParallelLinear(intermediate_size,
50
- hidden_size,
51
- bias=False,
52
- linear_method=linear_method)
53
- if hidden_act != "silu":
54
- raise ValueError(f"Unsupported activation: {hidden_act}. "
55
- "Only silu is supported for now.")
56
- self.act_fn = SiluAndMul()
57
-
58
- def forward(self, x):
59
- gate_up, _ = self.gate_up_proj(x)
60
- x = self.act_fn(gate_up)
61
- x, _ = self.c_proj(x)
62
- return x
63
-
64
-
65
-class QWenAttention(nn.Module):
66
-
67
- def __init__(
68
- self,
69
- hidden_size: int,
70
- num_heads: int,
71
- max_position_embeddings: int,
72
- rope_theta: float = 10000,
73
- rope_scaling: Optional[Dict[str, Any]] = None,
74
- linear_method: Optional[LinearMethodBase] = None,
75
- ):
76
- super().__init__()
77
- self.hidden_size = hidden_size
78
- tensor_model_parallel_world_size = get_tensor_model_parallel_world_size(
79
- )
80
- self.total_num_heads = num_heads
81
- assert self.total_num_heads % tensor_model_parallel_world_size == 0
82
- self.num_heads = (self.total_num_heads //
83
- tensor_model_parallel_world_size)
84
- self.head_dim = hidden_size // self.total_num_heads
85
- self.c_attn = QKVParallelLinear(
86
- hidden_size,
87
- self.head_dim,
88
- self.total_num_heads,
89
- bias=True,
90
- linear_method=linear_method,
91
- )
92
- self.c_proj = RowParallelLinear(
93
- self.total_num_heads * self.head_dim,
94
- hidden_size,
95
- bias=False,
96
- linear_method=linear_method,
97
- )
98
- self.scaling = self.head_dim**-0.5
99
-
100
- self.rotary_emb = get_rope(
101
- self.head_dim,
102
- rotary_dim=self.head_dim,
103
- max_position=max_position_embeddings,
104
- base=rope_theta,
105
- rope_scaling=rope_scaling,
106
- )
107
- self.attn = PagedAttention(self.num_heads, self.head_dim, self.scaling)
108
-
109
- def forward(
110
- self,
111
- positions: torch.Tensor,
112
- hidden_states: torch.Tensor,
113
- kv_cache: KVCache,
114
- input_metadata: InputMetadata,
115
- ) -> torch.Tensor:
116
- qkv, _ = self.c_attn(hidden_states)
117
- q, k, v = qkv.chunk(chunks=3, dim=-1)
118
- q, k = self.rotary_emb(positions, q, k)
119
- k_cache, v_cache = kv_cache
120
- attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
121
-
122
- output, _ = self.c_proj(attn_output)
123
- return output
124
-
125
-
126
-class QWenBlock(nn.Module):
127
-
128
- def __init__(
129
- self,
130
- config: QWenConfig,
131
- linear_method: Optional[LinearMethodBase] = None,
132
- ):
133
- super().__init__()
134
- self.ln_1 = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
135
-
136
- rope_theta = getattr(config, "rope_theta", 10000)
137
- rope_scaling = getattr(config, "rope_scaling", None)
138
- self.attn = QWenAttention(config.hidden_size,
139
- config.num_attention_heads,
140
- config.max_position_embeddings,
141
- rope_theta=rope_theta,
142
- rope_scaling=rope_scaling,
143
- linear_method=linear_method)
144
-
145
- self.ln_2 = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
146
-
147
- self.mlp = QWenMLP(config.hidden_size,
148
- config.intermediate_size // 2,
149
- linear_method=linear_method)
150
-
151
- def forward(
152
- self,
153
- positions: torch.Tensor,
154
- hidden_states: torch.Tensor,
155
- kv_cache: KVCache,
156
- input_metadata: InputMetadata,
157
- residual: Optional[torch.Tensor],
158
- ) -> Tuple[torch.Tensor, torch.Tensor]:
159
- # Self Attention
160
- if residual is None:
161
- residual = hidden_states
162
- hidden_states = self.ln_1(hidden_states)
163
- else:
164
- hidden_states, residual = self.ln_1(hidden_states, residual)
165
- hidden_states = self.attn(
166
- positions=positions,
167
- hidden_states=hidden_states,
168
- kv_cache=kv_cache,
169
- input_metadata=input_metadata,
170
- )
171
-
172
- # Fully Connected
173
- hidden_states, residual = self.ln_2(hidden_states, residual)
174
- hidden_states = self.mlp(hidden_states)
175
- return hidden_states, residual
176
-
177
-
178
-class QWenModel(nn.Module):
19
+class QWenLMHeadModel(LlamaForCausalLM):
20
21
def __init__(
22
self,
182
- config: QWenConfig,
23
+ config: Optional[PretrainedConfig] = None,
24
linear_method: Optional[LinearMethodBase] = None,
184
- ):
185
- super().__init__()
186
- self.config = config
187
- self.vocab_size = config.vocab_size
188
-
189
- self.wte = VocabParallelEmbedding(
190
- config.vocab_size,
191
- config.hidden_size,
192
- )
193
- self.h = nn.ModuleList([
194
- QWenBlock(config, linear_method)
195
- for _ in range(config.num_hidden_layers)
196
- ])
197
- self.ln_f = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
198
-
199
- def forward(
200
- self,
201
- input_ids: torch.Tensor,
202
- positions: torch.Tensor,
203
- kv_caches: List[KVCache],
204
- input_metadata: InputMetadata,
205
- ) -> torch.Tensor:
206
- hidden_states = self.wte(input_ids)
207
- residual = None
208
- for i in range(len(self.h)):
209
- layer = self.h[i]
210
- hidden_states, residual = layer(
211
- positions,
212
- hidden_states,
213
- kv_caches[i],
214
- input_metadata,
215
- residual,
216
- )
217
- hidden_states, _ = self.ln_f(hidden_states, residual)
218
- return hidden_states
219
-
220
-
221
-class QWenLMHeadModel(nn.Module):
222
-
223
- def __init__(
224
- self,
225
- config: QWenConfig,
226
- linear_method: Optional[LinearMethodBase] = None,
227
- ):
228
- super().__init__()
229
- self.config = config
230
- self.linear_method = linear_method
231
- self.transformer = QWenModel(config, linear_method)
232
- self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
233
- self.sampler = Sampler(config.vocab_size)
234
-
235
- def forward(
236
- self,
237
- input_ids: torch.Tensor,
238
- positions: torch.Tensor,
239
- kv_caches: List[KVCache],
240
- input_metadata: InputMetadata,
241
- ) -> torch.Tensor:
242
- hidden_states = self.transformer(input_ids, positions, kv_caches,
243
- input_metadata)
244
- return hidden_states
245
-
246
- def sample(
247
- self,
248
- hidden_states: torch.Tensor,
249
- sampling_metadata: SamplingMetadata,
250
- ) -> Optional[SamplerOutput]:
251
- next_tokens = self.sampler(self.lm_head.weight, hidden_states,
252
- sampling_metadata)
253
- return next_tokens
25
+ lora_config: Optional[LoRAConfig] = None,
26
+ ) -> None:
27
+ norm = RMSNorm(config.hidden_size, config.layer_norm_epsilon)
28
+ config.use_qkv_bias = True
29
+ config.intermediate_size = config.intermediate_size // 2
30
+ super().__init__(config=config,
31
+ linear_method=linear_method,
32
+ norm=norm,
33
+ lora_config=lora_config)
34
35
def load_weights(self,
36
model_name_or_path: str,
@@ -262,9 +42,24 @@ class QWenLMHeadModel(nn.Module):
42
("gate_up_proj", "w2", 0),
43
("gate_up_proj", "w1", 1),
44
]
45
+ param_weight_map = [
46
+ ("model", "transformer"),
47
+ (".self_attn.", ".attn."),
48
+ (".layers.", ".h."),
49
+ ("qkv_proj", "c_attn"),
50
+ (".self_attn.o_proj", ".self_attn.c_proj"),
51
+ ("norm", "ln_f"),
52
+ ("mlp.down_proj", "mlp.c_proj"),
53
+ ("input_layernorm", "ln_1"),
54
+ ("post_attention_layernorm", "ln_2"),
55
+ ("embed_tokens", "wte"),
56
+ ]
57
params_dict = dict(self.named_parameters())
58
for name, loaded_weight in hf_model_weights_iterator(
59
model_name_or_path, cache_dir, load_format, revision):
60
+ for (param_name, weight_name) in param_weight_map:
61
+ name = name.replace(weight_name, param_name)
62
+
63
if "rotary_emb.inv_freq" in name:
64
continue
65
for (param_name, weight_name, shard_id) in stacked_params_mapping:
vllm/model_executor/models/stablelm.py
+13
-270
@@ -17,283 +17,26 @@
17
# https://huggingface.co/stabilityai/stablelm-3b-4e1t/blob/main/modeling_stablelm_epoch.py
18
# https://huggingface.co/stabilityai/stablelm-3b-4e1t/blob/main/config.json
19
"""Inference-only StabeLM (https://github.com/Stability-AI/StableLM) model compatible with HuggingFace weights."""
20
-from typing import List, Optional, Tuple
20
+from typing import Optional
21
22
-import torch
23
-from torch import nn
22
from transformers import PretrainedConfig
23
26
-from vllm.model_executor.input_metadata import InputMetadata
27
-from vllm.model_executor.layers.activation import SiluAndMul
28
-from vllm.model_executor.layers.attention import PagedAttention
29
-from vllm.model_executor.layers.linear import (LinearMethodBase,
30
- MergedColumnParallelLinear,
31
- QKVParallelLinear,
32
- RowParallelLinear)
33
-from vllm.model_executor.layers.rotary_embedding import get_rope
34
-from vllm.model_executor.layers.sampler import Sampler
35
-from vllm.model_executor.layers.vocab_parallel_embedding import (
36
- VocabParallelEmbedding, ParallelLMHead)
37
-from vllm.model_executor.parallel_utils.parallel_state import (
38
- get_tensor_model_parallel_world_size)
39
-from vllm.model_executor.sampling_metadata import SamplingMetadata
40
-from vllm.model_executor.weight_utils import (default_weight_loader,
41
- hf_model_weights_iterator)
42
-from vllm.sequence import SamplerOutput
24
+from vllm.model_executor.layers.linear import LinearMethodBase
25
+from vllm.model_executor.layers.layernorm import LayerNorm
26
+from vllm.model_executor.models.llama import LlamaForCausalLM
27
+from vllm.config import LoRAConfig
28
44
-KVCache = Tuple[torch.Tensor, torch.Tensor]
29
46
-
47
-class StablelmMLP(nn.Module):
48
-
49
- def __init__(self,
50
- config: PretrainedConfig,
51
- linear_method: Optional[LinearMethodBase] = None) -> None:
52
- super().__init__()
53
- self.config = config
54
- self.hidden_size = config.hidden_size
55
- self.intermediate_size = config.intermediate_size
56
- self.gate_up_proj = MergedColumnParallelLinear(
57
- config.hidden_size, [config.intermediate_size] * 2,
58
- bias=False,
59
- linear_method=linear_method)
60
- self.down_proj = RowParallelLinear(config.intermediate_size,
61
- config.hidden_size,
62
- bias=False)
63
- self.act_fn = SiluAndMul()
64
-
65
- def forward(self, x: torch.Tensor) -> torch.Tensor:
66
- gate_up, _ = self.gate_up_proj(x)
67
- x = self.act_fn(gate_up)
68
- x, _ = self.down_proj(x)
69
- return x
70
-
71
-
72
-class StablelmAttention(nn.Module):
73
-
74
- def __init__(self,
75
- config: PretrainedConfig,
76
- linear_method: Optional[LinearMethodBase] = None) -> None:
77
- super().__init__()
78
- self.config = config
79
- self.hidden_size = config.hidden_size
80
- tp_size = get_tensor_model_parallel_world_size()
81
- self.total_num_heads = config.num_attention_heads
82
- self.num_heads = self.total_num_heads // tp_size
83
-
84
- self.total_num_key_value_heads = config.num_key_value_heads
85
- if self.total_num_key_value_heads >= tp_size:
86
- # Number of KV heads is greater than TP size, so we partition
87
- # the KV heads across multiple tensor parallel GPUs.
88
- assert self.total_num_key_value_heads % tp_size == 0
89
- else:
90
- # Number of KV heads is less than TP size, so we replicate
91
- # the KV heads across multiple tensor parallel GPUs.
92
- assert tp_size % self.total_num_key_value_heads == 0
93
- self.num_key_value_heads = max(
94
- 1, self.total_num_key_value_heads // tp_size)
95
- self.head_dim = self.hidden_size // self.total_num_heads
96
- self.max_position_embeddings = config.max_position_embeddings
97
- self.rotary_ndims = int(self.head_dim * self.config.rope_pct)
98
- self.scaling = self.head_dim**-0.5
99
- self.q_size = self.num_heads * self.head_dim
100
- self.kv_size = self.num_key_value_heads * self.head_dim
101
- self.qkv_bias = getattr(config, "use_qkv_bias", False)
102
- if (self.head_dim * self.num_heads * tp_size) != self.hidden_size:
103
- raise ValueError(
104
- f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}"
105
- f" and `num_heads`: {self.num_heads}).")
106
-
107
- self.qkv_proj = QKVParallelLinear(self.hidden_size,
108
- self.head_dim,
109
- self.total_num_heads,
110
- self.total_num_key_value_heads,
111
- self.qkv_bias,
112
- linear_method=linear_method)
113
- self.o_proj = RowParallelLinear(self.total_num_heads * self.head_dim,
114
- self.hidden_size,
115
- bias=False,
116
- linear_method=linear_method)
117
- self.rotary_ndims = int(self.head_dim * self.config.rope_pct)
118
- self.rotary_emb = get_rope(
119
- self.head_dim,
120
- rotary_dim=self.rotary_ndims,
121
- max_position=self.config.max_position_embeddings,
122
- base=self.config.rope_theta,
123
- )
124
- self.attn = PagedAttention(self.num_heads,
125
- self.head_dim,
126
- self.scaling,
127
- num_kv_heads=self.num_key_value_heads)
128
-
129
- def forward(
130
- self,
131
- positions: torch.Tensor,
132
- hidden_states: torch.Tensor,
133
- kv_cache: KVCache,
134
- input_metadata: InputMetadata,
135
- ) -> torch.Tensor:
136
- qkv, _ = self.qkv_proj(hidden_states)
137
- q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
138
- q, k = self.rotary_emb(positions, q, k)
139
- k_cache, v_cache = kv_cache
140
- attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
141
- output, _ = self.o_proj(attn_output)
142
- return output
143
-
144
-
145
-class StablelmDecoderLayer(nn.Module):
30
+class StablelmForCausalLM(LlamaForCausalLM):
31
32
def __init__(
33
self,
149
- config: PretrainedConfig,
34
+ config: Optional[PretrainedConfig] = None,
35
linear_method: Optional[LinearMethodBase] = None,
36
+ lora_config: Optional[LoRAConfig] = None,
37
) -> None:
152
- super().__init__()
153
- self.self_attn = StablelmAttention(config)
154
- self.mlp = StablelmMLP(config, linear_method)
155
- self.input_layernorm = nn.LayerNorm(config.hidden_size,
156
- eps=config.norm_eps)
157
- self.post_attention_layernorm = nn.LayerNorm(config.hidden_size,
158
- eps=config.norm_eps)
159
-
160
- def forward(
161
- self,
162
- positions: torch.Tensor,
163
- hidden_states: torch.Tensor,
164
- kv_cache: KVCache,
165
- input_metadata: InputMetadata,
166
- ) -> Tuple[torch.Tensor, torch.Tensor]:
167
- # Self Attention
168
- residual = hidden_states
169
- hidden_states = self.input_layernorm(hidden_states)
170
- hidden_states = self.self_attn(
171
- positions=positions,
172
- hidden_states=hidden_states,
173
- kv_cache=kv_cache,
174
- input_metadata=input_metadata,
175
- )
176
- hidden_states = residual + hidden_states
177
-
178
- # Fully Connected
179
- residual = hidden_states
180
- hidden_states = self.post_attention_layernorm(hidden_states)
181
- hidden_states = self.mlp(hidden_states)
182
- hidden_states = residual + hidden_states
183
-
184
- return hidden_states, residual
185
-
186
-
187
-class StableLMEpochModel(nn.Module):
188
-
189
- def __init__(self,
190
- config: PretrainedConfig,
191
- linear_method: Optional[LinearMethodBase] = None) -> None:
192
- super().__init__()
193
- # self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, config.pad_token_id)
194
- self.embed_tokens = VocabParallelEmbedding(
195
- config.vocab_size,
196
- config.hidden_size,
197
- )
198
- self.layers = nn.ModuleList([
199
- StablelmDecoderLayer(config, linear_method)
200
- for _ in range(config.num_hidden_layers)
201
- ])
202
- self.norm = nn.LayerNorm(config.hidden_size, eps=config.norm_eps)
203
-
204
- def forward(
205
- self,
206
- input_ids: torch.Tensor,
207
- positions: torch.Tensor,
208
- kv_caches: List[KVCache],
209
- input_metadata: InputMetadata,
210
- ) -> torch.Tensor:
211
- hidden_states = self.embed_tokens(input_ids)
212
- for i in range(len(self.layers)):
213
- layer = self.layers[i]
214
- hidden_states, residual = layer(
215
- positions,
216
- hidden_states,
217
- kv_caches[i],
218
- input_metadata,
219
- )
220
- hidden_states = self.norm(hidden_states)
221
- return hidden_states
222
-
223
-
224
-class StablelmForCausalLM(nn.Module):
225
-
226
- def __init__(
227
- self,
228
- config: PretrainedConfig,
229
- linear_method: Optional[LinearMethodBase] = None,
230
- ) -> None:
231
- super().__init__()
232
- self.config = config
233
- self.linear_method = linear_method
234
- self.model = StableLMEpochModel(config, linear_method)
235
- self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
236
- self.sampler = Sampler(config.vocab_size)
237
-
238
- def forward(
239
- self,
240
- input_ids: torch.Tensor,
241
- positions: torch.Tensor,
242
- kv_caches: List[KVCache],
243
- input_metadata: InputMetadata,
244
- ) -> torch.Tensor:
245
- hidden_states = self.model(input_ids, positions, kv_caches,
246
- input_metadata)
247
- return hidden_states
248
-
249
- def sample(
250
- self,
251
- hidden_states: torch.Tensor,
252
- sampling_metadata: SamplingMetadata,
253
- ) -> Optional[SamplerOutput]:
254
- next_tokens = self.sampler(self.lm_head.weight, hidden_states,
255
- sampling_metadata)
256
- return next_tokens
257
-
258
- def load_weights(self,
259
- model_name_or_path: str,
260
- cache_dir: Optional[str] = None,
261
- load_format: str = "auto",
262
- revision: Optional[str] = None):
263
- stacked_params_mapping = [
264
- # (param_name, shard_name, shard_id)
265
- ("qkv_proj", "q_proj", "q"),
266
- ("qkv_proj", "k_proj", "k"),
267
- ("qkv_proj", "v_proj", "v"),
268
- ("gate_up_proj", "gate_proj", 0),
269
- ("gate_up_proj", "up_proj", 1),
270
- ]
271
- params_dict = dict(self.named_parameters())
272
- for name, loaded_weight in hf_model_weights_iterator(
273
- model_name_or_path, cache_dir, load_format, revision):
274
- if "rotary_emb.inv_freq" in name:
275
- continue
276
- if ("rotary_emb.cos_cached" in name
277
- or "rotary_emb.sin_cached" in name):
278
- # Models trained using ColossalAI may include these tensors in
279
- # the checkpoint. Skip them.
280
- continue
281
- for (param_name, weight_name, shard_id) in stacked_params_mapping:
282
- if weight_name not in name:
283
- continue
284
- name = name.replace(weight_name, param_name)
285
- # Skip loading extra bias for GPTQ models.
286
- if name.endswith(".bias") and name not in params_dict:
287
- continue
288
- param = params_dict[name]
289
- weight_loader = param.weight_loader
290
- weight_loader(param, loaded_weight, shard_id)
291
- break
292
- else:
293
- # Skip loading extra bias for GPTQ models.
294
- if name.endswith(".bias") and name not in params_dict:
295
- continue
296
- param = params_dict[name]
297
- weight_loader = getattr(param, "weight_loader",
298
- default_weight_loader)
299
- weight_loader(param, loaded_weight)
38
+ norm = LayerNorm(config.hidden_size, config.norm_eps)
39
+ super().__init__(config=config,
40
+ linear_method=linear_method,
41
+ norm=norm,
42
+ lora_config=lora_config)
vllm/model_executor/models/yi.py
deleted
-330
@@ -1,330 +0,0 @@
1
-# coding=utf-8
2
-# Adapted from
3
-# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
4
-# Copyright 2023 The vLLM team.
5
-# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
6
-#
7
-# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
8
-# and OPT implementations in this library. It has been modified from its
9
-# original forms to accommodate minor architectural differences compared
10
-# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
11
-#
12
-# Licensed under the Apache License, Version 2.0 (the "License");
13
-# you may not use this file except in compliance with the License.
14
-# You may obtain a copy of the License at
15
-#
16
-# http://www.apache.org/licenses/LICENSE-2.0
17
-#
18
-# Unless required by applicable law or agreed to in writing, software
19
-# distributed under the License is distributed on an "AS IS" BASIS,
20
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21
-# See the License for the specific language governing permissions and
22
-# limitations under the License.
23
-"""Inference-only Yi model (https://01.ai) compatible with HuggingFace weights."""
24
-from typing import Any, Dict, List, Optional, Tuple
25
-
26
-import torch
27
-from torch import nn
28
-from vllm.transformers_utils.configs.yi import YiConfig
29
-
30
-from vllm.model_executor.input_metadata import InputMetadata
31
-from vllm.model_executor.layers.activation import SiluAndMul
32
-from vllm.model_executor.layers.attention import PagedAttention
33
-from vllm.model_executor.layers.layernorm import RMSNorm
34
-from vllm.model_executor.layers.linear import (LinearMethodBase,
35
- MergedColumnParallelLinear,
36
- QKVParallelLinear,
37
- RowParallelLinear)
38
-from vllm.model_executor.layers.rotary_embedding import get_rope
39
-from vllm.model_executor.layers.sampler import Sampler
40
-from vllm.model_executor.layers.vocab_parallel_embedding import (
41
- VocabParallelEmbedding, ParallelLMHead)
42
-from vllm.model_executor.parallel_utils.parallel_state import (
43
- get_tensor_model_parallel_world_size)
44
-from vllm.model_executor.sampling_metadata import SamplingMetadata
45
-from vllm.model_executor.weight_utils import (default_weight_loader,
46
- hf_model_weights_iterator)
47
-from vllm.sequence import SamplerOutput
48
-
49
-KVCache = Tuple[torch.Tensor, torch.Tensor]
50
-
51
-
52
-class YiMLP(nn.Module):
53
-
54
- def __init__(
55
- self,
56
- hidden_size: int,
57
- intermediate_size: int,
58
- hidden_act: str,
59
- linear_method: Optional[LinearMethodBase] = None,
60
- ) -> None:
61
- super().__init__()
62
- self.gate_up_proj = MergedColumnParallelLinear(
63
- hidden_size, [intermediate_size] * 2,
64
- bias=False,
65
- linear_method=linear_method)
66
- self.down_proj = RowParallelLinear(intermediate_size,
67
- hidden_size,
68
- bias=False,
69
- linear_method=linear_method)
70
- if hidden_act != "silu":
71
- raise ValueError(f"Unsupported activation: {hidden_act}. "
72
- "Only silu is supported for now.")
73
- self.act_fn = SiluAndMul()
74
-
75
- def forward(self, x):
76
- gate_up, _ = self.gate_up_proj(x)
77
- x = self.act_fn(gate_up)
78
- x, _ = self.down_proj(x)
79
- return x
80
-
81
-
82
-class YiAttention(nn.Module):
83
-
84
- def __init__(
85
- self,
86
- hidden_size: int,
87
- num_heads: int,
88
- num_kv_heads: int,
89
- rope_theta: float = 10000,
90
- rope_scaling: Optional[Dict[str, Any]] = None,
91
- max_position_embeddings: int = 8192,
92
- linear_method: Optional[LinearMethodBase] = None,
93
- ) -> None:
94
- super().__init__()
95
- self.hidden_size = hidden_size
96
- tp_size = get_tensor_model_parallel_world_size()
97
- self.total_num_heads = num_heads
98
- assert self.total_num_heads % tp_size == 0
99
- self.num_heads = self.total_num_heads // tp_size
100
- self.total_num_kv_heads = num_kv_heads
101
- if self.total_num_kv_heads >= tp_size:
102
- # Number of KV heads is greater than TP size, so we partition
103
- # the KV heads across multiple tensor parallel GPUs.
104
- assert self.total_num_kv_heads % tp_size == 0
105
- else:
106
- # Number of KV heads is less than TP size, so we replicate
107
- # the KV heads across multiple tensor parallel GPUs.
108
- assert tp_size % self.total_num_kv_heads == 0
109
- self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
110
- self.head_dim = hidden_size // self.total_num_heads
111
- self.q_size = self.num_heads * self.head_dim
112
- self.kv_size = self.num_kv_heads * self.head_dim
113
- self.scaling = self.head_dim**-0.5
114
- self.rope_theta = rope_theta
115
- self.max_position_embeddings = max_position_embeddings
116
-
117
- self.qkv_proj = QKVParallelLinear(
118
- hidden_size,
119
- self.head_dim,
120
- self.total_num_heads,
121
- self.total_num_kv_heads,
122
- bias=False,
123
- linear_method=linear_method,
124
- )
125
- self.o_proj = RowParallelLinear(
126
- self.total_num_heads * self.head_dim,
127
- hidden_size,
128
- bias=False,
129
- linear_method=linear_method,
130
- )
131
- self.rotary_emb = get_rope(
132
- self.head_dim,
133
- rotary_dim=self.head_dim,
134
- max_position=max_position_embeddings,
135
- base=self.rope_theta,
136
- rope_scaling=rope_scaling,
137
- )
138
- self.attn = PagedAttention(self.num_heads,
139
- self.head_dim,
140
- self.scaling,
141
- num_kv_heads=self.num_kv_heads)
142
-
143
- def forward(
144
- self,
145
- positions: torch.Tensor,
146
- hidden_states: torch.Tensor,
147
- kv_cache: KVCache,
148
- input_metadata: InputMetadata,
149
- ) -> torch.Tensor:
150
- qkv, _ = self.qkv_proj(hidden_states)
151
- q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
152
- q, k = self.rotary_emb(positions, q, k)
153
- k_cache, v_cache = kv_cache
154
- attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
155
- output, _ = self.o_proj(attn_output)
156
- return output
157
-
158
-
159
-class YiDecoderLayer(nn.Module):
160
-
161
- def __init__(
162
- self,
163
- config: YiConfig,
164
- linear_method: Optional[LinearMethodBase] = None,
165
- ) -> None:
166
- super().__init__()
167
- self.hidden_size = config.hidden_size
168
- rope_theta = getattr(config, "rope_theta", 10000)
169
- rope_scaling = getattr(config, "rope_scaling", None)
170
- max_position_embeddings = getattr(config, "max_position_embeddings",
171
- 8192)
172
- self.self_attn = YiAttention(
173
- hidden_size=self.hidden_size,
174
- num_heads=config.num_attention_heads,
175
- num_kv_heads=config.num_key_value_heads,
176
- rope_theta=rope_theta,
177
- rope_scaling=rope_scaling,
178
- max_position_embeddings=max_position_embeddings,
179
- linear_method=linear_method,
180
- )
181
- self.mlp = YiMLP(
182
- hidden_size=self.hidden_size,
183
- intermediate_size=config.intermediate_size,
184
- hidden_act=config.hidden_act,
185
- linear_method=linear_method,
186
- )
187
- self.ln1 = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
188
- self.ln2 = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
189
-
190
- def forward(
191
- self,
192
- positions: torch.Tensor,
193
- hidden_states: torch.Tensor,
194
- kv_cache: KVCache,
195
- input_metadata: InputMetadata,
196
- residual: Optional[torch.Tensor],
197
- ) -> Tuple[torch.Tensor, torch.Tensor]:
198
- # Self Attention
199
- if residual is None:
200
- residual = hidden_states
201
- hidden_states = self.ln1(hidden_states)
202
- else:
203
- hidden_states, residual = self.ln1(hidden_states, residual)
204
- hidden_states = self.self_attn(
205
- positions=positions,
206
- hidden_states=hidden_states,
207
- kv_cache=kv_cache,
208
- input_metadata=input_metadata,
209
- )
210
-
211
- # Fully Connected
212
- hidden_states, residual = self.ln2(hidden_states, residual)
213
- hidden_states = self.mlp(hidden_states)
214
- return hidden_states, residual
215
-
216
-
217
-class YiModel(nn.Module):
218
-
219
- def __init__(
220
- self,
221
- config: YiConfig,
222
- linear_method: Optional[LinearMethodBase] = None,
223
- ) -> None:
224
- super().__init__()
225
- self.config = config
226
- self.padding_idx = config.pad_token_id
227
- self.vocab_size = config.vocab_size
228
- self.embed_tokens = VocabParallelEmbedding(
229
- config.vocab_size,
230
- config.hidden_size,
231
- )
232
- self.layers = nn.ModuleList([
233
- YiDecoderLayer(config, linear_method)
234
- for _ in range(config.num_hidden_layers)
235
- ])
236
- self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
237
-
238
- def forward(
239
- self,
240
- input_ids: torch.Tensor,
241
- positions: torch.Tensor,
242
- kv_caches: List[KVCache],
243
- input_metadata: InputMetadata,
244
- ) -> torch.Tensor:
245
- hidden_states = self.embed_tokens(input_ids)
246
- residual = None
247
- for i in range(len(self.layers)):
248
- layer = self.layers[i]
249
- hidden_states, residual = layer(
250
- positions,
251
- hidden_states,
252
- kv_caches[i],
253
- input_metadata,
254
- residual,
255
- )
256
- hidden_states, _ = self.norm(hidden_states, residual)
257
- return hidden_states
258
-
259
-
260
-class YiForCausalLM(nn.Module):
261
-
262
- def __init__(
263
- self,
264
- config: YiConfig,
265
- linear_method: Optional[LinearMethodBase] = None,
266
- ) -> None:
267
- super().__init__()
268
- self.config = config
269
- self.linear_method = linear_method
270
- self.model = YiModel(config, linear_method)
271
- self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
272
- self.sampler = Sampler(config.vocab_size)
273
-
274
- def forward(
275
- self,
276
- input_ids: torch.Tensor,
277
- positions: torch.Tensor,
278
- kv_caches: List[KVCache],
279
- input_metadata: InputMetadata,
280
- ) -> torch.Tensor:
281
- hidden_states = self.model(input_ids, positions, kv_caches,
282
- input_metadata)
283
- return hidden_states
284
-
285
- def sample(
286
- self,
287
- hidden_states: torch.Tensor,
288
- sampling_metadata: SamplingMetadata,
289
- ) -> Optional[SamplerOutput]:
290
- next_tokens = self.sampler(self.lm_head.weight, hidden_states,
291
- sampling_metadata)
292
- return next_tokens
293
-
294
- def load_weights(self,
295
- model_name_or_path: str,
296
- cache_dir: Optional[str] = None,
297
- load_format: str = "auto",
298
- revision: Optional[str] = None):
299
- stacked_params_mapping = [
300
- # (param_name, shard_name, shard_id)
301
- ("qkv_proj", "q_proj", "q"),
302
- ("qkv_proj", "k_proj", "k"),
303
- ("qkv_proj", "v_proj", "v"),
304
- ("gate_up_proj", "gate_proj", 0),
305
- ("gate_up_proj", "up_proj", 1),
306
- ]
307
- params_dict = dict(self.named_parameters())
308
- for name, loaded_weight in hf_model_weights_iterator(
309
- model_name_or_path, cache_dir, load_format, revision):
310
- if "rotary_emb.inv_freq" in name:
311
- continue
312
- for (param_name, weight_name, shard_id) in stacked_params_mapping:
313
- if weight_name not in name:
314
- continue
315
- name = name.replace(weight_name, param_name)
316
- # Skip loading extra bias for GPTQ models.
317
- if name.endswith(".bias") and name not in params_dict:
318
- continue
319
- param = params_dict[name]
320
- weight_loader = param.weight_loader
321
- weight_loader(param, loaded_weight, shard_id)
322
- break
323
- else:
324
- # Skip loading extra bias for GPTQ models.
325
- if name.endswith(".bias") and name not in params_dict:
326
- continue
327
- param = params_dict[name]
328
- weight_loader = getattr(param, "weight_loader",
329
- default_weight_loader)
330
- weight_loader(param, loaded_weight)
vllm/transformers_utils/config.py
-4
@@ -5,14 +5,10 @@ from transformers import AutoConfig, PretrainedConfig
5
from vllm.transformers_utils.configs import *
6
7
_CONFIG_REGISTRY = {
8
- "aquila": AquilaConfig,
9
- "baichuan": BaiChuanConfig,
8
"chatglm": ChatGLMConfig,
9
"mpt": MPTConfig,
12
- "qwen": QWenConfig,
10
"RefinedWeb": RWConfig, # For tiiuae/falcon-40b(-instruct)
11
"RefinedWebModel": RWConfig, # For tiiuae/falcon-7b(-instruct)
15
- "yi": YiConfig,
12
}
13
14
vllm/transformers_utils/configs/__init__.py
-8
@@ -1,20 +1,12 @@
1
-from vllm.transformers_utils.configs.aquila import AquilaConfig
2
-from vllm.transformers_utils.configs.baichuan import BaiChuanConfig
1
from vllm.transformers_utils.configs.chatglm import ChatGLMConfig
2
from vllm.transformers_utils.configs.mpt import MPTConfig
5
-from vllm.transformers_utils.configs.qwen import QWenConfig
3
# RWConfig is for the original tiiuae/falcon-40b(-instruct) and
4
# tiiuae/falcon-7b(-instruct) models. Newer Falcon models will use the
5
# `FalconConfig` class from the official HuggingFace transformers library.
6
from vllm.transformers_utils.configs.falcon import RWConfig
10
-from vllm.transformers_utils.configs.yi import YiConfig
7
8
__all__ = [
13
- "AquilaConfig",
14
- "BaiChuanConfig",
9
"ChatGLMConfig",
10
"MPTConfig",
17
- "QWenConfig",
11
"RWConfig",
19
- "YiConfig",
12
]
vllm/transformers_utils/configs/aquila.py
deleted
-69
@@ -1,69 +0,0 @@
1
-# coding=utf-8
2
-# Copyright 2023 EleutherAI and the HuggingFace Inc. team. All rights reserved.
3
-#
4
-# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
5
-# and OPT implementations in this library. It has been modified from its
6
-# original forms to accommodate minor architectural differences compared
7
-# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
8
-#
9
-# Licensed under the Apache License, Version 2.0 (the "License");
10
-# you may not use this file except in compliance with the License.
11
-# You may obtain a copy of the License at
12
-#
13
-# http://www.apache.org/licenses/LICENSE-2.0
14
-#
15
-# Unless required by applicable law or agreed to in writing, software
16
-# distributed under the License is distributed on an "AS IS" BASIS,
17
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18
-# See the License for the specific language governing permissions and
19
-# limitations under the License.
20
-""" Aquila model configuration"""
21
-
22
-from transformers import PretrainedConfig
23
-
24
-
25
-class AquilaConfig(PretrainedConfig):
26
- model_type = "aquila"
27
- keys_to_ignore_at_inference = ["past_key_values"]
28
-
29
- def __init__(
30
- self,
31
- vocab_size=100008,
32
- hidden_size=4096,
33
- intermediate_size=11008,
34
- num_hidden_layers=32,
35
- num_attention_heads=32,
36
- num_key_value_heads=None,
37
- hidden_act="silu",
38
- max_position_embeddings=2048,
39
- initializer_range=0.006,
40
- rms_norm_eps=1e-5,
41
- use_cache=True,
42
- pad_token_id=0,
43
- bos_token_id=1,
44
- eos_token_id=2,
45
- tie_word_embeddings=False,
46
- **kwargs,
47
- ):
48
- self.vocab_size = vocab_size
49
- self.max_position_embeddings = max_position_embeddings
50
- self.hidden_size = hidden_size
51
- self.intermediate_size = intermediate_size
52
- self.num_hidden_layers = num_hidden_layers
53
- # for backward compatibility
54
- if num_key_value_heads is None:
55
- num_key_value_heads = num_attention_heads
56
-
57
- self.num_key_value_heads = num_key_value_heads
58
- self.num_attention_heads = num_attention_heads
59
- self.hidden_act = hidden_act
60
- self.initializer_range = initializer_range
61
- self.rms_norm_eps = rms_norm_eps
62
- self.use_cache = use_cache
63
- super().__init__(
64
- pad_token_id=pad_token_id,
65
- bos_token_id=bos_token_id,
66
- eos_token_id=eos_token_id,
67
- tie_word_embeddings=tie_word_embeddings,
68
- **kwargs,
69
- )
vllm/transformers_utils/configs/baichuan.py
deleted
-62
@@ -1,62 +0,0 @@
1
-# coding=utf-8
2
-# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
3
-#
4
-# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
5
-# and OPT implementations in this library. It has been modified from its
6
-# original forms to accommodate minor architectural differences compared
7
-# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
8
-#
9
-# Licensed under the Apache License, Version 2.0 (the "License");
10
-# you may not use this file except in compliance with the License.
11
-# You may obtain a copy of the License at
12
-#
13
-# http://www.apache.org/licenses/LICENSE-2.0
14
-#
15
-# Unless required by applicable law or agreed to in writing, software
16
-# distributed under the License is distributed on an "AS IS" BASIS,
17
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18
-# See the License for the specific language governing permissions and
19
-# limitations under the License.
20
-
21
-from transformers.configuration_utils import PretrainedConfig
22
-
23
-
24
-class BaiChuanConfig(PretrainedConfig):
25
- model_type = "baichuan"
26
- keys_to_ignore_at_inference = ["past_key_values"]
27
-
28
- def __init__(
29
- self,
30
- vocab_size=64000,
31
- hidden_size=4096,
32
- intermediate_size=11008,
33
- num_hidden_layers=32,
34
- num_attention_heads=32,
35
- hidden_act="silu",
36
- max_position_embeddings=4096,
37
- initializer_range=0.02,
38
- rms_norm_eps=1e-6,
39
- use_cache=True,
40
- pad_token_id=0,
41
- bos_token_id=1,
42
- eos_token_id=2,
43
- tie_word_embeddings=False,
44
- **kwargs,
45
- ):
46
- self.vocab_size = vocab_size
47
- self.max_position_embeddings = max_position_embeddings
48
- self.hidden_size = hidden_size
49
- self.intermediate_size = intermediate_size
50
- self.num_hidden_layers = num_hidden_layers
51
- self.num_attention_heads = num_attention_heads
52
- self.hidden_act = hidden_act
53
- self.initializer_range = initializer_range
54
- self.rms_norm_eps = rms_norm_eps
55
- self.use_cache = use_cache
56
- super().__init__(
57
- pad_token_id=pad_token_id,
58
- bos_token_id=bos_token_id,
59
- eos_token_id=eos_token_id,
60
- tie_word_embeddings=tie_word_embeddings,
61
- **kwargs,
62
- )
vllm/transformers_utils/configs/qwen.py
deleted
-60
@@ -1,60 +0,0 @@
1
-# Copyright (c) Alibaba Cloud.
2
-# LICENSE: https://huggingface.co/Qwen/Qwen-7B/blob/main/LICENSE
3
-
4
-from transformers import PretrainedConfig
5
-
6
-
7
-class QWenConfig(PretrainedConfig):
8
- model_type = "qwen"
9
- keys_to_ignore_at_inference = ["past_key_values"]
10
-
11
- def __init__(
12
- self,
13
- vocab_size=151936,
14
- hidden_size=4096,
15
- num_hidden_layers=32,
16
- num_attention_heads=32,
17
- emb_dropout_prob=0.0,
18
- attn_dropout_prob=0.0,
19
- layer_norm_epsilon=1e-6,
20
- initializer_range=0.02,
21
- max_position_embeddings=8192,
22
- scale_attn_weights=True,
23
- use_cache=True,
24
- bf16=False,
25
- fp16=False,
26
- fp32=False,
27
- kv_channels=128,
28
- rotary_pct=1.0,
29
- rotary_emb_base=10000,
30
- use_dynamic_ntk=True,
31
- use_logn_attn=True,
32
- use_flash_attn="auto",
33
- intermediate_size=22016,
34
- no_bias=True,
35
- tie_word_embeddings=False,
36
- **kwargs,
37
- ):
38
- self.vocab_size = vocab_size
39
- self.hidden_size = hidden_size
40
- self.intermediate_size = intermediate_size
41
- self.num_hidden_layers = num_hidden_layers
42
- self.num_attention_heads = num_attention_heads
43
- self.emb_dropout_prob = emb_dropout_prob
44
- self.attn_dropout_prob = attn_dropout_prob
45
- self.layer_norm_epsilon = layer_norm_epsilon
46
- self.initializer_range = initializer_range
47
- self.scale_attn_weights = scale_attn_weights
48
- self.use_cache = use_cache
49
- self.max_position_embeddings = max_position_embeddings
50
- self.bf16 = bf16
51
- self.fp16 = fp16
52
- self.fp32 = fp32
53
- self.kv_channels = kv_channels
54
- self.rotary_pct = rotary_pct
55
- self.rotary_emb_base = rotary_emb_base
56
- self.use_dynamic_ntk = use_dynamic_ntk
57
- self.use_logn_attn = use_logn_attn
58
- self.use_flash_attn = use_flash_attn
59
- self.no_bias = no_bias
60
- super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)
vllm/transformers_utils/configs/yi.py
deleted
-64
@@ -1,64 +0,0 @@
1
-""" Yi model configuration"""
2
-from transformers.configuration_utils import PretrainedConfig
3
-from transformers.utils import logging
4
-
5
-logger = logging.get_logger(__name__)
6
-
7
-Yi_PRETRAINED_CONFIG_ARCHIVE_MAP = {}
8
-
9
-
10
-class YiConfig(PretrainedConfig):
11
- r"""
12
- Reference:
13
- https://huggingface.co/01-ai/Yi-6B/blob/main/configuration_yi.py
14
- """
15
- model_type = "Yi"
16
- keys_to_ignore_at_inference = ["past_key_values"]
17
-
18
- def __init__(
19
- self,
20
- vocab_size=64000,
21
- hidden_size=4096,
22
- intermediate_size=11008,
23
- num_hidden_layers=32,
24
- num_attention_heads=32,
25
- num_key_value_heads=4,
26
- hidden_act="silu",
27
- max_position_embeddings=4096,
28
- initializer_range=0.02,
29
- rms_norm_eps=1e-5,
30
- use_cache=True,
31
- pad_token_id=0,
32
- bos_token_id=1,
33
- eos_token_id=2,
34
- tie_word_embeddings=False,
35
- output_attentions=False,
36
- rope_theta=5000000.0,
37
- **kwargs,
38
- ):
39
- self.vocab_size = vocab_size
40
- self.max_position_embeddings = max_position_embeddings
41
- self.hidden_size = hidden_size
42
- self.intermediate_size = intermediate_size
43
- self.num_hidden_layers = num_hidden_layers
44
- self.num_attention_heads = num_attention_heads
45
-
46
- # for backward compatibility
47
- if num_key_value_heads is None:
48
- num_key_value_heads = num_attention_heads
49
-
50
- self.num_key_value_heads = num_key_value_heads
51
- self.hidden_act = hidden_act
52
- self.initializer_range = initializer_range
53
- self.rms_norm_eps = rms_norm_eps
54
- self.use_cache = use_cache
55
- self.output_attentions = output_attentions
56
- self.rope_theta = rope_theta
57
-
58
- super().__init__(
59
- pad_token_id=pad_token_id,
60
- bos_token_id=bos_token_id,
61
- eos_token_id=eos_token_id,
62
- tie_word_embeddings=tie_word_embeddings,
63
- **kwargs,
64
- )