A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Revert "Refactor llama family models (#2637)" (#2851)
This reverts commit 5c976a7e1a1bec875bf6474824b7dff39e38de18.
Philipp Moritz committed
Feb 13, 2024 at 09:24 UTC
ea356004d4749627bd1c65b7f71c76f51b5c45be
17 files changed
+2720
-236
vllm/model_executor/layers/layernorm.py
-25
@@ -7,31 +7,6 @@ import torch.nn as nn
7
from vllm._C import ops
8
9
10
-class LayerNorm(nn.LayerNorm):
11
-
12
- def __init__(
13
- self,
14
- hidden_size: int,
15
- eps: float = 1e-6,
16
- ) -> None:
17
- super().__init__(hidden_size, eps=eps)
18
-
19
- def forward(
20
- self,
21
- x: torch.Tensor,
22
- residual: Optional[torch.Tensor] = None,
23
- ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
24
- """normalization."""
25
- if residual is not None:
26
- x = x + residual
27
- residual = x
28
- x = super().forward(x)
29
- if residual is None:
30
- return x
31
- else:
32
- return x, residual
33
-
34
-
10
class RMSNorm(nn.Module):
11
"""Root mean square normalization.
12
vllm/model_executor/models/__init__.py
+5
-4
@@ -10,8 +10,8 @@ logger = init_logger(__name__)
10
11
# Architecture -> (module, class).
12
_MODELS = {
13
- "AquilaModel": ("llama", "LlamaForCausalLM"),
14
- "AquilaForCausalLM": ("llama", "LlamaForCausalLM"), # AquilaChat2
13
+ "AquilaModel": ("aquila", "AquilaForCausalLM"),
14
+ "AquilaForCausalLM": ("aquila", "AquilaForCausalLM"), # AquilaChat2
15
"BaiChuanForCausalLM": ("baichuan", "BaiChuanForCausalLM"), # baichuan-7b
16
"BaichuanForCausalLM": ("baichuan", "BaichuanForCausalLM"), # baichuan-13b
17
"BloomForCausalLM": ("bloom", "BloomForCausalLM"),
@@ -24,12 +24,12 @@ _MODELS = {
24
"GPTBigCodeForCausalLM": ("gpt_bigcode", "GPTBigCodeForCausalLM"),
25
"GPTJForCausalLM": ("gpt_j", "GPTJForCausalLM"),
26
"GPTNeoXForCausalLM": ("gpt_neox", "GPTNeoXForCausalLM"),
27
- "InternLMForCausalLM": ("llama", "LlamaForCausalLM"),
27
+ "InternLMForCausalLM": ("internlm", "InternLMForCausalLM"),
28
"InternLM2ForCausalLM": ("internlm2", "InternLM2ForCausalLM"),
29
"LlamaForCausalLM": ("llama", "LlamaForCausalLM"),
30
# For decapoda-research/llama-*
31
"LLaMAForCausalLM": ("llama", "LlamaForCausalLM"),
32
- "MistralForCausalLM": ("llama", "LlamaForCausalLM"),
32
+ "MistralForCausalLM": ("mistral", "MistralForCausalLM"),
33
"MixtralForCausalLM": ("mixtral", "MixtralForCausalLM"),
34
"QuantMixtralForCausalLM": ("mixtral_quant", "MixtralForCausalLM"),
35
# transformers's mpt class has lower case
@@ -41,6 +41,7 @@ _MODELS = {
41
"Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
42
"RWForCausalLM": ("falcon", "FalconForCausalLM"),
43
"StableLMEpochForCausalLM": ("stablelm", "StablelmForCausalLM"),
44
+ "YiForCausalLM": ("yi", "YiForCausalLM")
45
}
46
47
# Models not supported by ROCm.
vllm/model_executor/models/aquila.py
new
+342
@@ -0,0 +1,342 @@
1
+# coding=utf-8
2
+# Adapted from
3
+# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
4
+# Copyright 2023 The vLLM team.
5
+# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
6
+#
7
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
8
+# and OPT implementations in this library. It has been modified from its
9
+# original forms to accommodate minor architectural differences compared
10
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
11
+#
12
+# Licensed under the Apache License, Version 2.0 (the "License");
13
+# you may not use this file except in compliance with the License.
14
+# You may obtain a copy of the License at
15
+#
16
+# http://www.apache.org/licenses/LICENSE-2.0
17
+#
18
+# Unless required by applicable law or agreed to in writing, software
19
+# distributed under the License is distributed on an "AS IS" BASIS,
20
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21
+# See the License for the specific language governing permissions and
22
+# limitations under the License.
23
+"""Inference-only LLaMA model compatible with HuggingFace weights."""
24
+from typing import Any, Dict, List, Optional, Tuple
25
+
26
+import torch
27
+from torch import nn
28
+
29
+from vllm.model_executor.input_metadata import InputMetadata
30
+from vllm.model_executor.layers.activation import SiluAndMul
31
+from vllm.model_executor.layers.attention import PagedAttention
32
+from vllm.model_executor.layers.linear import (LinearMethodBase,
33
+ MergedColumnParallelLinear,
34
+ QKVParallelLinear,
35
+ RowParallelLinear)
36
+from vllm.model_executor.layers.rotary_embedding import get_rope
37
+from vllm.model_executor.layers.sampler import Sampler
38
+from vllm.model_executor.layers.vocab_parallel_embedding import (
39
+ VocabParallelEmbedding, ParallelLMHead)
40
+from vllm.model_executor.parallel_utils.parallel_state import (
41
+ get_tensor_model_parallel_world_size)
42
+from vllm.model_executor.sampling_metadata import SamplingMetadata
43
+from vllm.model_executor.weight_utils import (default_weight_loader,
44
+ hf_model_weights_iterator)
45
+from vllm.sequence import SamplerOutput
46
+from vllm.transformers_utils.configs.aquila import AquilaConfig
47
+
48
+KVCache = Tuple[torch.Tensor, torch.Tensor]
49
+
50
+
51
+class AquilaMLP(nn.Module):
52
+
53
+ def __init__(
54
+ self,
55
+ hidden_size: int,
56
+ intermediate_size: int,
57
+ hidden_act: str,
58
+ linear_method: Optional[LinearMethodBase] = None,
59
+ ):
60
+ super().__init__()
61
+ self.gate_up_proj = MergedColumnParallelLinear(
62
+ hidden_size, [intermediate_size] * 2,
63
+ bias=False,
64
+ linear_method=linear_method)
65
+ self.down_proj = RowParallelLinear(intermediate_size,
66
+ hidden_size,
67
+ bias=False,
68
+ linear_method=linear_method)
69
+ if hidden_act != "silu":
70
+ raise ValueError(f"Unsupported activation: {hidden_act}. "
71
+ "Only silu is supported for now.")
72
+ self.act_fn = SiluAndMul()
73
+
74
+ def forward(self, x):
75
+ gate_up, _ = self.gate_up_proj(x)
76
+ x = self.act_fn(gate_up)
77
+ x, _ = self.down_proj(x)
78
+ return x
79
+
80
+
81
+class AquilaRMSNorm(nn.Module):
82
+
83
+ def __init__(self, hidden_size, eps=1e-6):
84
+ """
85
+ AquilaRMSNorm is equivalent to T5LayerNorm
86
+ """
87
+ super().__init__()
88
+ self.weight = nn.Parameter(torch.ones(hidden_size))
89
+ self.variance_epsilon = eps
90
+
91
+ def forward(self, hidden_states):
92
+ input_dtype = hidden_states.dtype
93
+ variance = hidden_states.to(torch.float32).pow(2).mean(-1,
94
+ keepdim=True)
95
+ hidden_states = hidden_states * torch.rsqrt(variance +
96
+ self.variance_epsilon)
97
+
98
+ return (self.weight * hidden_states).to(input_dtype)
99
+
100
+
101
+class AquilaAttention(nn.Module):
102
+
103
+ def __init__(
104
+ self,
105
+ hidden_size: int,
106
+ num_heads: int,
107
+ num_kv_heads: int,
108
+ rope_theta: float = 10000,
109
+ max_position_embeddings: int = 8192,
110
+ rope_scaling: Optional[Dict[str, Any]] = None,
111
+ linear_method: Optional[LinearMethodBase] = None,
112
+ ):
113
+ super().__init__()
114
+ self.hidden_size = hidden_size
115
+ tp_size = get_tensor_model_parallel_world_size()
116
+ self.total_num_heads = num_heads
117
+ assert self.total_num_heads % tp_size == 0
118
+ self.num_heads = self.total_num_heads // tp_size
119
+ self.total_num_kv_heads = num_kv_heads
120
+ assert self.total_num_kv_heads % tp_size == 0
121
+ self.num_kv_heads = self.total_num_kv_heads // tp_size
122
+ self.head_dim = hidden_size // self.total_num_heads
123
+ self.q_size = self.num_heads * self.head_dim
124
+ self.kv_size = self.num_kv_heads * self.head_dim
125
+ self.scaling = self.head_dim**-0.5
126
+ self.rope_theta = rope_theta
127
+ self.max_position_embeddings = max_position_embeddings
128
+
129
+ self.qkv_proj = QKVParallelLinear(
130
+ hidden_size,
131
+ self.head_dim,
132
+ self.total_num_heads,
133
+ self.total_num_kv_heads,
134
+ bias=False,
135
+ linear_method=linear_method,
136
+ )
137
+ self.o_proj = RowParallelLinear(
138
+ self.total_num_heads * self.head_dim,
139
+ hidden_size,
140
+ bias=False,
141
+ linear_method=linear_method,
142
+ )
143
+ self.rotary_emb = get_rope(
144
+ self.head_dim,
145
+ rotary_dim=self.head_dim,
146
+ max_position=self.max_position_embeddings,
147
+ base=self.rope_theta,
148
+ rope_scaling=rope_scaling,
149
+ )
150
+ self.attn = PagedAttention(self.num_heads,
151
+ self.head_dim,
152
+ self.scaling,
153
+ num_kv_heads=self.num_kv_heads)
154
+
155
+ def forward(
156
+ self,
157
+ positions: torch.Tensor,
158
+ hidden_states: torch.Tensor,
159
+ kv_cache: KVCache,
160
+ input_metadata: InputMetadata,
161
+ ) -> torch.Tensor:
162
+ qkv, _ = self.qkv_proj(hidden_states)
163
+ q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
164
+ q, k = self.rotary_emb(positions, q, k)
165
+ k_cache, v_cache = kv_cache
166
+ attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
167
+ output, _ = self.o_proj(attn_output)
168
+ return output
169
+
170
+
171
+class AquilaDecoderLayer(nn.Module):
172
+
173
+ def __init__(
174
+ self,
175
+ config: AquilaConfig,
176
+ linear_method: Optional[LinearMethodBase] = None,
177
+ ):
178
+ super().__init__()
179
+ self.hidden_size = config.hidden_size
180
+ rope_theta = getattr(config, "rope_theta", 10000)
181
+ rope_scaling = getattr(config, "rope_scaling", None)
182
+ max_position_embeddings = getattr(config, "max_position_embeddings",
183
+ 8192)
184
+ self.self_attn = AquilaAttention(
185
+ hidden_size=self.hidden_size,
186
+ num_heads=config.num_attention_heads,
187
+ num_kv_heads=config.num_key_value_heads,
188
+ rope_theta=rope_theta,
189
+ max_position_embeddings=max_position_embeddings,
190
+ rope_scaling=rope_scaling,
191
+ linear_method=linear_method,
192
+ )
193
+ self.mlp = AquilaMLP(
194
+ hidden_size=self.hidden_size,
195
+ intermediate_size=config.intermediate_size,
196
+ hidden_act=config.hidden_act,
197
+ linear_method=linear_method,
198
+ )
199
+ self.input_layernorm = AquilaRMSNorm(config.hidden_size,
200
+ eps=config.rms_norm_eps)
201
+ self.post_attention_layernorm = AquilaRMSNorm(config.hidden_size,
202
+ eps=config.rms_norm_eps)
203
+
204
+ def forward(
205
+ self,
206
+ positions: torch.Tensor,
207
+ hidden_states: torch.Tensor,
208
+ kv_cache: KVCache,
209
+ input_metadata: InputMetadata,
210
+ ) -> torch.Tensor:
211
+ # Self Attention
212
+ residual = hidden_states
213
+ hidden_states = self.input_layernorm(hidden_states)
214
+ hidden_states = self.self_attn(
215
+ positions=positions,
216
+ hidden_states=hidden_states,
217
+ kv_cache=kv_cache,
218
+ input_metadata=input_metadata,
219
+ )
220
+ hidden_states = residual + hidden_states
221
+
222
+ # Fully Connected
223
+ residual = hidden_states
224
+ hidden_states = self.post_attention_layernorm(hidden_states)
225
+ hidden_states = self.mlp(hidden_states)
226
+ hidden_states = residual + hidden_states
227
+ return hidden_states
228
+
229
+
230
+class AquilaModel(nn.Module):
231
+
232
+ def __init__(
233
+ self,
234
+ config: AquilaConfig,
235
+ linear_method: Optional[LinearMethodBase] = None,
236
+ ):
237
+ super().__init__()
238
+ self.config = config
239
+ self.padding_idx = config.pad_token_id
240
+ self.vocab_size = config.vocab_size
241
+ self.embed_tokens = VocabParallelEmbedding(
242
+ config.vocab_size,
243
+ config.hidden_size,
244
+ )
245
+ self.layers = nn.ModuleList([
246
+ AquilaDecoderLayer(config, linear_method)
247
+ for _ in range(config.num_hidden_layers)
248
+ ])
249
+ self.norm = AquilaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
250
+
251
+ def forward(
252
+ self,
253
+ input_ids: torch.Tensor,
254
+ positions: torch.Tensor,
255
+ kv_caches: List[KVCache],
256
+ input_metadata: InputMetadata,
257
+ ) -> torch.Tensor:
258
+ hidden_states = self.embed_tokens(input_ids)
259
+ for i in range(len(self.layers)):
260
+ layer = self.layers[i]
261
+ hidden_states = layer(
262
+ positions,
263
+ hidden_states,
264
+ kv_caches[i],
265
+ input_metadata,
266
+ )
267
+ hidden_states = self.norm(hidden_states)
268
+
269
+ return hidden_states
270
+
271
+
272
+class AquilaForCausalLM(nn.Module):
273
+
274
+ def __init__(
275
+ self,
276
+ config,
277
+ linear_method: Optional[LinearMethodBase] = None,
278
+ ):
279
+ super().__init__()
280
+ self.config = config
281
+ self.linear_method = linear_method
282
+ self.model = AquilaModel(config, linear_method)
283
+ self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
284
+ self.sampler = Sampler(config.vocab_size)
285
+
286
+ def forward(
287
+ self,
288
+ input_ids: torch.Tensor,
289
+ positions: torch.Tensor,
290
+ kv_caches: List[KVCache],
291
+ input_metadata: InputMetadata,
292
+ ) -> torch.Tensor:
293
+ hidden_states = self.model(input_ids, positions, kv_caches,
294
+ input_metadata)
295
+ return hidden_states
296
+
297
+ def sample(
298
+ self,
299
+ hidden_states: torch.Tensor,
300
+ sampling_metadata: SamplingMetadata,
301
+ ) -> Optional[SamplerOutput]:
302
+ next_tokens = self.sampler(self.lm_head.weight, hidden_states,
303
+ sampling_metadata)
304
+ return next_tokens
305
+
306
+ def load_weights(self,
307
+ model_name_or_path: str,
308
+ cache_dir: Optional[str] = None,
309
+ load_format: str = "auto",
310
+ revision: Optional[str] = None):
311
+ stacked_params_mapping = [
312
+ # (param_name, shard_name, shard_id)
313
+ ("qkv_proj", "q_proj", "q"),
314
+ ("qkv_proj", "k_proj", "k"),
315
+ ("qkv_proj", "v_proj", "v"),
316
+ ("gate_up_proj", "gate_proj", 0),
317
+ ("gate_up_proj", "up_proj", 1),
318
+ ]
319
+ params_dict = dict(self.named_parameters())
320
+ for name, loaded_weight in hf_model_weights_iterator(
321
+ model_name_or_path, cache_dir, load_format, revision):
322
+ if "rotary_emb.inv_freq" in name:
323
+ continue
324
+ for (param_name, weight_name, shard_id) in stacked_params_mapping:
325
+ if weight_name not in name:
326
+ continue
327
+ name = name.replace(weight_name, param_name)
328
+ # Skip loading extra bias for GPTQ models.
329
+ if name.endswith(".bias") and name not in params_dict:
330
+ continue
331
+ param = params_dict[name]
332
+ weight_loader = param.weight_loader
333
+ weight_loader(param, loaded_weight, shard_id)
334
+ break
335
+ else:
336
+ # Skip loading extra bias for GPTQ models.
337
+ if name.endswith(".bias") and name not in params_dict:
338
+ continue
339
+ param = params_dict[name]
340
+ weight_loader = getattr(param, "weight_loader",
341
+ default_weight_loader)
342
+ weight_loader(param, loaded_weight)
vllm/model_executor/models/baichuan.py
+303
-32
@@ -18,19 +18,305 @@
18
# See the License for the specific language governing permissions and
19
# limitations under the License.
20
"""Inference-only BaiChuan model compatible with HuggingFace weights."""
21
-from typing import Optional
21
+import math
22
+from typing import List, Optional, Tuple
23
24
import torch
24
-from transformers import PretrainedConfig
25
-from vllm.config import LoRAConfig
25
+from torch import nn
26
27
-from vllm.model_executor.layers.linear import LinearMethodBase
28
-from vllm.model_executor.models.llama import LlamaForCausalLM
27
+from vllm.model_executor.input_metadata import InputMetadata
28
+from vllm.model_executor.layers.activation import SiluAndMul
29
+from vllm.model_executor.layers.attention import PagedAttention
30
+from vllm.model_executor.layers.layernorm import RMSNorm
31
+from vllm.model_executor.layers.linear import (LinearMethodBase,
32
+ MergedColumnParallelLinear,
33
+ QKVParallelLinear,
34
+ RowParallelLinear)
35
+from vllm.model_executor.layers.rotary_embedding import get_rope
36
+from vllm.model_executor.layers.sampler import Sampler
37
+from vllm.model_executor.layers.vocab_parallel_embedding import (
38
+ VocabParallelEmbedding, ParallelLMHead)
39
+from vllm.model_executor.parallel_utils.parallel_state import (
40
+ get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
41
+from vllm.model_executor.sampling_metadata import SamplingMetadata
42
from vllm.model_executor.weight_utils import (default_weight_loader,
43
hf_model_weights_iterator)
44
+from vllm.sequence import SamplerOutput
45
+from vllm.transformers_utils.configs.baichuan import BaiChuanConfig
46
47
+KVCache = Tuple[torch.Tensor, torch.Tensor]
48
33
-class BaiChuanBaseForCausalLM(LlamaForCausalLM):
49
+
50
+def _get_alibi_slopes(total_num_heads: int) -> torch.Tensor:
51
+ closest_power_of_2 = 2**math.floor(math.log2(total_num_heads))
52
+ base = torch.tensor(
53
+ 2**(-(2**-(math.log2(closest_power_of_2) - 3))),
54
+ dtype=torch.float32,
55
+ )
56
+ powers = torch.arange(1, 1 + closest_power_of_2, dtype=torch.int32)
57
+ slopes = torch.pow(base, powers)
58
+
59
+ if closest_power_of_2 != total_num_heads:
60
+ extra_base = torch.tensor(
61
+ 2**(-(2**-(math.log2(2 * closest_power_of_2) - 3))),
62
+ dtype=torch.float32,
63
+ )
64
+ num_remaining_heads = min(closest_power_of_2,
65
+ total_num_heads - closest_power_of_2)
66
+ extra_powers = torch.arange(start=1,
67
+ end=1 + 2 * num_remaining_heads,
68
+ step=2,
69
+ dtype=torch.int32)
70
+ slopes = torch.cat(
71
+ [slopes, torch.pow(extra_base, extra_powers)], dim=0)
72
+ return slopes
73
+
74
+
75
+class BaiChuanMLP(nn.Module):
76
+
77
+ def __init__(
78
+ self,
79
+ hidden_size: int,
80
+ intermediate_size: int,
81
+ hidden_act: str,
82
+ linear_method: Optional[LinearMethodBase] = None,
83
+ ):
84
+ super().__init__()
85
+ self.gate_up_proj = MergedColumnParallelLinear(
86
+ hidden_size, [intermediate_size] * 2,
87
+ bias=False,
88
+ linear_method=linear_method)
89
+ self.down_proj = RowParallelLinear(intermediate_size,
90
+ hidden_size,
91
+ bias=False,
92
+ linear_method=linear_method)
93
+ if hidden_act != "silu":
94
+ raise ValueError(f"Unsupported activation: {hidden_act}. "
95
+ "Only silu is supported for now.")
96
+ self.act_fn = SiluAndMul()
97
+
98
+ def forward(self, x):
99
+ gate_up, _ = self.gate_up_proj(x)
100
+ x = self.act_fn(gate_up)
101
+ x, _ = self.down_proj(x)
102
+ return x
103
+
104
+
105
+class BaiChuanAttention(nn.Module):
106
+ """Multi-headed attention from 'Attention Is All You Need' paper"""
107
+
108
+ def __init__(
109
+ self,
110
+ hidden_size: int,
111
+ num_heads: int,
112
+ position_embedding: str,
113
+ rope_theta: float = 10000,
114
+ max_position_embeddings: int = 8192,
115
+ linear_method: Optional[LinearMethodBase] = None,
116
+ ):
117
+ super().__init__()
118
+ self.hidden_size = hidden_size
119
+ tensor_model_parallel_world_size = get_tensor_model_parallel_world_size(
120
+ )
121
+ self.total_num_heads = num_heads
122
+ assert self.total_num_heads % tensor_model_parallel_world_size == 0
123
+ self.num_heads = (self.total_num_heads //
124
+ tensor_model_parallel_world_size)
125
+ self.head_dim = hidden_size // self.total_num_heads
126
+ self.postion_embedding = position_embedding
127
+ self.rope_theta = rope_theta
128
+ self.max_position_embeddings = max_position_embeddings
129
+
130
+ # pylint: disable=invalid-name
131
+ self.W_pack = QKVParallelLinear(
132
+ hidden_size,
133
+ self.head_dim,
134
+ self.total_num_heads,
135
+ self.total_num_heads,
136
+ bias=False,
137
+ linear_method=linear_method,
138
+ )
139
+ self.o_proj = RowParallelLinear(
140
+ self.total_num_heads * self.head_dim,
141
+ hidden_size,
142
+ bias=False,
143
+ linear_method=linear_method,
144
+ )
145
+ # Create the alibi slopes and slice them.
146
+ if self.postion_embedding == "ALIBI":
147
+ tp_rank = get_tensor_model_parallel_rank()
148
+ head_start = tp_rank * self.num_heads
149
+ head_end = (tp_rank + 1) * self.num_heads
150
+ alibi_slopes = _get_alibi_slopes(self.total_num_heads)
151
+ alibi_slopes = alibi_slopes[head_start:head_end].tolist()
152
+
153
+ scaling = self.head_dim**-0.5
154
+ self.attn = PagedAttention(self.num_heads,
155
+ self.head_dim,
156
+ scaling,
157
+ alibi_slopes=alibi_slopes)
158
+ else:
159
+ self.rotary_emb = get_rope(
160
+ self.head_dim,
161
+ rotary_dim=self.head_dim,
162
+ max_position=self.max_position_embeddings,
163
+ base=self.rope_theta,
164
+ )
165
+ self.scaling = self.head_dim**-0.5
166
+ self.attn = PagedAttention(self.num_heads, self.head_dim,
167
+ self.scaling)
168
+
169
+ def forward(
170
+ self,
171
+ positions: torch.Tensor,
172
+ hidden_states: torch.Tensor,
173
+ kv_cache: KVCache,
174
+ input_metadata: InputMetadata,
175
+ ) -> torch.Tensor:
176
+ qkv, _ = self.W_pack(hidden_states)
177
+ q, k, v = qkv.chunk(chunks=3, dim=-1)
178
+ if self.postion_embedding != "ALIBI":
179
+ q, k = self.rotary_emb(positions, q, k)
180
+ k_cache, v_cache = kv_cache
181
+ attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
182
+ output, _ = self.o_proj(attn_output)
183
+ return output
184
+
185
+
186
+class BaiChuanDecoderLayer(nn.Module):
187
+
188
+ def __init__(self,
189
+ config: BaiChuanConfig,
190
+ position_embedding: str,
191
+ linear_method: Optional[LinearMethodBase] = None):
192
+ super().__init__()
193
+ self.hidden_size = config.hidden_size
194
+ rope_theta = getattr(config, "rope_theta", 10000)
195
+ max_position_embeddings = getattr(config, "max_position_embeddings",
196
+ 8192)
197
+ self.self_attn = BaiChuanAttention(
198
+ hidden_size=self.hidden_size,
199
+ num_heads=config.num_attention_heads,
200
+ position_embedding=position_embedding,
201
+ rope_theta=rope_theta,
202
+ max_position_embeddings=max_position_embeddings,
203
+ linear_method=linear_method,
204
+ )
205
+ self.mlp = BaiChuanMLP(
206
+ hidden_size=self.hidden_size,
207
+ intermediate_size=config.intermediate_size,
208
+ hidden_act=config.hidden_act,
209
+ linear_method=linear_method,
210
+ )
211
+ self.input_layernorm = RMSNorm(config.hidden_size,
212
+ eps=config.rms_norm_eps)
213
+ self.post_attention_layernorm = RMSNorm(config.hidden_size,
214
+ eps=config.rms_norm_eps)
215
+
216
+ def forward(
217
+ self,
218
+ positions: torch.Tensor,
219
+ hidden_states: torch.Tensor,
220
+ kv_cache: KVCache,
221
+ input_metadata: InputMetadata,
222
+ residual: Optional[torch.Tensor],
223
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
224
+ # Self Attention
225
+ if residual is None:
226
+ residual = hidden_states
227
+ hidden_states = self.input_layernorm(hidden_states)
228
+ else:
229
+ hidden_states, residual = self.input_layernorm(
230
+ hidden_states, residual)
231
+ hidden_states = self.self_attn(
232
+ positions=positions,
233
+ hidden_states=hidden_states,
234
+ kv_cache=kv_cache,
235
+ input_metadata=input_metadata,
236
+ )
237
+
238
+ # Fully Connected
239
+ hidden_states, residual = self.post_attention_layernorm(
240
+ hidden_states, residual)
241
+ hidden_states = self.mlp(hidden_states)
242
+ return hidden_states, residual
243
+
244
+
245
+class BaiChuanModel(nn.Module):
246
+
247
+ def __init__(self,
248
+ config: BaiChuanConfig,
249
+ position_embedding: str,
250
+ linear_method: Optional[LinearMethodBase] = None):
251
+ super().__init__()
252
+ self.config = config
253
+ self.padding_idx = config.pad_token_id
254
+ self.vocab_size = config.vocab_size
255
+
256
+ self.embed_tokens = VocabParallelEmbedding(
257
+ config.vocab_size,
258
+ config.hidden_size,
259
+ )
260
+ self.layers = nn.ModuleList([
261
+ BaiChuanDecoderLayer(config, position_embedding, linear_method)
262
+ for _ in range(config.num_hidden_layers)
263
+ ])
264
+ self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
265
+
266
+ def forward(
267
+ self,
268
+ input_ids: torch.Tensor,
269
+ positions: torch.Tensor,
270
+ kv_caches: List[KVCache],
271
+ input_metadata: InputMetadata,
272
+ ) -> torch.Tensor:
273
+ hidden_states = self.embed_tokens(input_ids)
274
+ residual = None
275
+ for i in range(len(self.layers)):
276
+ layer = self.layers[i]
277
+ hidden_states, residual = layer(
278
+ positions,
279
+ hidden_states,
280
+ kv_caches[i],
281
+ input_metadata,
282
+ residual,
283
+ )
284
+ hidden_states, _ = self.norm(hidden_states, residual)
285
+ return hidden_states
286
+
287
+
288
+class BaiChuanBaseForCausalLM(nn.Module):
289
+
290
+ def __init__(self,
291
+ config,
292
+ position_embedding: str,
293
+ linear_method: Optional[LinearMethodBase] = None):
294
+ super().__init__()
295
+ self.config = config
296
+ self.linear_method = linear_method
297
+ self.model = BaiChuanModel(config, position_embedding, linear_method)
298
+ self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
299
+ self.sampler = Sampler(config.vocab_size)
300
+
301
+ def forward(
302
+ self,
303
+ input_ids: torch.Tensor,
304
+ positions: torch.Tensor,
305
+ kv_caches: List[KVCache],
306
+ input_metadata: InputMetadata,
307
+ ) -> torch.Tensor:
308
+ hidden_states = self.model(input_ids, positions, kv_caches,
309
+ input_metadata)
310
+ return hidden_states
311
+
312
+ def sample(
313
+ self,
314
+ hidden_states: torch.Tensor,
315
+ sampling_metadata: SamplingMetadata,
316
+ ) -> Optional[SamplerOutput]:
317
+ next_tokens = self.sampler(self.lm_head.weight, hidden_states,
318
+ sampling_metadata)
319
+ return next_tokens
320
321
def load_weights(self,
322
model_name_or_path: str,
@@ -42,15 +328,9 @@ class BaiChuanBaseForCausalLM(LlamaForCausalLM):
328
("gate_up_proj", "gate_proj", 0),
329
("gate_up_proj", "up_proj", 1),
330
]
45
- param_weight_map = [
46
- ("qkv_proj", "W_pack"),
47
- ]
331
params_dict = dict(self.named_parameters())
332
for name, loaded_weight in hf_model_weights_iterator(
333
model_name_or_path, cache_dir, load_format, revision):
51
- for (param_name, weight_name) in param_weight_map:
52
- name = name.replace(weight_name, param_name)
53
-
334
if "rotary_emb.inv_freq" in name:
335
continue
336
if name == "lm_head.weight":
@@ -88,28 +368,19 @@ class BaiChuanBaseForCausalLM(LlamaForCausalLM):
368
class BaichuanForCausalLM(BaiChuanBaseForCausalLM):
369
"""Baichuan 13B and Baichuan2 7B/13B."""
370
91
- def __init__(
92
- self,
93
- config: Optional[PretrainedConfig] = None,
94
- linear_method: Optional[LinearMethodBase] = None,
95
- lora_config: Optional[LoRAConfig] = None,
96
- ) -> None:
97
- if config.hidden_size != 4096: # baichuan 13b, baichuan2 13b
98
- config.postion_embedding = "ALIBI"
99
- super().__init__(config=config,
100
- linear_method=linear_method,
101
- lora_config=lora_config)
371
+ def __init__(self,
372
+ config,
373
+ linear_method: Optional[LinearMethodBase] = None):
374
+ if config.hidden_size == 4096: # baichuan2 7b
375
+ super().__init__(config, "ROPE", linear_method)
376
+ else: # baichuan 13b, baichuan2 13b
377
+ super().__init__(config, "ALIBI", linear_method)
378
379
380
class BaiChuanForCausalLM(BaiChuanBaseForCausalLM):
381
"""Baichuan 7B."""
382
107
- def __init__(
108
- self,
109
- config: Optional[PretrainedConfig] = None,
110
- linear_method: Optional[LinearMethodBase] = None,
111
- lora_config: Optional[LoRAConfig] = None,
112
- ) -> None:
113
- super().__init__(config=config,
114
- linear_method=linear_method,
115
- lora_config=lora_config)
383
+ def __init__(self,
384
+ config,
385
+ linear_method: Optional[LinearMethodBase] = None):
386
+ super().__init__(config, "ROPE", linear_method)
vllm/model_executor/models/internlm.py
new
+299
@@ -0,0 +1,299 @@
1
+# -*- coding: utf-8 -*-
2
+from typing import Any, Dict, List, Optional, Tuple
3
+
4
+import torch
5
+from torch import nn
6
+from transformers import LlamaConfig
7
+
8
+from vllm.model_executor.input_metadata import InputMetadata
9
+from vllm.model_executor.layers.activation import SiluAndMul
10
+from vllm.model_executor.layers.attention import PagedAttention
11
+from vllm.model_executor.layers.layernorm import RMSNorm
12
+from vllm.model_executor.layers.linear import (LinearMethodBase,
13
+ MergedColumnParallelLinear,
14
+ QKVParallelLinear,
15
+ RowParallelLinear)
16
+from vllm.model_executor.layers.rotary_embedding import get_rope
17
+from vllm.model_executor.layers.sampler import Sampler
18
+from vllm.model_executor.layers.vocab_parallel_embedding import (
19
+ VocabParallelEmbedding, ParallelLMHead)
20
+from vllm.model_executor.parallel_utils.parallel_state import (
21
+ get_tensor_model_parallel_world_size)
22
+from vllm.model_executor.sampling_metadata import SamplingMetadata
23
+from vllm.model_executor.weight_utils import (default_weight_loader,
24
+ hf_model_weights_iterator)
25
+from vllm.sequence import SamplerOutput
26
+
27
+KVCache = Tuple[torch.Tensor, torch.Tensor]
28
+
29
+
30
+class InternLMMLP(nn.Module):
31
+
32
+ def __init__(
33
+ self,
34
+ hidden_size: int,
35
+ intermediate_size: int,
36
+ hidden_act: str,
37
+ linear_method: Optional[LinearMethodBase] = None,
38
+ ):
39
+ super().__init__()
40
+ self.gate_up_proj = MergedColumnParallelLinear(
41
+ hidden_size, [intermediate_size] * 2,
42
+ bias=False,
43
+ linear_method=linear_method)
44
+ self.down_proj = RowParallelLinear(intermediate_size,
45
+ hidden_size,
46
+ bias=False,
47
+ linear_method=linear_method)
48
+ if hidden_act != "silu":
49
+ raise ValueError(f"Unsupported activation: {hidden_act}. "
50
+ "Only silu is supported for now.")
51
+ self.act_fn = SiluAndMul()
52
+
53
+ def forward(self, x):
54
+ gate_up, _ = self.gate_up_proj(x)
55
+ x = self.act_fn(gate_up)
56
+ x, _ = self.down_proj(x)
57
+ return x
58
+
59
+
60
+class InternLMAttention(nn.Module):
61
+
62
+ def __init__(
63
+ self,
64
+ hidden_size: int,
65
+ num_heads: int,
66
+ bias: bool,
67
+ rope_theta: float = 10000,
68
+ max_position_embeddings: int = 8192,
69
+ linear_method: Optional[LinearMethodBase] = None,
70
+ rope_scaling: Optional[Dict[str, Any]] = None,
71
+ ):
72
+ super().__init__()
73
+ self.hidden_size = hidden_size
74
+ tensor_model_parallel_world_size = (
75
+ get_tensor_model_parallel_world_size())
76
+ self.total_num_heads = num_heads
77
+ assert self.total_num_heads % tensor_model_parallel_world_size == 0
78
+ self.num_heads = (self.total_num_heads //
79
+ tensor_model_parallel_world_size)
80
+ self.head_dim = hidden_size // self.total_num_heads
81
+ self.scaling = self.head_dim**-0.5
82
+ self.rope_theta = rope_theta
83
+ self.max_position_embeddings = max_position_embeddings
84
+
85
+ self.qkv_proj = QKVParallelLinear(
86
+ hidden_size,
87
+ self.head_dim,
88
+ self.total_num_heads,
89
+ bias=bias,
90
+ linear_method=linear_method,
91
+ )
92
+ self.o_proj = RowParallelLinear(
93
+ self.total_num_heads * self.head_dim,
94
+ hidden_size,
95
+ bias=bias,
96
+ linear_method=linear_method,
97
+ )
98
+ self.rotary_emb = get_rope(
99
+ self.head_dim,
100
+ rotary_dim=self.head_dim,
101
+ max_position=self.max_position_embeddings,
102
+ base=self.rope_theta,
103
+ rope_scaling=rope_scaling,
104
+ )
105
+ self.attn = PagedAttention(self.num_heads, self.head_dim, self.scaling)
106
+
107
+ def forward(
108
+ self,
109
+ positions: torch.Tensor,
110
+ hidden_states: torch.Tensor,
111
+ kv_cache: KVCache,
112
+ input_metadata: InputMetadata,
113
+ ) -> torch.Tensor:
114
+ qkv, _ = self.qkv_proj(hidden_states)
115
+ q, k, v = qkv.chunk(chunks=3, dim=-1)
116
+ q, k = self.rotary_emb(positions, q, k)
117
+ k_cache, v_cache = kv_cache
118
+ attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
119
+ output, _ = self.o_proj(attn_output)
120
+ return output
121
+
122
+
123
+class InternLMDecoderLayer(nn.Module):
124
+
125
+ def __init__(
126
+ self,
127
+ config: LlamaConfig,
128
+ linear_method: Optional[LinearMethodBase] = None,
129
+ ):
130
+ super().__init__()
131
+ self.hidden_size = config.hidden_size
132
+ rope_theta = getattr(config, "rope_theta", 10000)
133
+ max_position_embeddings = getattr(config, "max_position_embeddings",
134
+ 8192)
135
+ self.self_attn = InternLMAttention(
136
+ hidden_size=self.hidden_size,
137
+ num_heads=config.num_attention_heads,
138
+ bias=config.bias,
139
+ rope_theta=rope_theta,
140
+ max_position_embeddings=max_position_embeddings,
141
+ linear_method=linear_method,
142
+ rope_scaling=getattr(config, "rope_scaling", None),
143
+ )
144
+ self.mlp = InternLMMLP(
145
+ hidden_size=self.hidden_size,
146
+ intermediate_size=config.intermediate_size,
147
+ hidden_act=config.hidden_act,
148
+ linear_method=linear_method,
149
+ )
150
+ self.input_layernorm = RMSNorm(config.hidden_size,
151
+ eps=config.rms_norm_eps)
152
+ self.post_attention_layernorm = RMSNorm(config.hidden_size,
153
+ eps=config.rms_norm_eps)
154
+
155
+ def forward(
156
+ self,
157
+ positions: torch.Tensor,
158
+ hidden_states: torch.Tensor,
159
+ kv_cache: KVCache,
160
+ input_metadata: InputMetadata,
161
+ residual: Optional[torch.Tensor],
162
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
163
+ # Self Attention
164
+ if residual is None:
165
+ residual = hidden_states
166
+ hidden_states = self.input_layernorm(hidden_states)
167
+ else:
168
+ hidden_states, residual = self.input_layernorm(
169
+ hidden_states, residual)
170
+ hidden_states = self.self_attn(
171
+ positions=positions,
172
+ hidden_states=hidden_states,
173
+ kv_cache=kv_cache,
174
+ input_metadata=input_metadata,
175
+ )
176
+
177
+ # Fully Connected
178
+ hidden_states, residual = self.post_attention_layernorm(
179
+ hidden_states, residual)
180
+ hidden_states = self.mlp(hidden_states)
181
+ return hidden_states, residual
182
+
183
+
184
+class InternLMModel(nn.Module):
185
+
186
+ def __init__(
187
+ self,
188
+ config: LlamaConfig,
189
+ linear_method: Optional[LinearMethodBase] = None,
190
+ ):
191
+ super().__init__()
192
+ self.config = config
193
+ self.padding_idx = config.pad_token_id
194
+ self.vocab_size = config.vocab_size
195
+
196
+ vocab_size = ((config.vocab_size + 63) // 64) * 64
197
+ self.embed_tokens = VocabParallelEmbedding(
198
+ vocab_size,
199
+ config.hidden_size,
200
+ )
201
+ self.layers = nn.ModuleList([
202
+ InternLMDecoderLayer(config, linear_method)
203
+ for _ in range(config.num_hidden_layers)
204
+ ])
205
+ self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
206
+
207
+ def forward(
208
+ self,
209
+ input_ids: torch.Tensor,
210
+ positions: torch.Tensor,
211
+ kv_caches: List[KVCache],
212
+ input_metadata: InputMetadata,
213
+ ) -> torch.Tensor:
214
+ hidden_states = self.embed_tokens(input_ids)
215
+ residual = None
216
+ for i in range(len(self.layers)):
217
+ layer = self.layers[i]
218
+ hidden_states, residual = layer(
219
+ positions,
220
+ hidden_states,
221
+ kv_caches[i],
222
+ input_metadata,
223
+ residual,
224
+ )
225
+ hidden_states, _ = self.norm(hidden_states, residual)
226
+ return hidden_states
227
+
228
+
229
+class InternLMForCausalLM(nn.Module):
230
+
231
+ def __init__(
232
+ self,
233
+ config,
234
+ linear_method: Optional[LinearMethodBase] = None,
235
+ ):
236
+ super().__init__()
237
+ self.config = config
238
+ self.linear_method = linear_method
239
+ self.model = InternLMModel(config, linear_method)
240
+ self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
241
+ self.sampler = Sampler(config.vocab_size)
242
+
243
+ def forward(
244
+ self,
245
+ input_ids: torch.Tensor,
246
+ positions: torch.Tensor,
247
+ kv_caches: List[KVCache],
248
+ input_metadata: InputMetadata,
249
+ ) -> torch.Tensor:
250
+ hidden_states = self.model(input_ids, positions, kv_caches,
251
+ input_metadata)
252
+ return hidden_states
253
+
254
+ def sample(
255
+ self,
256
+ hidden_states: torch.Tensor,
257
+ sampling_metadata: SamplingMetadata,
258
+ ) -> Optional[SamplerOutput]:
259
+ next_tokens = self.sampler(self.lm_head.weight, hidden_states,
260
+ sampling_metadata)
261
+ return next_tokens
262
+
263
+ def load_weights(self,
264
+ model_name_or_path: str,
265
+ cache_dir: Optional[str] = None,
266
+ load_format: str = "auto",
267
+ revision: Optional[str] = None):
268
+ stacked_params_mapping = [
269
+ # (param_name, shard_name, shard_id)
270
+ ("qkv_proj", "q_proj", "q"),
271
+ ("qkv_proj", "k_proj", "k"),
272
+ ("qkv_proj", "v_proj", "v"),
273
+ ("gate_up_proj", "gate_proj", 0),
274
+ ("gate_up_proj", "up_proj", 1),
275
+ ]
276
+ params_dict = dict(self.named_parameters())
277
+ for name, loaded_weight in hf_model_weights_iterator(
278
+ model_name_or_path, cache_dir, load_format, revision):
279
+ if "rotary_emb.inv_freq" in name:
280
+ continue
281
+ for (param_name, weight_name, shard_id) in stacked_params_mapping:
282
+ if weight_name not in name:
283
+ continue
284
+ name = name.replace(weight_name, param_name)
285
+ # Skip loading extra bias for GPTQ models.
286
+ if name.endswith(".bias") and name not in params_dict:
287
+ continue
288
+ param = params_dict[name]
289
+ weight_loader = param.weight_loader
290
+ weight_loader(param, loaded_weight, shard_id)
291
+ break
292
+ else:
293
+ # Skip loading extra bias for GPTQ models.
294
+ if name.endswith(".bias") and name not in params_dict:
295
+ continue
296
+ param = params_dict[name]
297
+ weight_loader = getattr(param, "weight_loader",
298
+ default_weight_loader)
299
+ weight_loader(param, loaded_weight)
vllm/model_executor/models/internlm2.py
+260
-25
@@ -1,27 +1,276 @@
1
# -*- coding: utf-8 -*-
2
-from typing import Optional
2
+from typing import Any, Dict, List, Optional, Tuple
3
4
import torch
5
+from torch import nn
6
from transformers import PretrainedConfig
6
-from vllm.config import LoRAConfig
7
8
-from vllm.model_executor.layers.linear import LinearMethodBase
9
-from vllm.model_executor.models.llama import LlamaForCausalLM
8
+from vllm.model_executor.input_metadata import InputMetadata
9
+from vllm.model_executor.layers.activation import SiluAndMul
10
+from vllm.model_executor.layers.attention import PagedAttention
11
+from vllm.model_executor.layers.layernorm import RMSNorm
12
+from vllm.model_executor.layers.linear import (LinearMethodBase,
13
+ MergedColumnParallelLinear,
14
+ QKVParallelLinear,
15
+ RowParallelLinear)
16
+from vllm.model_executor.layers.rotary_embedding import get_rope
17
+from vllm.model_executor.layers.sampler import Sampler
18
+from vllm.model_executor.layers.vocab_parallel_embedding import (
19
+ VocabParallelEmbedding, ParallelLMHead)
20
+from vllm.model_executor.parallel_utils.parallel_state import (
21
+ get_tensor_model_parallel_world_size)
22
+from vllm.model_executor.sampling_metadata import SamplingMetadata
23
from vllm.model_executor.weight_utils import (default_weight_loader,
24
hf_model_weights_iterator)
25
+from vllm.sequence import SamplerOutput
26
27
+KVCache = Tuple[torch.Tensor, torch.Tensor]
28
14
-class InternLM2ForCausalLM(LlamaForCausalLM):
29
+
30
+class InternLM2MLP(nn.Module):
31
+
32
+ def __init__(
33
+ self,
34
+ hidden_size: int,
35
+ intermediate_size: int,
36
+ hidden_act: str,
37
+ linear_method: Optional[LinearMethodBase] = None,
38
+ ) -> None:
39
+ super().__init__()
40
+ self.gate_up_proj = MergedColumnParallelLinear(
41
+ hidden_size, [intermediate_size] * 2,
42
+ bias=False,
43
+ linear_method=linear_method)
44
+ self.w2 = RowParallelLinear(intermediate_size,
45
+ hidden_size,
46
+ bias=False,
47
+ linear_method=linear_method)
48
+ if hidden_act != "silu":
49
+ raise ValueError(f"Unsupported activation: {hidden_act}. "
50
+ "Only silu is supported for now.")
51
+ self.act_fn = SiluAndMul()
52
+
53
+ def forward(self, x):
54
+ gate_up, _ = self.gate_up_proj(x)
55
+ x = self.act_fn(gate_up)
56
+ x, _ = self.w2(x)
57
+ return x
58
+
59
+
60
+class InternLM2Attention(nn.Module):
61
+
62
+ def __init__(
63
+ self,
64
+ hidden_size: int,
65
+ num_heads: int,
66
+ num_kv_heads: int,
67
+ rope_theta: float = 10000,
68
+ rope_scaling: Optional[Dict[str, Any]] = None,
69
+ max_position_embeddings: int = 8192,
70
+ linear_method: Optional[LinearMethodBase] = None,
71
+ ) -> None:
72
+ super().__init__()
73
+ self.hidden_size = hidden_size
74
+ tp_size = get_tensor_model_parallel_world_size()
75
+ self.total_num_heads = num_heads
76
+ assert self.total_num_heads % tp_size == 0
77
+ self.num_heads = self.total_num_heads // tp_size
78
+ self.total_num_kv_heads = num_kv_heads
79
+ if self.total_num_kv_heads >= tp_size:
80
+ # Number of KV heads is greater than TP size, so we partition
81
+ # the KV heads across multiple tensor parallel GPUs.
82
+ assert self.total_num_kv_heads % tp_size == 0
83
+ else:
84
+ # Number of KV heads is less than TP size, so we replicate
85
+ # the KV heads across multiple tensor parallel GPUs.
86
+ assert tp_size % self.total_num_kv_heads == 0
87
+ self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
88
+ self.head_dim = hidden_size // self.total_num_heads
89
+ self.q_size = self.num_heads * self.head_dim
90
+ self.kv_size = self.num_kv_heads * self.head_dim
91
+ self.scaling = self.head_dim**-0.5
92
+ self.rope_theta = rope_theta
93
+ self.max_position_embeddings = max_position_embeddings
94
+
95
+ self.wqkv = QKVParallelLinear(
96
+ hidden_size,
97
+ self.head_dim,
98
+ self.total_num_heads,
99
+ self.total_num_kv_heads,
100
+ bias=False,
101
+ linear_method=linear_method,
102
+ )
103
+ self.wo = RowParallelLinear(
104
+ self.total_num_heads * self.head_dim,
105
+ hidden_size,
106
+ bias=False,
107
+ linear_method=linear_method,
108
+ )
109
+
110
+ self.rotary_emb = get_rope(
111
+ self.head_dim,
112
+ rotary_dim=self.head_dim,
113
+ max_position=max_position_embeddings,
114
+ base=rope_theta,
115
+ rope_scaling=rope_scaling,
116
+ )
117
+ self.attn = PagedAttention(self.num_heads,
118
+ self.head_dim,
119
+ self.scaling,
120
+ num_kv_heads=self.num_kv_heads)
121
+
122
+ def forward(
123
+ self,
124
+ positions: torch.Tensor,
125
+ hidden_states: torch.Tensor,
126
+ kv_cache: KVCache,
127
+ input_metadata: InputMetadata,
128
+ ) -> torch.Tensor:
129
+ qkv, _ = self.wqkv(hidden_states)
130
+ q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
131
+ q, k = self.rotary_emb(positions, q, k)
132
+ k_cache, v_cache = kv_cache
133
+ attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
134
+ output, _ = self.wo(attn_output)
135
+ return output
136
+
137
+
138
+class InternLMDecoderLayer(nn.Module):
139
+
140
+ def __init__(
141
+ self,
142
+ config: PretrainedConfig,
143
+ linear_method: Optional[LinearMethodBase] = None,
144
+ ) -> None:
145
+ super().__init__()
146
+ self.hidden_size = config.hidden_size
147
+ rope_theta = getattr(config, "rope_theta", 10000)
148
+ rope_scaling = getattr(config, "rope_scaling", None)
149
+ max_position_embeddings = getattr(config, "max_position_embeddings",
150
+ 8192)
151
+ self.attention = InternLM2Attention(
152
+ hidden_size=self.hidden_size,
153
+ num_heads=config.num_attention_heads,
154
+ num_kv_heads=config.num_key_value_heads,
155
+ rope_theta=rope_theta,
156
+ rope_scaling=rope_scaling,
157
+ max_position_embeddings=max_position_embeddings,
158
+ linear_method=linear_method,
159
+ )
160
+ self.feed_forward = InternLM2MLP(
161
+ hidden_size=self.hidden_size,
162
+ intermediate_size=config.intermediate_size,
163
+ hidden_act=config.hidden_act,
164
+ linear_method=linear_method,
165
+ )
166
+ self.attention_norm = RMSNorm(config.hidden_size,
167
+ eps=config.rms_norm_eps)
168
+ self.ffn_norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
169
+
170
+ def forward(
171
+ self,
172
+ positions: torch.Tensor,
173
+ hidden_states: torch.Tensor,
174
+ kv_cache: KVCache,
175
+ input_metadata: InputMetadata,
176
+ residual: Optional[torch.Tensor],
177
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
178
+ # Self Attention
179
+ if residual is None:
180
+ residual = hidden_states
181
+ hidden_states = self.attention_norm(hidden_states)
182
+ else:
183
+ hidden_states, residual = self.attention_norm(
184
+ hidden_states, residual)
185
+ hidden_states = self.attention(
186
+ positions=positions,
187
+ hidden_states=hidden_states,
188
+ kv_cache=kv_cache,
189
+ input_metadata=input_metadata,
190
+ )
191
+
192
+ # Fully Connected
193
+ hidden_states, residual = self.ffn_norm(hidden_states, residual)
194
+ hidden_states = self.feed_forward(hidden_states)
195
+ return hidden_states, residual
196
+
197
+
198
+class InternLM2Model(nn.Module):
199
200
def __init__(
201
self,
18
- config: Optional[PretrainedConfig] = None,
202
+ config: PretrainedConfig,
203
linear_method: Optional[LinearMethodBase] = None,
20
- lora_config: Optional[LoRAConfig] = None,
204
) -> None:
22
- super().__init__(config=config,
23
- linear_method=linear_method,
24
- lora_config=lora_config)
205
+ super().__init__()
206
+ self.config = config
207
+ self.padding_idx = config.pad_token_id
208
+ self.vocab_size = config.vocab_size
209
+ self.tok_embeddings = VocabParallelEmbedding(
210
+ config.vocab_size,
211
+ config.hidden_size,
212
+ )
213
+ self.layers = nn.ModuleList([
214
+ InternLMDecoderLayer(config, linear_method)
215
+ for _ in range(config.num_hidden_layers)
216
+ ])
217
+ self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
218
+
219
+ def forward(
220
+ self,
221
+ input_ids: torch.Tensor,
222
+ positions: torch.Tensor,
223
+ kv_caches: List[KVCache],
224
+ input_metadata: InputMetadata,
225
+ ) -> torch.Tensor:
226
+ hidden_states = self.tok_embeddings(input_ids)
227
+ residual = None
228
+ for i in range(len(self.layers)):
229
+ layer = self.layers[i]
230
+ hidden_states, residual = layer(
231
+ positions,
232
+ hidden_states,
233
+ kv_caches[i],
234
+ input_metadata,
235
+ residual,
236
+ )
237
+ hidden_states, _ = self.norm(hidden_states, residual)
238
+ return hidden_states
239
+
240
+
241
+class InternLM2ForCausalLM(nn.Module):
242
+
243
+ def __init__(
244
+ self,
245
+ config: PretrainedConfig,
246
+ linear_method: Optional[LinearMethodBase] = None,
247
+ ) -> None:
248
+ super().__init__()
249
+ self.config = config
250
+ self.linear_method = linear_method
251
+ self.model = InternLM2Model(config, linear_method)
252
+ self.output = ParallelLMHead(config.vocab_size, config.hidden_size)
253
+ self.sampler = Sampler(config.vocab_size)
254
+
255
+ def forward(
256
+ self,
257
+ input_ids: torch.Tensor,
258
+ positions: torch.Tensor,
259
+ kv_caches: List[KVCache],
260
+ input_metadata: InputMetadata,
261
+ ) -> torch.Tensor:
262
+ hidden_states = self.model(input_ids, positions, kv_caches,
263
+ input_metadata)
264
+ return hidden_states
265
+
266
+ def sample(
267
+ self,
268
+ hidden_states: torch.Tensor,
269
+ sampling_metadata: SamplingMetadata,
270
+ ) -> Optional[SamplerOutput]:
271
+ next_tokens = self.sampler(self.output.weight, hidden_states,
272
+ sampling_metadata)
273
+ return next_tokens
274
275
def load_weights(self,
276
model_name_or_path: str,
@@ -33,23 +282,9 @@ class InternLM2ForCausalLM(LlamaForCausalLM):
282
("gate_up_proj", "w1", 0),
283
("gate_up_proj", "w3", 1),
284
]
36
- param_weight_map = [
37
- ("qkv_proj", "wqkv"),
38
- ("o_proj", "wo"),
39
- ("down_proj", "w2"),
40
- ("input_layernorm", "attention_norm"),
41
- ("post_attention_layernorm", "ffn_norm"),
42
- ("embed_tokens", "tok_embeddings"),
43
- (".self_attn.", ".attention."),
44
- ("mlp", "feed_forward"),
45
- ("lm_head", "output"),
46
- ]
285
params_dict = dict(self.named_parameters())
286
for name, loaded_weight in hf_model_weights_iterator(
287
model_name_or_path, cache_dir, load_format, revision):
50
- for (param_name, weight_name) in param_weight_map:
51
- name = name.replace(weight_name, param_name)
52
-
288
if "rotary_emb.inv_freq" in name:
289
continue
290
for (param_name, weight_name, shard_id) in stacked_params_mapping:
@@ -68,7 +303,7 @@ class InternLM2ForCausalLM(LlamaForCausalLM):
303
if name.endswith(".bias") and name not in params_dict:
304
continue
305
param = params_dict[name]
71
- if "qkv_proj" in name:
306
+ if "wqkv" in name:
307
config = self.config
308
kv_groups = config.num_attention_heads // config.num_key_value_heads
309
head_dim = config.hidden_size // config.num_attention_heads
vllm/model_executor/models/llama.py
+56
-106
@@ -21,9 +21,8 @@
21
# See the License for the specific language governing permissions and
22
# limitations under the License.
23
"""Inference-only LLaMA model compatible with HuggingFace weights."""
24
-from typing import List, Optional, Tuple
24
+from typing import Any, Dict, List, Optional, Tuple
25
26
-import math
26
import torch
27
from torch import nn
28
from transformers import LlamaConfig
@@ -41,60 +40,34 @@ from vllm.model_executor.layers.sampler import Sampler
40
from vllm.model_executor.layers.vocab_parallel_embedding import (
41
VocabParallelEmbedding, ParallelLMHead, DEFAULT_VOCAB_PADDING_SIZE)
42
from vllm.model_executor.parallel_utils.parallel_state import (
44
- get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
43
+ get_tensor_model_parallel_world_size)
44
from vllm.model_executor.sampling_metadata import SamplingMetadata
45
from vllm.model_executor.weight_utils import (default_weight_loader,
46
hf_model_weights_iterator)
47
from vllm.sequence import SamplerOutput
48
from vllm.config import LoRAConfig
49
51
-from copy import deepcopy
52
-
50
KVCache = Tuple[torch.Tensor, torch.Tensor]
51
52
56
-def _get_alibi_slopes(total_num_heads: int) -> torch.Tensor:
57
- closest_power_of_2 = 2**math.floor(math.log2(total_num_heads))
58
- base = torch.tensor(
59
- 2**(-(2**-(math.log2(closest_power_of_2) - 3))),
60
- dtype=torch.float32,
61
- )
62
- powers = torch.arange(1, 1 + closest_power_of_2, dtype=torch.int32)
63
- slopes = torch.pow(base, powers)
64
-
65
- if closest_power_of_2 != total_num_heads:
66
- extra_base = torch.tensor(
67
- 2**(-(2**-(math.log2(2 * closest_power_of_2) - 3))),
68
- dtype=torch.float32,
69
- )
70
- num_remaining_heads = min(closest_power_of_2,
71
- total_num_heads - closest_power_of_2)
72
- extra_powers = torch.arange(start=1,
73
- end=1 + 2 * num_remaining_heads,
74
- step=2,
75
- dtype=torch.int32)
76
- slopes = torch.cat(
77
- [slopes, torch.pow(extra_base, extra_powers)], dim=0)
78
- return slopes
79
-
80
-
53
class LlamaMLP(nn.Module):
54
55
def __init__(
56
self,
85
- config: LlamaConfig,
57
+ hidden_size: int,
58
+ intermediate_size: int,
59
+ hidden_act: str,
60
linear_method: Optional[LinearMethodBase] = None,
61
) -> None:
62
super().__init__()
63
self.gate_up_proj = MergedColumnParallelLinear(
90
- config.hidden_size, [config.intermediate_size] * 2,
64
+ hidden_size, [intermediate_size] * 2,
65
bias=False,
66
linear_method=linear_method)
93
- self.down_proj = RowParallelLinear(config.intermediate_size,
94
- config.hidden_size,
67
+ self.down_proj = RowParallelLinear(intermediate_size,
68
+ hidden_size,
69
bias=False,
70
linear_method=linear_method)
97
- hidden_act = getattr(config, "hidden_act", "silu")
71
if hidden_act != "silu":
72
raise ValueError(f"Unsupported activation: {hidden_act}. "
73
"Only silu is supported for now.")
@@ -111,19 +84,21 @@ class LlamaAttention(nn.Module):
84
85
def __init__(
86
self,
114
- config: LlamaConfig,
87
+ hidden_size: int,
88
+ num_heads: int,
89
+ num_kv_heads: int,
90
+ rope_theta: float = 10000,
91
+ rope_scaling: Optional[Dict[str, Any]] = None,
92
+ max_position_embeddings: int = 8192,
93
linear_method: Optional[LinearMethodBase] = None,
94
) -> None:
95
super().__init__()
118
- self.hidden_size = config.hidden_size
96
+ self.hidden_size = hidden_size
97
tp_size = get_tensor_model_parallel_world_size()
120
- self.total_num_heads = getattr(config, "num_attention_heads", None)
98
+ self.total_num_heads = num_heads
99
assert self.total_num_heads % tp_size == 0
100
self.num_heads = self.total_num_heads // tp_size
123
-
124
- # defaut to mha
125
- self.total_num_kv_heads = getattr(config, "num_key_value_heads",
126
- self.total_num_heads)
101
+ self.total_num_kv_heads = num_kv_heads
102
if self.total_num_kv_heads >= tp_size:
103
# Number of KV heads is greater than TP size, so we partition
104
# the KV heads across multiple tensor parallel GPUs.
@@ -133,68 +108,39 @@ class LlamaAttention(nn.Module):
108
# the KV heads across multiple tensor parallel GPUs.
109
assert tp_size % self.total_num_kv_heads == 0
110
self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
136
- self.head_dim = self.hidden_size // self.total_num_heads
111
+ self.head_dim = hidden_size // self.total_num_heads
112
self.q_size = self.num_heads * self.head_dim
113
self.kv_size = self.num_kv_heads * self.head_dim
114
self.scaling = self.head_dim**-0.5
140
- max_position_embeddings = getattr(config, "max_position_embeddings",
141
- 8192)
142
- self.max_position_embeddings = config.max_position_embeddings
115
+ self.rope_theta = rope_theta
116
+ self.max_position_embeddings = max_position_embeddings
117
144
- # internlm
145
- bias = getattr(config, "bias", False)
146
-
147
- # stablelm
148
- qkv_bias = getattr(config, "use_qkv_bias", False)
118
self.qkv_proj = QKVParallelLinear(
150
- self.hidden_size,
119
+ hidden_size,
120
self.head_dim,
121
self.total_num_heads,
122
self.total_num_kv_heads,
154
- bias=bias or qkv_bias,
123
+ bias=False,
124
linear_method=linear_method,
125
)
126
self.o_proj = RowParallelLinear(
127
self.total_num_heads * self.head_dim,
159
- self.hidden_size,
160
- bias=bias,
128
+ hidden_size,
129
+ bias=False,
130
linear_method=linear_method,
131
)
132
164
- # mistral
165
- sliding_window = getattr(config, "sliding_window", None)
166
-
167
- self.postion_embedding = getattr(config, "postion_embedding", "ROPE")
168
- # Create the alibi slopes and slice them.
169
- if self.postion_embedding == "ALIBI":
170
- tp_rank = get_tensor_model_parallel_rank()
171
- head_start = tp_rank * self.num_heads
172
- head_end = (tp_rank + 1) * self.num_heads
173
- alibi_slopes = _get_alibi_slopes(self.total_num_heads)
174
- alibi_slopes = alibi_slopes[head_start:head_end].tolist()
175
-
176
- self.attn = PagedAttention(self.num_heads,
177
- self.head_dim,
178
- self.scaling,
179
- alibi_slopes=alibi_slopes,
180
- sliding_window=sliding_window)
181
- else:
182
- rope_theta = getattr(config, "rope_theta", 10000)
183
- rope_scaling = getattr(config, "rope_scaling", None)
184
- # stablelm
185
- rope_pct = getattr(config, "rope_pct", 1)
186
- self.rotary_emb = get_rope(
187
- self.head_dim,
188
- rotary_dim=int(self.head_dim * rope_pct),
189
- max_position=max_position_embeddings,
190
- base=rope_theta,
191
- rope_scaling=rope_scaling,
192
- )
193
- self.attn = PagedAttention(self.num_heads,
194
- self.head_dim,
195
- self.scaling,
196
- num_kv_heads=self.num_kv_heads,
197
- sliding_window=sliding_window)
133
+ self.rotary_emb = get_rope(
134
+ self.head_dim,
135
+ rotary_dim=self.head_dim,
136
+ max_position=max_position_embeddings,
137
+ base=rope_theta,
138
+ rope_scaling=rope_scaling,
139
+ )
140
+ self.attn = PagedAttention(self.num_heads,
141
+ self.head_dim,
142
+ self.scaling,
143
+ num_kv_heads=self.num_kv_heads)
144
145
def forward(
146
self,
@@ -205,8 +151,7 @@ class LlamaAttention(nn.Module):
151
) -> torch.Tensor:
152
qkv, _ = self.qkv_proj(hidden_states)
153
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
208
- if self.postion_embedding != "ALIBI":
209
- q, k = self.rotary_emb(positions, q, k)
154
+ q, k = self.rotary_emb(positions, q, k)
155
k_cache, v_cache = kv_cache
156
attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
157
output, _ = self.o_proj(attn_output)
@@ -219,20 +164,32 @@ class LlamaDecoderLayer(nn.Module):
164
self,
165
config: LlamaConfig,
166
linear_method: Optional[LinearMethodBase] = None,
222
- norm: Optional[torch.Tensor] = None,
167
) -> None:
168
super().__init__()
169
self.hidden_size = config.hidden_size
170
+ rope_theta = getattr(config, "rope_theta", 10000)
171
+ rope_scaling = getattr(config, "rope_scaling", None)
172
+ max_position_embeddings = getattr(config, "max_position_embeddings",
173
+ 8192)
174
self.self_attn = LlamaAttention(
227
- config,
175
+ hidden_size=self.hidden_size,
176
+ num_heads=config.num_attention_heads,
177
+ num_kv_heads=config.num_key_value_heads,
178
+ rope_theta=rope_theta,
179
+ rope_scaling=rope_scaling,
180
+ max_position_embeddings=max_position_embeddings,
181
linear_method=linear_method,
182
)
183
self.mlp = LlamaMLP(
231
- config,
184
+ hidden_size=self.hidden_size,
185
+ intermediate_size=config.intermediate_size,
186
+ hidden_act=config.hidden_act,
187
linear_method=linear_method,
188
)
234
- self.input_layernorm = deepcopy(norm)
235
- self.post_attention_layernorm = deepcopy(norm)
189
+ self.input_layernorm = RMSNorm(config.hidden_size,
190
+ eps=config.rms_norm_eps)
191
+ self.post_attention_layernorm = RMSNorm(config.hidden_size,
192
+ eps=config.rms_norm_eps)
193
194
def forward(
195
self,
@@ -269,7 +226,6 @@ class LlamaModel(nn.Module):
226
self,
227
config: LlamaConfig,
228
linear_method: Optional[LinearMethodBase] = None,
272
- norm: Optional[torch.Tensor] = None,
229
lora_config: Optional[LoRAConfig] = None,
230
) -> None:
231
super().__init__()
@@ -285,10 +241,10 @@ class LlamaModel(nn.Module):
241
org_num_embeddings=config.vocab_size,
242
)
243
self.layers = nn.ModuleList([
288
- LlamaDecoderLayer(config, linear_method, norm)
244
+ LlamaDecoderLayer(config, linear_method)
245
for _ in range(config.num_hidden_layers)
246
])
291
- self.norm = norm
247
+ self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
248
249
def forward(
250
self,
@@ -319,18 +275,12 @@ class LlamaForCausalLM(nn.Module):
275
self,
276
config: LlamaConfig,
277
linear_method: Optional[LinearMethodBase] = None,
322
- norm: Optional[torch.Tensor] = None,
278
lora_config: Optional[LoRAConfig] = None,
279
) -> None:
280
super().__init__()
281
self.config = config
282
self.linear_method = linear_method
328
- if norm is None:
329
- norm = RMSNorm(config.hidden_size, config.rms_norm_eps)
330
- self.model = LlamaModel(config,
331
- linear_method,
332
- norm=norm,
333
- lora_config=lora_config)
283
+ self.model = LlamaModel(config, linear_method, lora_config=lora_config)
284
unpadded_vocab_size = config.vocab_size
285
if lora_config:
286
unpadded_vocab_size += lora_config.lora_extra_vocab_size
vllm/model_executor/models/mistral.py
new
+352
@@ -0,0 +1,352 @@
1
+# coding=utf-8
2
+# Adapted from
3
+# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
4
+# Copyright 2023 The vLLM team.
5
+# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
6
+#
7
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
8
+# and OPT implementations in this library. It has been modified from its
9
+# original forms to accommodate minor architectural differences compared
10
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
11
+#
12
+# Licensed under the Apache License, Version 2.0 (the "License");
13
+# you may not use this file except in compliance with the License.
14
+# You may obtain a copy of the License at
15
+#
16
+# http://www.apache.org/licenses/LICENSE-2.0
17
+#
18
+# Unless required by applicable law or agreed to in writing, software
19
+# distributed under the License is distributed on an "AS IS" BASIS,
20
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21
+# See the License for the specific language governing permissions and
22
+# limitations under the License.
23
+"""Inference-only Mistral model compatible with HuggingFace weights."""
24
+from typing import List, Optional, Tuple
25
+
26
+import torch
27
+from torch import nn
28
+from transformers import MistralConfig
29
+
30
+from vllm.model_executor.input_metadata import InputMetadata
31
+from vllm.model_executor.layers.activation import SiluAndMul
32
+from vllm.model_executor.layers.attention import PagedAttention
33
+from vllm.model_executor.layers.layernorm import RMSNorm
34
+from vllm.model_executor.layers.linear import (LinearMethodBase,
35
+ MergedColumnParallelLinear,
36
+ QKVParallelLinear,
37
+ RowParallelLinear)
38
+from vllm.model_executor.layers.rotary_embedding import get_rope
39
+from vllm.model_executor.layers.sampler import Sampler
40
+from vllm.model_executor.layers.vocab_parallel_embedding import (
41
+ VocabParallelEmbedding, ParallelLMHead, DEFAULT_VOCAB_PADDING_SIZE)
42
+from vllm.model_executor.parallel_utils.parallel_state import (
43
+ get_tensor_model_parallel_world_size)
44
+from vllm.model_executor.sampling_metadata import SamplingMetadata
45
+from vllm.model_executor.weight_utils import (default_weight_loader,
46
+ hf_model_weights_iterator)
47
+from vllm.sequence import SamplerOutput
48
+from vllm.config import LoRAConfig
49
+
50
+KVCache = Tuple[torch.Tensor, torch.Tensor]
51
+
52
+
53
+class MistralMLP(nn.Module):
54
+
55
+ def __init__(
56
+ self,
57
+ hidden_size: int,
58
+ intermediate_size: int,
59
+ hidden_act: str,
60
+ linear_method: Optional[LinearMethodBase] = None,
61
+ ) -> None:
62
+ super().__init__()
63
+ self.gate_up_proj = MergedColumnParallelLinear(
64
+ hidden_size, [intermediate_size] * 2,
65
+ bias=False,
66
+ linear_method=linear_method)
67
+ self.down_proj = RowParallelLinear(intermediate_size,
68
+ hidden_size,
69
+ bias=False,
70
+ linear_method=linear_method)
71
+ if hidden_act != "silu":
72
+ raise ValueError(f"Unsupported activation: {hidden_act}. "
73
+ "Only silu is supported for now.")
74
+ self.act_fn = SiluAndMul()
75
+
76
+ def forward(self, x):
77
+ gate_up, _ = self.gate_up_proj(x)
78
+ x = self.act_fn(gate_up)
79
+ x, _ = self.down_proj(x)
80
+ return x
81
+
82
+
83
+class MistralAttention(nn.Module):
84
+
85
+ def __init__(self,
86
+ hidden_size: int,
87
+ num_heads: int,
88
+ num_kv_heads: int,
89
+ max_position: int = 4096 * 32,
90
+ rope_theta: float = 10000,
91
+ linear_method: Optional[LinearMethodBase] = None,
92
+ sliding_window: Optional[int] = None) -> None:
93
+ super().__init__()
94
+ self.hidden_size = hidden_size
95
+ tp_size = get_tensor_model_parallel_world_size()
96
+ self.total_num_heads = num_heads
97
+ assert self.total_num_heads % tp_size == 0
98
+ self.num_heads = self.total_num_heads // tp_size
99
+ self.total_num_kv_heads = num_kv_heads
100
+ if self.total_num_kv_heads >= tp_size:
101
+ # Number of KV heads is greater than TP size, so we partition
102
+ # the KV heads across multiple tensor parallel GPUs.
103
+ assert self.total_num_kv_heads % tp_size == 0
104
+ else:
105
+ # Number of KV heads is less than TP size, so we replicate
106
+ # the KV heads across multiple tensor parallel GPUs.
107
+ assert tp_size % self.total_num_kv_heads == 0
108
+ self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
109
+ self.head_dim = hidden_size // self.total_num_heads
110
+ self.q_size = self.num_heads * self.head_dim
111
+ self.kv_size = self.num_kv_heads * self.head_dim
112
+ self.scaling = self.head_dim**-0.5
113
+ self.rope_theta = rope_theta
114
+ self.sliding_window = sliding_window
115
+
116
+ self.qkv_proj = QKVParallelLinear(
117
+ hidden_size,
118
+ self.head_dim,
119
+ self.total_num_heads,
120
+ self.total_num_kv_heads,
121
+ bias=False,
122
+ linear_method=linear_method,
123
+ )
124
+ self.o_proj = RowParallelLinear(
125
+ self.total_num_heads * self.head_dim,
126
+ hidden_size,
127
+ bias=False,
128
+ linear_method=linear_method,
129
+ )
130
+
131
+ self.rotary_emb = get_rope(
132
+ self.head_dim,
133
+ rotary_dim=self.head_dim,
134
+ max_position=max_position,
135
+ base=self.rope_theta,
136
+ )
137
+ self.attn = PagedAttention(self.num_heads,
138
+ self.head_dim,
139
+ self.scaling,
140
+ num_kv_heads=self.num_kv_heads,
141
+ sliding_window=self.sliding_window)
142
+
143
+ def forward(
144
+ self,
145
+ positions: torch.Tensor,
146
+ hidden_states: torch.Tensor,
147
+ kv_cache: KVCache,
148
+ input_metadata: InputMetadata,
149
+ ) -> torch.Tensor:
150
+ qkv, _ = self.qkv_proj(hidden_states)
151
+ q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
152
+ q, k = self.rotary_emb(positions, q, k)
153
+ k_cache, v_cache = kv_cache
154
+ attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
155
+ output, _ = self.o_proj(attn_output)
156
+ return output
157
+
158
+
159
+class MistralDecoderLayer(nn.Module):
160
+
161
+ def __init__(
162
+ self,
163
+ config: MistralConfig,
164
+ linear_method: Optional[LinearMethodBase] = None,
165
+ ) -> None:
166
+ super().__init__()
167
+ self.hidden_size = config.hidden_size
168
+ # Requires transformers > 4.32.0
169
+ rope_theta = getattr(config, "rope_theta", 10000)
170
+ self.self_attn = MistralAttention(
171
+ hidden_size=self.hidden_size,
172
+ num_heads=config.num_attention_heads,
173
+ max_position=config.max_position_embeddings,
174
+ num_kv_heads=config.num_key_value_heads,
175
+ rope_theta=rope_theta,
176
+ linear_method=linear_method,
177
+ sliding_window=config.sliding_window)
178
+ self.mlp = MistralMLP(
179
+ hidden_size=self.hidden_size,
180
+ intermediate_size=config.intermediate_size,
181
+ hidden_act=config.hidden_act,
182
+ linear_method=linear_method,
183
+ )
184
+ self.input_layernorm = RMSNorm(config.hidden_size,
185
+ eps=config.rms_norm_eps)
186
+ self.post_attention_layernorm = RMSNorm(config.hidden_size,
187
+ eps=config.rms_norm_eps)
188
+
189
+ def forward(
190
+ self,
191
+ positions: torch.Tensor,
192
+ hidden_states: torch.Tensor,
193
+ kv_cache: KVCache,
194
+ input_metadata: InputMetadata,
195
+ residual: Optional[torch.Tensor],
196
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
197
+ # Self Attention
198
+ if residual is None:
199
+ residual = hidden_states
200
+ hidden_states = self.input_layernorm(hidden_states)
201
+ else:
202
+ hidden_states, residual = self.input_layernorm(
203
+ hidden_states, residual)
204
+ hidden_states = self.self_attn(
205
+ positions=positions,
206
+ hidden_states=hidden_states,
207
+ kv_cache=kv_cache,
208
+ input_metadata=input_metadata,
209
+ )
210
+
211
+ # Fully Connected
212
+ hidden_states, residual = self.post_attention_layernorm(
213
+ hidden_states, residual)
214
+ hidden_states = self.mlp(hidden_states)
215
+ return hidden_states, residual
216
+
217
+
218
+class MistralModel(nn.Module):
219
+
220
+ def __init__(
221
+ self,
222
+ config: MistralConfig,
223
+ linear_method: Optional[LinearMethodBase] = None,
224
+ lora_config: Optional[LoRAConfig] = None,
225
+ ) -> None:
226
+ super().__init__()
227
+ self.config = config
228
+ self.padding_idx = config.pad_token_id
229
+ lora_vocab = (lora_config.lora_extra_vocab_size *
230
+ (lora_config.max_loras or 1)) if lora_config else 0
231
+ self.vocab_size = config.vocab_size + lora_vocab
232
+ self.org_vocab_size = config.vocab_size
233
+
234
+ self.embed_tokens = VocabParallelEmbedding(
235
+ self.vocab_size,
236
+ config.hidden_size,
237
+ org_num_embeddings=config.vocab_size,
238
+ )
239
+ self.layers = nn.ModuleList([
240
+ MistralDecoderLayer(config, linear_method)
241
+ for _ in range(config.num_hidden_layers)
242
+ ])
243
+ self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
244
+
245
+ def forward(
246
+ self,
247
+ input_ids: torch.Tensor,
248
+ positions: torch.Tensor,
249
+ kv_caches: List[KVCache],
250
+ input_metadata: InputMetadata,
251
+ ) -> torch.Tensor:
252
+ hidden_states = self.embed_tokens(input_ids)
253
+ residual = None
254
+ for i in range(len(self.layers)):
255
+ layer = self.layers[i]
256
+ hidden_states, residual = layer(
257
+ positions,
258
+ hidden_states,
259
+ kv_caches[i],
260
+ input_metadata,
261
+ residual,
262
+ )
263
+ hidden_states, _ = self.norm(hidden_states, residual)
264
+ return hidden_states
265
+
266
+
267
+class MistralForCausalLM(nn.Module):
268
+ supports_lora = True
269
+
270
+ def __init__(
271
+ self,
272
+ config: MistralConfig,
273
+ linear_method: Optional[LinearMethodBase] = None,
274
+ lora_config: Optional[LoRAConfig] = None,
275
+ ) -> None:
276
+ super().__init__()
277
+ self.config = config
278
+ self.linear_method = linear_method
279
+ self.model = MistralModel(config,
280
+ linear_method,
281
+ lora_config=lora_config)
282
+ unpadded_vocab_size = config.vocab_size
283
+ if lora_config:
284
+ unpadded_vocab_size += lora_config.lora_extra_vocab_size
285
+ self.lm_head = ParallelLMHead(
286
+ unpadded_vocab_size,
287
+ config.hidden_size,
288
+ org_num_embeddings=config.vocab_size,
289
+ padding_size=DEFAULT_VOCAB_PADDING_SIZE
290
+ # We need bigger padding if using lora for kernel
291
+ # compatibility
292
+ if not lora_config else lora_config.lora_vocab_padding_size,
293
+ )
294
+ self.sampler = Sampler(unpadded_vocab_size, config.vocab_size)
295
+
296
+ def forward(
297
+ self,
298
+ input_ids: torch.Tensor,
299
+ positions: torch.Tensor,
300
+ kv_caches: List[KVCache],
301
+ input_metadata: InputMetadata,
302
+ ) -> torch.Tensor:
303
+ hidden_states = self.model(input_ids, positions, kv_caches,
304
+ input_metadata)
305
+ return hidden_states
306
+
307
+ def sample(
308
+ self,
309
+ hidden_states: torch.Tensor,
310
+ sampling_metadata: SamplingMetadata,
311
+ ) -> Optional[SamplerOutput]:
312
+ next_tokens = self.sampler(self.lm_head.weight, hidden_states,
313
+ sampling_metadata)
314
+ return next_tokens
315
+
316
+ def load_weights(self,
317
+ model_name_or_path: str,
318
+ cache_dir: Optional[str] = None,
319
+ load_format: str = "auto",
320
+ revision: Optional[str] = None):
321
+ stacked_params_mapping = [
322
+ # (param_name, shard_name, shard_id)
323
+ ("qkv_proj", "q_proj", "q"),
324
+ ("qkv_proj", "k_proj", "k"),
325
+ ("qkv_proj", "v_proj", "v"),
326
+ ("gate_up_proj", "gate_proj", 0),
327
+ ("gate_up_proj", "up_proj", 1),
328
+ ]
329
+ params_dict = dict(self.named_parameters())
330
+ for name, loaded_weight in hf_model_weights_iterator(
331
+ model_name_or_path, cache_dir, load_format, revision):
332
+ if "rotary_emb.inv_freq" in name:
333
+ continue
334
+ for (param_name, weight_name, shard_id) in stacked_params_mapping:
335
+ if weight_name not in name:
336
+ continue
337
+ name = name.replace(weight_name, param_name)
338
+ # Skip loading extra bias for GPTQ models.
339
+ if name.endswith(".bias") and name not in params_dict:
340
+ continue
341
+ param = params_dict[name]
342
+ weight_loader = param.weight_loader
343
+ weight_loader(param, loaded_weight, shard_id)
344
+ break
345
+ else:
346
+ # Skip loading extra bias for GPTQ models.
347
+ if name.endswith(".bias") and name not in params_dict:
348
+ continue
349
+ param = params_dict[name]
350
+ weight_loader = getattr(param, "weight_loader",
351
+ default_weight_loader)
352
+ weight_loader(param, loaded_weight)
vllm/model_executor/models/qwen.py
+236
-31
@@ -4,33 +4,253 @@
4
# Copyright (c) Alibaba Cloud.
5
# LICENSE: https://huggingface.co/Qwen/Qwen-7B/blob/main/LICENSE
6
"""Inference-only QWen model compatible with HuggingFace weights."""
7
-from typing import Optional
7
+from typing import Any, Dict, List, Optional, Tuple
8
9
-from transformers import PretrainedConfig
10
-from vllm.config import LoRAConfig
9
+import torch
10
+from torch import nn
11
12
-from vllm.model_executor.layers.linear import LinearMethodBase
12
+from vllm.model_executor.input_metadata import InputMetadata
13
+from vllm.model_executor.layers.activation import SiluAndMul
14
+from vllm.model_executor.layers.attention import PagedAttention
15
from vllm.model_executor.layers.layernorm import RMSNorm
14
-from vllm.model_executor.models.llama import LlamaForCausalLM
16
+from vllm.model_executor.layers.linear import (LinearMethodBase,
17
+ MergedColumnParallelLinear,
18
+ QKVParallelLinear,
19
+ RowParallelLinear)
20
+from vllm.model_executor.layers.rotary_embedding import get_rope
21
+from vllm.model_executor.layers.sampler import Sampler
22
+from vllm.model_executor.layers.vocab_parallel_embedding import (
23
+ VocabParallelEmbedding, ParallelLMHead)
24
+from vllm.model_executor.parallel_utils.parallel_state import (
25
+ get_tensor_model_parallel_world_size)
26
+from vllm.model_executor.sampling_metadata import SamplingMetadata
27
from vllm.model_executor.weight_utils import (default_weight_loader,
28
hf_model_weights_iterator)
29
+from vllm.sequence import SamplerOutput
30
+from vllm.transformers_utils.configs.qwen import QWenConfig
31
32
+KVCache = Tuple[torch.Tensor, torch.Tensor]
33
19
-class QWenLMHeadModel(LlamaForCausalLM):
34
+
35
+class QWenMLP(nn.Module):
36
+
37
+ def __init__(
38
+ self,
39
+ hidden_size: int,
40
+ intermediate_size: int,
41
+ hidden_act: str = "silu",
42
+ linear_method: Optional[LinearMethodBase] = None,
43
+ ):
44
+ super().__init__()
45
+ self.gate_up_proj = MergedColumnParallelLinear(
46
+ hidden_size, [intermediate_size] * 2,
47
+ bias=False,
48
+ linear_method=linear_method)
49
+ self.c_proj = RowParallelLinear(intermediate_size,
50
+ hidden_size,
51
+ bias=False,
52
+ linear_method=linear_method)
53
+ if hidden_act != "silu":
54
+ raise ValueError(f"Unsupported activation: {hidden_act}. "
55
+ "Only silu is supported for now.")
56
+ self.act_fn = SiluAndMul()
57
+
58
+ def forward(self, x):
59
+ gate_up, _ = self.gate_up_proj(x)
60
+ x = self.act_fn(gate_up)
61
+ x, _ = self.c_proj(x)
62
+ return x
63
+
64
+
65
+class QWenAttention(nn.Module):
66
+
67
+ def __init__(
68
+ self,
69
+ hidden_size: int,
70
+ num_heads: int,
71
+ max_position_embeddings: int,
72
+ rope_theta: float = 10000,
73
+ rope_scaling: Optional[Dict[str, Any]] = None,
74
+ linear_method: Optional[LinearMethodBase] = None,
75
+ ):
76
+ super().__init__()
77
+ self.hidden_size = hidden_size
78
+ tensor_model_parallel_world_size = get_tensor_model_parallel_world_size(
79
+ )
80
+ self.total_num_heads = num_heads
81
+ assert self.total_num_heads % tensor_model_parallel_world_size == 0
82
+ self.num_heads = (self.total_num_heads //
83
+ tensor_model_parallel_world_size)
84
+ self.head_dim = hidden_size // self.total_num_heads
85
+ self.c_attn = QKVParallelLinear(
86
+ hidden_size,
87
+ self.head_dim,
88
+ self.total_num_heads,
89
+ bias=True,
90
+ linear_method=linear_method,
91
+ )
92
+ self.c_proj = RowParallelLinear(
93
+ self.total_num_heads * self.head_dim,
94
+ hidden_size,
95
+ bias=False,
96
+ linear_method=linear_method,
97
+ )
98
+ self.scaling = self.head_dim**-0.5
99
+
100
+ self.rotary_emb = get_rope(
101
+ self.head_dim,
102
+ rotary_dim=self.head_dim,
103
+ max_position=max_position_embeddings,
104
+ base=rope_theta,
105
+ rope_scaling=rope_scaling,
106
+ )
107
+ self.attn = PagedAttention(self.num_heads, self.head_dim, self.scaling)
108
+
109
+ def forward(
110
+ self,
111
+ positions: torch.Tensor,
112
+ hidden_states: torch.Tensor,
113
+ kv_cache: KVCache,
114
+ input_metadata: InputMetadata,
115
+ ) -> torch.Tensor:
116
+ qkv, _ = self.c_attn(hidden_states)
117
+ q, k, v = qkv.chunk(chunks=3, dim=-1)
118
+ q, k = self.rotary_emb(positions, q, k)
119
+ k_cache, v_cache = kv_cache
120
+ attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
121
+
122
+ output, _ = self.c_proj(attn_output)
123
+ return output
124
+
125
+
126
+class QWenBlock(nn.Module):
127
+
128
+ def __init__(
129
+ self,
130
+ config: QWenConfig,
131
+ linear_method: Optional[LinearMethodBase] = None,
132
+ ):
133
+ super().__init__()
134
+ self.ln_1 = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
135
+
136
+ rope_theta = getattr(config, "rope_theta", 10000)
137
+ rope_scaling = getattr(config, "rope_scaling", None)
138
+ self.attn = QWenAttention(config.hidden_size,
139
+ config.num_attention_heads,
140
+ config.max_position_embeddings,
141
+ rope_theta=rope_theta,
142
+ rope_scaling=rope_scaling,
143
+ linear_method=linear_method)
144
+
145
+ self.ln_2 = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
146
+
147
+ self.mlp = QWenMLP(config.hidden_size,
148
+ config.intermediate_size // 2,
149
+ linear_method=linear_method)
150
+
151
+ def forward(
152
+ self,
153
+ positions: torch.Tensor,
154
+ hidden_states: torch.Tensor,
155
+ kv_cache: KVCache,
156
+ input_metadata: InputMetadata,
157
+ residual: Optional[torch.Tensor],
158
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
159
+ # Self Attention
160
+ if residual is None:
161
+ residual = hidden_states
162
+ hidden_states = self.ln_1(hidden_states)
163
+ else:
164
+ hidden_states, residual = self.ln_1(hidden_states, residual)
165
+ hidden_states = self.attn(
166
+ positions=positions,
167
+ hidden_states=hidden_states,
168
+ kv_cache=kv_cache,
169
+ input_metadata=input_metadata,
170
+ )
171
+
172
+ # Fully Connected
173
+ hidden_states, residual = self.ln_2(hidden_states, residual)
174
+ hidden_states = self.mlp(hidden_states)
175
+ return hidden_states, residual
176
+
177
+
178
+class QWenModel(nn.Module):
179
180
def __init__(
181
self,
23
- config: Optional[PretrainedConfig] = None,
182
+ config: QWenConfig,
183
linear_method: Optional[LinearMethodBase] = None,
25
- lora_config: Optional[LoRAConfig] = None,
26
- ) -> None:
27
- norm = RMSNorm(config.hidden_size, config.layer_norm_epsilon)
28
- config.use_qkv_bias = True
29
- config.intermediate_size = config.intermediate_size // 2
30
- super().__init__(config=config,
31
- linear_method=linear_method,
32
- norm=norm,
33
- lora_config=lora_config)
184
+ ):
185
+ super().__init__()
186
+ self.config = config
187
+ self.vocab_size = config.vocab_size
188
+
189
+ self.wte = VocabParallelEmbedding(
190
+ config.vocab_size,
191
+ config.hidden_size,
192
+ )
193
+ self.h = nn.ModuleList([
194
+ QWenBlock(config, linear_method)
195
+ for _ in range(config.num_hidden_layers)
196
+ ])
197
+ self.ln_f = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
198
+
199
+ def forward(
200
+ self,
201
+ input_ids: torch.Tensor,
202
+ positions: torch.Tensor,
203
+ kv_caches: List[KVCache],
204
+ input_metadata: InputMetadata,
205
+ ) -> torch.Tensor:
206
+ hidden_states = self.wte(input_ids)
207
+ residual = None
208
+ for i in range(len(self.h)):
209
+ layer = self.h[i]
210
+ hidden_states, residual = layer(
211
+ positions,
212
+ hidden_states,
213
+ kv_caches[i],
214
+ input_metadata,
215
+ residual,
216
+ )
217
+ hidden_states, _ = self.ln_f(hidden_states, residual)
218
+ return hidden_states
219
+
220
+
221
+class QWenLMHeadModel(nn.Module):
222
+
223
+ def __init__(
224
+ self,
225
+ config: QWenConfig,
226
+ linear_method: Optional[LinearMethodBase] = None,
227
+ ):
228
+ super().__init__()
229
+ self.config = config
230
+ self.linear_method = linear_method
231
+ self.transformer = QWenModel(config, linear_method)
232
+ self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
233
+ self.sampler = Sampler(config.vocab_size)
234
+
235
+ def forward(
236
+ self,
237
+ input_ids: torch.Tensor,
238
+ positions: torch.Tensor,
239
+ kv_caches: List[KVCache],
240
+ input_metadata: InputMetadata,
241
+ ) -> torch.Tensor:
242
+ hidden_states = self.transformer(input_ids, positions, kv_caches,
243
+ input_metadata)
244
+ return hidden_states
245
+
246
+ def sample(
247
+ self,
248
+ hidden_states: torch.Tensor,
249
+ sampling_metadata: SamplingMetadata,
250
+ ) -> Optional[SamplerOutput]:
251
+ next_tokens = self.sampler(self.lm_head.weight, hidden_states,
252
+ sampling_metadata)
253
+ return next_tokens
254
255
def load_weights(self,
256
model_name_or_path: str,
@@ -42,24 +262,9 @@ class QWenLMHeadModel(LlamaForCausalLM):
262
("gate_up_proj", "w2", 0),
263
("gate_up_proj", "w1", 1),
264
]
45
- param_weight_map = [
46
- ("model", "transformer"),
47
- (".self_attn.", ".attn."),
48
- (".layers.", ".h."),
49
- ("qkv_proj", "c_attn"),
50
- (".self_attn.o_proj", ".self_attn.c_proj"),
51
- ("norm", "ln_f"),
52
- ("mlp.down_proj", "mlp.c_proj"),
53
- ("input_layernorm", "ln_1"),
54
- ("post_attention_layernorm", "ln_2"),
55
- ("embed_tokens", "wte"),
56
- ]
265
params_dict = dict(self.named_parameters())
266
for name, loaded_weight in hf_model_weights_iterator(
267
model_name_or_path, cache_dir, load_format, revision):
60
- for (param_name, weight_name) in param_weight_map:
61
- name = name.replace(weight_name, param_name)
62
-
268
if "rotary_emb.inv_freq" in name:
269
continue
270
for (param_name, weight_name, shard_id) in stacked_params_mapping:
vllm/model_executor/models/stablelm.py
+270
-13
@@ -17,26 +17,283 @@
17
# https://huggingface.co/stabilityai/stablelm-3b-4e1t/blob/main/modeling_stablelm_epoch.py
18
# https://huggingface.co/stabilityai/stablelm-3b-4e1t/blob/main/config.json
19
"""Inference-only StabeLM (https://github.com/Stability-AI/StableLM) model compatible with HuggingFace weights."""
20
-from typing import Optional
20
+from typing import List, Optional, Tuple
21
22
+import torch
23
+from torch import nn
24
from transformers import PretrainedConfig
25
24
-from vllm.model_executor.layers.linear import LinearMethodBase
25
-from vllm.model_executor.layers.layernorm import LayerNorm
26
-from vllm.model_executor.models.llama import LlamaForCausalLM
27
-from vllm.config import LoRAConfig
26
+from vllm.model_executor.input_metadata import InputMetadata
27
+from vllm.model_executor.layers.activation import SiluAndMul
28
+from vllm.model_executor.layers.attention import PagedAttention
29
+from vllm.model_executor.layers.linear import (LinearMethodBase,
30
+ MergedColumnParallelLinear,
31
+ QKVParallelLinear,
32
+ RowParallelLinear)
33
+from vllm.model_executor.layers.rotary_embedding import get_rope
34
+from vllm.model_executor.layers.sampler import Sampler
35
+from vllm.model_executor.layers.vocab_parallel_embedding import (
36
+ VocabParallelEmbedding, ParallelLMHead)
37
+from vllm.model_executor.parallel_utils.parallel_state import (
38
+ get_tensor_model_parallel_world_size)
39
+from vllm.model_executor.sampling_metadata import SamplingMetadata
40
+from vllm.model_executor.weight_utils import (default_weight_loader,
41
+ hf_model_weights_iterator)
42
+from vllm.sequence import SamplerOutput
43
44
+KVCache = Tuple[torch.Tensor, torch.Tensor]
45
30
-class StablelmForCausalLM(LlamaForCausalLM):
46
+
47
+class StablelmMLP(nn.Module):
48
+
49
+ def __init__(self,
50
+ config: PretrainedConfig,
51
+ linear_method: Optional[LinearMethodBase] = None) -> None:
52
+ super().__init__()
53
+ self.config = config
54
+ self.hidden_size = config.hidden_size
55
+ self.intermediate_size = config.intermediate_size
56
+ self.gate_up_proj = MergedColumnParallelLinear(
57
+ config.hidden_size, [config.intermediate_size] * 2,
58
+ bias=False,
59
+ linear_method=linear_method)
60
+ self.down_proj = RowParallelLinear(config.intermediate_size,
61
+ config.hidden_size,
62
+ bias=False)
63
+ self.act_fn = SiluAndMul()
64
+
65
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
66
+ gate_up, _ = self.gate_up_proj(x)
67
+ x = self.act_fn(gate_up)
68
+ x, _ = self.down_proj(x)
69
+ return x
70
+
71
+
72
+class StablelmAttention(nn.Module):
73
+
74
+ def __init__(self,
75
+ config: PretrainedConfig,
76
+ linear_method: Optional[LinearMethodBase] = None) -> None:
77
+ super().__init__()
78
+ self.config = config
79
+ self.hidden_size = config.hidden_size
80
+ tp_size = get_tensor_model_parallel_world_size()
81
+ self.total_num_heads = config.num_attention_heads
82
+ self.num_heads = self.total_num_heads // tp_size
83
+
84
+ self.total_num_key_value_heads = config.num_key_value_heads
85
+ if self.total_num_key_value_heads >= tp_size:
86
+ # Number of KV heads is greater than TP size, so we partition
87
+ # the KV heads across multiple tensor parallel GPUs.
88
+ assert self.total_num_key_value_heads % tp_size == 0
89
+ else:
90
+ # Number of KV heads is less than TP size, so we replicate
91
+ # the KV heads across multiple tensor parallel GPUs.
92
+ assert tp_size % self.total_num_key_value_heads == 0
93
+ self.num_key_value_heads = max(
94
+ 1, self.total_num_key_value_heads // tp_size)
95
+ self.head_dim = self.hidden_size // self.total_num_heads
96
+ self.max_position_embeddings = config.max_position_embeddings
97
+ self.rotary_ndims = int(self.head_dim * self.config.rope_pct)
98
+ self.scaling = self.head_dim**-0.5
99
+ self.q_size = self.num_heads * self.head_dim
100
+ self.kv_size = self.num_key_value_heads * self.head_dim
101
+ self.qkv_bias = getattr(config, "use_qkv_bias", False)
102
+ if (self.head_dim * self.num_heads * tp_size) != self.hidden_size:
103
+ raise ValueError(
104
+ f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}"
105
+ f" and `num_heads`: {self.num_heads}).")
106
+
107
+ self.qkv_proj = QKVParallelLinear(self.hidden_size,
108
+ self.head_dim,
109
+ self.total_num_heads,
110
+ self.total_num_key_value_heads,
111
+ self.qkv_bias,
112
+ linear_method=linear_method)
113
+ self.o_proj = RowParallelLinear(self.total_num_heads * self.head_dim,
114
+ self.hidden_size,
115
+ bias=False,
116
+ linear_method=linear_method)
117
+ self.rotary_ndims = int(self.head_dim * self.config.rope_pct)
118
+ self.rotary_emb = get_rope(
119
+ self.head_dim,
120
+ rotary_dim=self.rotary_ndims,
121
+ max_position=self.config.max_position_embeddings,
122
+ base=self.config.rope_theta,
123
+ )
124
+ self.attn = PagedAttention(self.num_heads,
125
+ self.head_dim,
126
+ self.scaling,
127
+ num_kv_heads=self.num_key_value_heads)
128
+
129
+ def forward(
130
+ self,
131
+ positions: torch.Tensor,
132
+ hidden_states: torch.Tensor,
133
+ kv_cache: KVCache,
134
+ input_metadata: InputMetadata,
135
+ ) -> torch.Tensor:
136
+ qkv, _ = self.qkv_proj(hidden_states)
137
+ q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
138
+ q, k = self.rotary_emb(positions, q, k)
139
+ k_cache, v_cache = kv_cache
140
+ attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
141
+ output, _ = self.o_proj(attn_output)
142
+ return output
143
+
144
+
145
+class StablelmDecoderLayer(nn.Module):
146
147
def __init__(
148
self,
34
- config: Optional[PretrainedConfig] = None,
149
+ config: PretrainedConfig,
150
linear_method: Optional[LinearMethodBase] = None,
36
- lora_config: Optional[LoRAConfig] = None,
151
) -> None:
38
- norm = LayerNorm(config.hidden_size, config.norm_eps)
39
- super().__init__(config=config,
40
- linear_method=linear_method,
41
- norm=norm,
42
- lora_config=lora_config)
152
+ super().__init__()
153
+ self.self_attn = StablelmAttention(config)
154
+ self.mlp = StablelmMLP(config, linear_method)
155
+ self.input_layernorm = nn.LayerNorm(config.hidden_size,
156
+ eps=config.norm_eps)
157
+ self.post_attention_layernorm = nn.LayerNorm(config.hidden_size,
158
+ eps=config.norm_eps)
159
+
160
+ def forward(
161
+ self,
162
+ positions: torch.Tensor,
163
+ hidden_states: torch.Tensor,
164
+ kv_cache: KVCache,
165
+ input_metadata: InputMetadata,
166
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
167
+ # Self Attention
168
+ residual = hidden_states
169
+ hidden_states = self.input_layernorm(hidden_states)
170
+ hidden_states = self.self_attn(
171
+ positions=positions,
172
+ hidden_states=hidden_states,
173
+ kv_cache=kv_cache,
174
+ input_metadata=input_metadata,
175
+ )
176
+ hidden_states = residual + hidden_states
177
+
178
+ # Fully Connected
179
+ residual = hidden_states
180
+ hidden_states = self.post_attention_layernorm(hidden_states)
181
+ hidden_states = self.mlp(hidden_states)
182
+ hidden_states = residual + hidden_states
183
+
184
+ return hidden_states, residual
185
+
186
+
187
+class StableLMEpochModel(nn.Module):
188
+
189
+ def __init__(self,
190
+ config: PretrainedConfig,
191
+ linear_method: Optional[LinearMethodBase] = None) -> None:
192
+ super().__init__()
193
+ # self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, config.pad_token_id)
194
+ self.embed_tokens = VocabParallelEmbedding(
195
+ config.vocab_size,
196
+ config.hidden_size,
197
+ )
198
+ self.layers = nn.ModuleList([
199
+ StablelmDecoderLayer(config, linear_method)
200
+ for _ in range(config.num_hidden_layers)
201
+ ])
202
+ self.norm = nn.LayerNorm(config.hidden_size, eps=config.norm_eps)
203
+
204
+ def forward(
205
+ self,
206
+ input_ids: torch.Tensor,
207
+ positions: torch.Tensor,
208
+ kv_caches: List[KVCache],
209
+ input_metadata: InputMetadata,
210
+ ) -> torch.Tensor:
211
+ hidden_states = self.embed_tokens(input_ids)
212
+ for i in range(len(self.layers)):
213
+ layer = self.layers[i]
214
+ hidden_states, residual = layer(
215
+ positions,
216
+ hidden_states,
217
+ kv_caches[i],
218
+ input_metadata,
219
+ )
220
+ hidden_states = self.norm(hidden_states)
221
+ return hidden_states
222
+
223
+
224
+class StablelmForCausalLM(nn.Module):
225
+
226
+ def __init__(
227
+ self,
228
+ config: PretrainedConfig,
229
+ linear_method: Optional[LinearMethodBase] = None,
230
+ ) -> None:
231
+ super().__init__()
232
+ self.config = config
233
+ self.linear_method = linear_method
234
+ self.model = StableLMEpochModel(config, linear_method)
235
+ self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
236
+ self.sampler = Sampler(config.vocab_size)
237
+
238
+ def forward(
239
+ self,
240
+ input_ids: torch.Tensor,
241
+ positions: torch.Tensor,
242
+ kv_caches: List[KVCache],
243
+ input_metadata: InputMetadata,
244
+ ) -> torch.Tensor:
245
+ hidden_states = self.model(input_ids, positions, kv_caches,
246
+ input_metadata)
247
+ return hidden_states
248
+
249
+ def sample(
250
+ self,
251
+ hidden_states: torch.Tensor,
252
+ sampling_metadata: SamplingMetadata,
253
+ ) -> Optional[SamplerOutput]:
254
+ next_tokens = self.sampler(self.lm_head.weight, hidden_states,
255
+ sampling_metadata)
256
+ return next_tokens
257
+
258
+ def load_weights(self,
259
+ model_name_or_path: str,
260
+ cache_dir: Optional[str] = None,
261
+ load_format: str = "auto",
262
+ revision: Optional[str] = None):
263
+ stacked_params_mapping = [
264
+ # (param_name, shard_name, shard_id)
265
+ ("qkv_proj", "q_proj", "q"),
266
+ ("qkv_proj", "k_proj", "k"),
267
+ ("qkv_proj", "v_proj", "v"),
268
+ ("gate_up_proj", "gate_proj", 0),
269
+ ("gate_up_proj", "up_proj", 1),
270
+ ]
271
+ params_dict = dict(self.named_parameters())
272
+ for name, loaded_weight in hf_model_weights_iterator(
273
+ model_name_or_path, cache_dir, load_format, revision):
274
+ if "rotary_emb.inv_freq" in name:
275
+ continue
276
+ if ("rotary_emb.cos_cached" in name
277
+ or "rotary_emb.sin_cached" in name):
278
+ # Models trained using ColossalAI may include these tensors in
279
+ # the checkpoint. Skip them.
280
+ continue
281
+ for (param_name, weight_name, shard_id) in stacked_params_mapping:
282
+ if weight_name not in name:
283
+ continue
284
+ name = name.replace(weight_name, param_name)
285
+ # Skip loading extra bias for GPTQ models.
286
+ if name.endswith(".bias") and name not in params_dict:
287
+ continue
288
+ param = params_dict[name]
289
+ weight_loader = param.weight_loader
290
+ weight_loader(param, loaded_weight, shard_id)
291
+ break
292
+ else:
293
+ # Skip loading extra bias for GPTQ models.
294
+ if name.endswith(".bias") and name not in params_dict:
295
+ continue
296
+ param = params_dict[name]
297
+ weight_loader = getattr(param, "weight_loader",
298
+ default_weight_loader)
299
+ weight_loader(param, loaded_weight)
vllm/model_executor/models/yi.py
new
+330
@@ -0,0 +1,330 @@
1
+# coding=utf-8
2
+# Adapted from
3
+# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
4
+# Copyright 2023 The vLLM team.
5
+# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
6
+#
7
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
8
+# and OPT implementations in this library. It has been modified from its
9
+# original forms to accommodate minor architectural differences compared
10
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
11
+#
12
+# Licensed under the Apache License, Version 2.0 (the "License");
13
+# you may not use this file except in compliance with the License.
14
+# You may obtain a copy of the License at
15
+#
16
+# http://www.apache.org/licenses/LICENSE-2.0
17
+#
18
+# Unless required by applicable law or agreed to in writing, software
19
+# distributed under the License is distributed on an "AS IS" BASIS,
20
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21
+# See the License for the specific language governing permissions and
22
+# limitations under the License.
23
+"""Inference-only Yi model (https://01.ai) compatible with HuggingFace weights."""
24
+from typing import Any, Dict, List, Optional, Tuple
25
+
26
+import torch
27
+from torch import nn
28
+from vllm.transformers_utils.configs.yi import YiConfig
29
+
30
+from vllm.model_executor.input_metadata import InputMetadata
31
+from vllm.model_executor.layers.activation import SiluAndMul
32
+from vllm.model_executor.layers.attention import PagedAttention
33
+from vllm.model_executor.layers.layernorm import RMSNorm
34
+from vllm.model_executor.layers.linear import (LinearMethodBase,
35
+ MergedColumnParallelLinear,
36
+ QKVParallelLinear,
37
+ RowParallelLinear)
38
+from vllm.model_executor.layers.rotary_embedding import get_rope
39
+from vllm.model_executor.layers.sampler import Sampler
40
+from vllm.model_executor.layers.vocab_parallel_embedding import (
41
+ VocabParallelEmbedding, ParallelLMHead)
42
+from vllm.model_executor.parallel_utils.parallel_state import (
43
+ get_tensor_model_parallel_world_size)
44
+from vllm.model_executor.sampling_metadata import SamplingMetadata
45
+from vllm.model_executor.weight_utils import (default_weight_loader,
46
+ hf_model_weights_iterator)
47
+from vllm.sequence import SamplerOutput
48
+
49
+KVCache = Tuple[torch.Tensor, torch.Tensor]
50
+
51
+
52
+class YiMLP(nn.Module):
53
+
54
+ def __init__(
55
+ self,
56
+ hidden_size: int,
57
+ intermediate_size: int,
58
+ hidden_act: str,
59
+ linear_method: Optional[LinearMethodBase] = None,
60
+ ) -> None:
61
+ super().__init__()
62
+ self.gate_up_proj = MergedColumnParallelLinear(
63
+ hidden_size, [intermediate_size] * 2,
64
+ bias=False,
65
+ linear_method=linear_method)
66
+ self.down_proj = RowParallelLinear(intermediate_size,
67
+ hidden_size,
68
+ bias=False,
69
+ linear_method=linear_method)
70
+ if hidden_act != "silu":
71
+ raise ValueError(f"Unsupported activation: {hidden_act}. "
72
+ "Only silu is supported for now.")
73
+ self.act_fn = SiluAndMul()
74
+
75
+ def forward(self, x):
76
+ gate_up, _ = self.gate_up_proj(x)
77
+ x = self.act_fn(gate_up)
78
+ x, _ = self.down_proj(x)
79
+ return x
80
+
81
+
82
+class YiAttention(nn.Module):
83
+
84
+ def __init__(
85
+ self,
86
+ hidden_size: int,
87
+ num_heads: int,
88
+ num_kv_heads: int,
89
+ rope_theta: float = 10000,
90
+ rope_scaling: Optional[Dict[str, Any]] = None,
91
+ max_position_embeddings: int = 8192,
92
+ linear_method: Optional[LinearMethodBase] = None,
93
+ ) -> None:
94
+ super().__init__()
95
+ self.hidden_size = hidden_size
96
+ tp_size = get_tensor_model_parallel_world_size()
97
+ self.total_num_heads = num_heads
98
+ assert self.total_num_heads % tp_size == 0
99
+ self.num_heads = self.total_num_heads // tp_size
100
+ self.total_num_kv_heads = num_kv_heads
101
+ if self.total_num_kv_heads >= tp_size:
102
+ # Number of KV heads is greater than TP size, so we partition
103
+ # the KV heads across multiple tensor parallel GPUs.
104
+ assert self.total_num_kv_heads % tp_size == 0
105
+ else:
106
+ # Number of KV heads is less than TP size, so we replicate
107
+ # the KV heads across multiple tensor parallel GPUs.
108
+ assert tp_size % self.total_num_kv_heads == 0
109
+ self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
110
+ self.head_dim = hidden_size // self.total_num_heads
111
+ self.q_size = self.num_heads * self.head_dim
112
+ self.kv_size = self.num_kv_heads * self.head_dim
113
+ self.scaling = self.head_dim**-0.5
114
+ self.rope_theta = rope_theta
115
+ self.max_position_embeddings = max_position_embeddings
116
+
117
+ self.qkv_proj = QKVParallelLinear(
118
+ hidden_size,
119
+ self.head_dim,
120
+ self.total_num_heads,
121
+ self.total_num_kv_heads,
122
+ bias=False,
123
+ linear_method=linear_method,
124
+ )
125
+ self.o_proj = RowParallelLinear(
126
+ self.total_num_heads * self.head_dim,
127
+ hidden_size,
128
+ bias=False,
129
+ linear_method=linear_method,
130
+ )
131
+ self.rotary_emb = get_rope(
132
+ self.head_dim,
133
+ rotary_dim=self.head_dim,
134
+ max_position=max_position_embeddings,
135
+ base=self.rope_theta,
136
+ rope_scaling=rope_scaling,
137
+ )
138
+ self.attn = PagedAttention(self.num_heads,
139
+ self.head_dim,
140
+ self.scaling,
141
+ num_kv_heads=self.num_kv_heads)
142
+
143
+ def forward(
144
+ self,
145
+ positions: torch.Tensor,
146
+ hidden_states: torch.Tensor,
147
+ kv_cache: KVCache,
148
+ input_metadata: InputMetadata,
149
+ ) -> torch.Tensor:
150
+ qkv, _ = self.qkv_proj(hidden_states)
151
+ q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
152
+ q, k = self.rotary_emb(positions, q, k)
153
+ k_cache, v_cache = kv_cache
154
+ attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
155
+ output, _ = self.o_proj(attn_output)
156
+ return output
157
+
158
+
159
+class YiDecoderLayer(nn.Module):
160
+
161
+ def __init__(
162
+ self,
163
+ config: YiConfig,
164
+ linear_method: Optional[LinearMethodBase] = None,
165
+ ) -> None:
166
+ super().__init__()
167
+ self.hidden_size = config.hidden_size
168
+ rope_theta = getattr(config, "rope_theta", 10000)
169
+ rope_scaling = getattr(config, "rope_scaling", None)
170
+ max_position_embeddings = getattr(config, "max_position_embeddings",
171
+ 8192)
172
+ self.self_attn = YiAttention(
173
+ hidden_size=self.hidden_size,
174
+ num_heads=config.num_attention_heads,
175
+ num_kv_heads=config.num_key_value_heads,
176
+ rope_theta=rope_theta,
177
+ rope_scaling=rope_scaling,
178
+ max_position_embeddings=max_position_embeddings,
179
+ linear_method=linear_method,
180
+ )
181
+ self.mlp = YiMLP(
182
+ hidden_size=self.hidden_size,
183
+ intermediate_size=config.intermediate_size,
184
+ hidden_act=config.hidden_act,
185
+ linear_method=linear_method,
186
+ )
187
+ self.ln1 = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
188
+ self.ln2 = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
189
+
190
+ def forward(
191
+ self,
192
+ positions: torch.Tensor,
193
+ hidden_states: torch.Tensor,
194
+ kv_cache: KVCache,
195
+ input_metadata: InputMetadata,
196
+ residual: Optional[torch.Tensor],
197
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
198
+ # Self Attention
199
+ if residual is None:
200
+ residual = hidden_states
201
+ hidden_states = self.ln1(hidden_states)
202
+ else:
203
+ hidden_states, residual = self.ln1(hidden_states, residual)
204
+ hidden_states = self.self_attn(
205
+ positions=positions,
206
+ hidden_states=hidden_states,
207
+ kv_cache=kv_cache,
208
+ input_metadata=input_metadata,
209
+ )
210
+
211
+ # Fully Connected
212
+ hidden_states, residual = self.ln2(hidden_states, residual)
213
+ hidden_states = self.mlp(hidden_states)
214
+ return hidden_states, residual
215
+
216
+
217
+class YiModel(nn.Module):
218
+
219
+ def __init__(
220
+ self,
221
+ config: YiConfig,
222
+ linear_method: Optional[LinearMethodBase] = None,
223
+ ) -> None:
224
+ super().__init__()
225
+ self.config = config
226
+ self.padding_idx = config.pad_token_id
227
+ self.vocab_size = config.vocab_size
228
+ self.embed_tokens = VocabParallelEmbedding(
229
+ config.vocab_size,
230
+ config.hidden_size,
231
+ )
232
+ self.layers = nn.ModuleList([
233
+ YiDecoderLayer(config, linear_method)
234
+ for _ in range(config.num_hidden_layers)
235
+ ])
236
+ self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
237
+
238
+ def forward(
239
+ self,
240
+ input_ids: torch.Tensor,
241
+ positions: torch.Tensor,
242
+ kv_caches: List[KVCache],
243
+ input_metadata: InputMetadata,
244
+ ) -> torch.Tensor:
245
+ hidden_states = self.embed_tokens(input_ids)
246
+ residual = None
247
+ for i in range(len(self.layers)):
248
+ layer = self.layers[i]
249
+ hidden_states, residual = layer(
250
+ positions,
251
+ hidden_states,
252
+ kv_caches[i],
253
+ input_metadata,
254
+ residual,
255
+ )
256
+ hidden_states, _ = self.norm(hidden_states, residual)
257
+ return hidden_states
258
+
259
+
260
+class YiForCausalLM(nn.Module):
261
+
262
+ def __init__(
263
+ self,
264
+ config: YiConfig,
265
+ linear_method: Optional[LinearMethodBase] = None,
266
+ ) -> None:
267
+ super().__init__()
268
+ self.config = config
269
+ self.linear_method = linear_method
270
+ self.model = YiModel(config, linear_method)
271
+ self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
272
+ self.sampler = Sampler(config.vocab_size)
273
+
274
+ def forward(
275
+ self,
276
+ input_ids: torch.Tensor,
277
+ positions: torch.Tensor,
278
+ kv_caches: List[KVCache],
279
+ input_metadata: InputMetadata,
280
+ ) -> torch.Tensor:
281
+ hidden_states = self.model(input_ids, positions, kv_caches,
282
+ input_metadata)
283
+ return hidden_states
284
+
285
+ def sample(
286
+ self,
287
+ hidden_states: torch.Tensor,
288
+ sampling_metadata: SamplingMetadata,
289
+ ) -> Optional[SamplerOutput]:
290
+ next_tokens = self.sampler(self.lm_head.weight, hidden_states,
291
+ sampling_metadata)
292
+ return next_tokens
293
+
294
+ def load_weights(self,
295
+ model_name_or_path: str,
296
+ cache_dir: Optional[str] = None,
297
+ load_format: str = "auto",
298
+ revision: Optional[str] = None):
299
+ stacked_params_mapping = [
300
+ # (param_name, shard_name, shard_id)
301
+ ("qkv_proj", "q_proj", "q"),
302
+ ("qkv_proj", "k_proj", "k"),
303
+ ("qkv_proj", "v_proj", "v"),
304
+ ("gate_up_proj", "gate_proj", 0),
305
+ ("gate_up_proj", "up_proj", 1),
306
+ ]
307
+ params_dict = dict(self.named_parameters())
308
+ for name, loaded_weight in hf_model_weights_iterator(
309
+ model_name_or_path, cache_dir, load_format, revision):
310
+ if "rotary_emb.inv_freq" in name:
311
+ continue
312
+ for (param_name, weight_name, shard_id) in stacked_params_mapping:
313
+ if weight_name not in name:
314
+ continue
315
+ name = name.replace(weight_name, param_name)
316
+ # Skip loading extra bias for GPTQ models.
317
+ if name.endswith(".bias") and name not in params_dict:
318
+ continue
319
+ param = params_dict[name]
320
+ weight_loader = param.weight_loader
321
+ weight_loader(param, loaded_weight, shard_id)
322
+ break
323
+ else:
324
+ # Skip loading extra bias for GPTQ models.
325
+ if name.endswith(".bias") and name not in params_dict:
326
+ continue
327
+ param = params_dict[name]
328
+ weight_loader = getattr(param, "weight_loader",
329
+ default_weight_loader)
330
+ weight_loader(param, loaded_weight)
vllm/transformers_utils/config.py
+4
@@ -5,10 +5,14 @@ from transformers import AutoConfig, PretrainedConfig
5
from vllm.transformers_utils.configs import *
6
7
_CONFIG_REGISTRY = {
8
+ "aquila": AquilaConfig,
9
+ "baichuan": BaiChuanConfig,
10
"chatglm": ChatGLMConfig,
11
"mpt": MPTConfig,
12
+ "qwen": QWenConfig,
13
"RefinedWeb": RWConfig, # For tiiuae/falcon-40b(-instruct)
14
"RefinedWebModel": RWConfig, # For tiiuae/falcon-7b(-instruct)
15
+ "yi": YiConfig,
16
}
17
18
vllm/transformers_utils/configs/__init__.py
+8
@@ -1,12 +1,20 @@
1
+from vllm.transformers_utils.configs.aquila import AquilaConfig
2
+from vllm.transformers_utils.configs.baichuan import BaiChuanConfig
3
from vllm.transformers_utils.configs.chatglm import ChatGLMConfig
4
from vllm.transformers_utils.configs.mpt import MPTConfig
5
+from vllm.transformers_utils.configs.qwen import QWenConfig
6
# RWConfig is for the original tiiuae/falcon-40b(-instruct) and
7
# tiiuae/falcon-7b(-instruct) models. Newer Falcon models will use the
8
# `FalconConfig` class from the official HuggingFace transformers library.
9
from vllm.transformers_utils.configs.falcon import RWConfig
10
+from vllm.transformers_utils.configs.yi import YiConfig
11
12
__all__ = [
13
+ "AquilaConfig",
14
+ "BaiChuanConfig",
15
"ChatGLMConfig",
16
"MPTConfig",
17
+ "QWenConfig",
18
"RWConfig",
19
+ "YiConfig",
20
]
vllm/transformers_utils/configs/aquila.py
new
+69
@@ -0,0 +1,69 @@
1
+# coding=utf-8
2
+# Copyright 2023 EleutherAI and the HuggingFace Inc. team. All rights reserved.
3
+#
4
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
5
+# and OPT implementations in this library. It has been modified from its
6
+# original forms to accommodate minor architectural differences compared
7
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
8
+#
9
+# Licensed under the Apache License, Version 2.0 (the "License");
10
+# you may not use this file except in compliance with the License.
11
+# You may obtain a copy of the License at
12
+#
13
+# http://www.apache.org/licenses/LICENSE-2.0
14
+#
15
+# Unless required by applicable law or agreed to in writing, software
16
+# distributed under the License is distributed on an "AS IS" BASIS,
17
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18
+# See the License for the specific language governing permissions and
19
+# limitations under the License.
20
+""" Aquila model configuration"""
21
+
22
+from transformers import PretrainedConfig
23
+
24
+
25
+class AquilaConfig(PretrainedConfig):
26
+ model_type = "aquila"
27
+ keys_to_ignore_at_inference = ["past_key_values"]
28
+
29
+ def __init__(
30
+ self,
31
+ vocab_size=100008,
32
+ hidden_size=4096,
33
+ intermediate_size=11008,
34
+ num_hidden_layers=32,
35
+ num_attention_heads=32,
36
+ num_key_value_heads=None,
37
+ hidden_act="silu",
38
+ max_position_embeddings=2048,
39
+ initializer_range=0.006,
40
+ rms_norm_eps=1e-5,
41
+ use_cache=True,
42
+ pad_token_id=0,
43
+ bos_token_id=1,
44
+ eos_token_id=2,
45
+ tie_word_embeddings=False,
46
+ **kwargs,
47
+ ):
48
+ self.vocab_size = vocab_size
49
+ self.max_position_embeddings = max_position_embeddings
50
+ self.hidden_size = hidden_size
51
+ self.intermediate_size = intermediate_size
52
+ self.num_hidden_layers = num_hidden_layers
53
+ # for backward compatibility
54
+ if num_key_value_heads is None:
55
+ num_key_value_heads = num_attention_heads
56
+
57
+ self.num_key_value_heads = num_key_value_heads
58
+ self.num_attention_heads = num_attention_heads
59
+ self.hidden_act = hidden_act
60
+ self.initializer_range = initializer_range
61
+ self.rms_norm_eps = rms_norm_eps
62
+ self.use_cache = use_cache
63
+ super().__init__(
64
+ pad_token_id=pad_token_id,
65
+ bos_token_id=bos_token_id,
66
+ eos_token_id=eos_token_id,
67
+ tie_word_embeddings=tie_word_embeddings,
68
+ **kwargs,
69
+ )
vllm/transformers_utils/configs/baichuan.py
new
+62
@@ -0,0 +1,62 @@
1
+# coding=utf-8
2
+# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
3
+#
4
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
5
+# and OPT implementations in this library. It has been modified from its
6
+# original forms to accommodate minor architectural differences compared
7
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
8
+#
9
+# Licensed under the Apache License, Version 2.0 (the "License");
10
+# you may not use this file except in compliance with the License.
11
+# You may obtain a copy of the License at
12
+#
13
+# http://www.apache.org/licenses/LICENSE-2.0
14
+#
15
+# Unless required by applicable law or agreed to in writing, software
16
+# distributed under the License is distributed on an "AS IS" BASIS,
17
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18
+# See the License for the specific language governing permissions and
19
+# limitations under the License.
20
+
21
+from transformers.configuration_utils import PretrainedConfig
22
+
23
+
24
+class BaiChuanConfig(PretrainedConfig):
25
+ model_type = "baichuan"
26
+ keys_to_ignore_at_inference = ["past_key_values"]
27
+
28
+ def __init__(
29
+ self,
30
+ vocab_size=64000,
31
+ hidden_size=4096,
32
+ intermediate_size=11008,
33
+ num_hidden_layers=32,
34
+ num_attention_heads=32,
35
+ hidden_act="silu",
36
+ max_position_embeddings=4096,
37
+ initializer_range=0.02,
38
+ rms_norm_eps=1e-6,
39
+ use_cache=True,
40
+ pad_token_id=0,
41
+ bos_token_id=1,
42
+ eos_token_id=2,
43
+ tie_word_embeddings=False,
44
+ **kwargs,
45
+ ):
46
+ self.vocab_size = vocab_size
47
+ self.max_position_embeddings = max_position_embeddings
48
+ self.hidden_size = hidden_size
49
+ self.intermediate_size = intermediate_size
50
+ self.num_hidden_layers = num_hidden_layers
51
+ self.num_attention_heads = num_attention_heads
52
+ self.hidden_act = hidden_act
53
+ self.initializer_range = initializer_range
54
+ self.rms_norm_eps = rms_norm_eps
55
+ self.use_cache = use_cache
56
+ super().__init__(
57
+ pad_token_id=pad_token_id,
58
+ bos_token_id=bos_token_id,
59
+ eos_token_id=eos_token_id,
60
+ tie_word_embeddings=tie_word_embeddings,
61
+ **kwargs,
62
+ )
vllm/transformers_utils/configs/qwen.py
new
+60
@@ -0,0 +1,60 @@
1
+# Copyright (c) Alibaba Cloud.
2
+# LICENSE: https://huggingface.co/Qwen/Qwen-7B/blob/main/LICENSE
3
+
4
+from transformers import PretrainedConfig
5
+
6
+
7
+class QWenConfig(PretrainedConfig):
8
+ model_type = "qwen"
9
+ keys_to_ignore_at_inference = ["past_key_values"]
10
+
11
+ def __init__(
12
+ self,
13
+ vocab_size=151936,
14
+ hidden_size=4096,
15
+ num_hidden_layers=32,
16
+ num_attention_heads=32,
17
+ emb_dropout_prob=0.0,
18
+ attn_dropout_prob=0.0,
19
+ layer_norm_epsilon=1e-6,
20
+ initializer_range=0.02,
21
+ max_position_embeddings=8192,
22
+ scale_attn_weights=True,
23
+ use_cache=True,
24
+ bf16=False,
25
+ fp16=False,
26
+ fp32=False,
27
+ kv_channels=128,
28
+ rotary_pct=1.0,
29
+ rotary_emb_base=10000,
30
+ use_dynamic_ntk=True,
31
+ use_logn_attn=True,
32
+ use_flash_attn="auto",
33
+ intermediate_size=22016,
34
+ no_bias=True,
35
+ tie_word_embeddings=False,
36
+ **kwargs,
37
+ ):
38
+ self.vocab_size = vocab_size
39
+ self.hidden_size = hidden_size
40
+ self.intermediate_size = intermediate_size
41
+ self.num_hidden_layers = num_hidden_layers
42
+ self.num_attention_heads = num_attention_heads
43
+ self.emb_dropout_prob = emb_dropout_prob
44
+ self.attn_dropout_prob = attn_dropout_prob
45
+ self.layer_norm_epsilon = layer_norm_epsilon
46
+ self.initializer_range = initializer_range
47
+ self.scale_attn_weights = scale_attn_weights
48
+ self.use_cache = use_cache
49
+ self.max_position_embeddings = max_position_embeddings
50
+ self.bf16 = bf16
51
+ self.fp16 = fp16
52
+ self.fp32 = fp32
53
+ self.kv_channels = kv_channels
54
+ self.rotary_pct = rotary_pct
55
+ self.rotary_emb_base = rotary_emb_base
56
+ self.use_dynamic_ntk = use_dynamic_ntk
57
+ self.use_logn_attn = use_logn_attn
58
+ self.use_flash_attn = use_flash_attn
59
+ self.no_bias = no_bias
60
+ super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)
vllm/transformers_utils/configs/yi.py
new
+64
@@ -0,0 +1,64 @@
1
+""" Yi model configuration"""
2
+from transformers.configuration_utils import PretrainedConfig
3
+from transformers.utils import logging
4
+
5
+logger = logging.get_logger(__name__)
6
+
7
+Yi_PRETRAINED_CONFIG_ARCHIVE_MAP = {}
8
+
9
+
10
+class YiConfig(PretrainedConfig):
11
+ r"""
12
+ Reference:
13
+ https://huggingface.co/01-ai/Yi-6B/blob/main/configuration_yi.py
14
+ """
15
+ model_type = "Yi"
16
+ keys_to_ignore_at_inference = ["past_key_values"]
17
+
18
+ def __init__(
19
+ self,
20
+ vocab_size=64000,
21
+ hidden_size=4096,
22
+ intermediate_size=11008,
23
+ num_hidden_layers=32,
24
+ num_attention_heads=32,
25
+ num_key_value_heads=4,
26
+ hidden_act="silu",
27
+ max_position_embeddings=4096,
28
+ initializer_range=0.02,
29
+ rms_norm_eps=1e-5,
30
+ use_cache=True,
31
+ pad_token_id=0,
32
+ bos_token_id=1,
33
+ eos_token_id=2,
34
+ tie_word_embeddings=False,
35
+ output_attentions=False,
36
+ rope_theta=5000000.0,
37
+ **kwargs,
38
+ ):
39
+ self.vocab_size = vocab_size
40
+ self.max_position_embeddings = max_position_embeddings
41
+ self.hidden_size = hidden_size
42
+ self.intermediate_size = intermediate_size
43
+ self.num_hidden_layers = num_hidden_layers
44
+ self.num_attention_heads = num_attention_heads
45
+
46
+ # for backward compatibility
47
+ if num_key_value_heads is None:
48
+ num_key_value_heads = num_attention_heads
49
+
50
+ self.num_key_value_heads = num_key_value_heads
51
+ self.hidden_act = hidden_act
52
+ self.initializer_range = initializer_range
53
+ self.rms_norm_eps = rms_norm_eps
54
+ self.use_cache = use_cache
55
+ self.output_attentions = output_attentions
56
+ self.rope_theta = rope_theta
57
+
58
+ super().__init__(
59
+ pad_token_id=pad_token_id,
60
+ bos_token_id=bos_token_id,
61
+ eos_token_id=eos_token_id,
62
+ tie_word_embeddings=tie_word_embeddings,
63
+ **kwargs,
64
+ )