1
+# coding=utf-8
2
+# Adapted from
3
+# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
4
+# Copyright 2023 The vLLM team.
5
+# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
6
+#
7
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
8
+# and OPT implementations in this library. It has been modified from its
9
+# original forms to accommodate minor architectural differences compared
10
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
11
+#
12
+# Licensed under the Apache License, Version 2.0 (the "License");
13
+# you may not use this file except in compliance with the License.
14
+# You may obtain a copy of the License at
15
+#
16
+# http://www.apache.org/licenses/LICENSE-2.0
17
+#
18
+# Unless required by applicable law or agreed to in writing, software
19
+# distributed under the License is distributed on an "AS IS" BASIS,
20
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21
+# See the License for the specific language governing permissions and
22
+# limitations under the License.
23
+"""Inference-only LLaMA model compatible with HuggingFace weights.
24
+
25
+The input of the model is flattened to a 1D tensor of tokens. The model uses
26
+InputMetadata to extract the original 2D shape of the input.
27
+"""
28
+from typing import Dict, List, Optional, Tuple
29
+
30
+import torch
31
+from torch import nn
32
+
33
+from vllm.model_executor.input_metadata import InputMetadata
34
+from vllm.model_executor.layers.activation import SiluAndMul
35
+from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
36
+from vllm.model_executor.layers.sampler import Sampler
37
+from vllm.model_executor.weight_utils import (hf_model_weights_iterator,
38
+ load_tensor_parallel_weights)
39
+from vllm.model_executor.parallel_utils.parallel_state import (
40
+ get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
41
+from vllm.model_executor.parallel_utils.tensor_parallel import (
42
+ VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
43
+from vllm.sequence import SequenceOutputs
44
+from vllm.transformers_utils.configs.aquila import AquilaConfig
45
+
46
+KVCache = Tuple[torch.Tensor, torch.Tensor]
47
+
48
+
49
+class AquilaMLP(nn.Module):
50
+
51
+ def __init__(
52
+ self,
53
+ hidden_size: int,
54
+ intermediate_size: int,
55
+ hidden_act: str,
56
+ ):
57
+ super().__init__()
58
+ self.gate_up_proj = ColumnParallelLinear(hidden_size,
59
+ 2 * intermediate_size,
60
+ bias=False,
61
+ gather_output=False,
62
+ perform_initialization=False)
63
+ self.down_proj = RowParallelLinear(intermediate_size,
64
+ hidden_size,
65
+ bias=False,
66
+ input_is_parallel=True,
67
+ perform_initialization=False)
68
+ if hidden_act != "silu":
69
+ raise ValueError(f"Unsupported activation: {hidden_act}. "
70
+ "Only silu is supported for now.")
71
+ self.act_fn = SiluAndMul()
72
+
73
+ def forward(self, x):
74
+ gate_up, _ = self.gate_up_proj(x)
75
+ x = self.act_fn(gate_up)
76
+ x, _ = self.down_proj(x)
77
+ return x
78
+
79
+
80
+class AquilaRMSNorm(nn.Module):
81
+
82
+ def __init__(self, hidden_size, eps=1e-6):
83
+ """
84
+ AquilaRMSNorm is equivalent to T5LayerNorm
85
+ """
86
+ super().__init__()
87
+ self.weight = nn.Parameter(torch.ones(hidden_size))
88
+ self.variance_epsilon = eps
89
+
90
+ def forward(self, hidden_states):
91
+ input_dtype = hidden_states.dtype
92
+ variance = hidden_states.to(torch.float32).pow(2).mean(-1,
93
+ keepdim=True)
94
+ hidden_states = hidden_states * torch.rsqrt(variance +
95
+ self.variance_epsilon)
96
+
97
+ return (self.weight * hidden_states).to(input_dtype)
98
+
99
+
100
+class AquilaAttention(nn.Module):
101
+
102
+ def __init__(
103
+ self,
104
+ hidden_size: int,
105
+ num_heads: int,
106
+ num_kv_heads: int,
107
+ ):
108
+ super().__init__()
109
+ self.hidden_size = hidden_size
110
+ tp_size = get_tensor_model_parallel_world_size()
111
+ self.total_num_heads = num_heads
112
+ assert self.total_num_heads % tp_size == 0
113
+ self.num_heads = self.total_num_heads // tp_size
114
+ self.total_num_kv_heads = num_kv_heads
115
+ assert self.total_num_kv_heads % tp_size == 0
116
+ self.num_kv_heads = self.total_num_kv_heads // tp_size
117
+ self.head_dim = hidden_size // self.total_num_heads
118
+ self.q_size = self.num_heads * self.head_dim
119
+ self.kv_size = self.num_kv_heads * self.head_dim
120
+ self.scaling = self.head_dim**-0.5
121
+
122
+ self.qkv_proj = ColumnParallelLinear(
123
+ hidden_size,
124
+ (self.total_num_heads + 2 * self.total_num_kv_heads) *
125
+ self.head_dim,
126
+ bias=False,
127
+ gather_output=False,
128
+ perform_initialization=False,
129
+ )
130
+ self.o_proj = RowParallelLinear(
131
+ self.total_num_heads * self.head_dim,
132
+ hidden_size,
133
+ bias=False,
134
+ input_is_parallel=True,
135
+ perform_initialization=False,
136
+ )
137
+ self.attn = PagedAttentionWithRoPE(
138
+ self.num_heads,
139
+ self.head_dim,
140
+ self.scaling,
141
+ rotary_dim=self.head_dim,
142
+ )
143
+
144
+ def forward(
145
+ self,
146
+ positions: torch.Tensor,
147
+ hidden_states: torch.Tensor,
148
+ kv_cache: KVCache,
149
+ input_metadata: InputMetadata,
150
+ cache_event: Optional[torch.cuda.Event],
151
+ ) -> torch.Tensor:
152
+ qkv, _ = self.qkv_proj(hidden_states)
153
+ q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
154
+ k_cache, v_cache = kv_cache
155
+ attn_output = self.attn(positions, q, k, v, k_cache, v_cache,
156
+ input_metadata, cache_event)
157
+ output, _ = self.o_proj(attn_output)
158
+ return output
159
+
160
+
161
+class AquilaDecoderLayer(nn.Module):
162
+
163
+ def __init__(self, config: AquilaConfig):
164
+ super().__init__()
165
+ self.hidden_size = config.hidden_size
166
+ self.self_attn = AquilaAttention(
167
+ hidden_size=self.hidden_size,
168
+ num_heads=config.num_attention_heads,
169
+ num_kv_heads=config.num_attention_heads,
170
+ )
171
+ self.mlp = AquilaMLP(
172
+ hidden_size=self.hidden_size,
173
+ intermediate_size=config.intermediate_size,
174
+ hidden_act=config.hidden_act,
175
+ )
176
+ self.input_layernorm = AquilaRMSNorm(config.hidden_size,
177
+ eps=config.rms_norm_eps)
178
+ self.post_attention_layernorm = AquilaRMSNorm(config.hidden_size,
179
+ eps=config.rms_norm_eps)
180
+
181
+ def forward(
182
+ self,
183
+ positions: torch.Tensor,
184
+ hidden_states: torch.Tensor,
185
+ kv_cache: KVCache,
186
+ input_metadata: InputMetadata,
187
+ cache_event: Optional[torch.cuda.Event],
188
+ ) -> torch.Tensor:
189
+ # Self Attention
190
+ residual = hidden_states
191
+ hidden_states = self.input_layernorm(hidden_states)
192
+ hidden_states = self.self_attn(
193
+ positions=positions,
194
+ hidden_states=hidden_states,
195
+ kv_cache=kv_cache,
196
+ input_metadata=input_metadata,
197
+ cache_event=cache_event,
198
+ )
199
+ hidden_states = residual + hidden_states
200
+
201
+ # Fully Connected
202
+ residual = hidden_states
203
+ hidden_states = self.post_attention_layernorm(hidden_states)
204
+ hidden_states = self.mlp(hidden_states)
205
+ hidden_states = residual + hidden_states
206
+ return hidden_states
207
+
208
+
209
+class AquilaModel(nn.Module):
210
+
211
+ def __init__(self, config: AquilaConfig):
212
+ super().__init__()
213
+ self.config = config
214
+ self.padding_idx = config.pad_token_id
215
+ self.vocab_size = config.vocab_size
216
+
217
+ #vocab_size = ((config.vocab_size + 63) // 64) * 64
218
+ self.embed_tokens = VocabParallelEmbedding(
219
+ config.vocab_size,
220
+ config.hidden_size,
221
+ perform_initialization=False)
222
+ self.layers = nn.ModuleList([
223
+ AquilaDecoderLayer(config) for _ in range(config.num_hidden_layers)
224
+ ])
225
+ self.norm = AquilaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
226
+
227
+ def forward(
228
+ self,
229
+ input_ids: torch.Tensor,
230
+ positions: torch.Tensor,
231
+ kv_caches: List[KVCache],
232
+ input_metadata: InputMetadata,
233
+ cache_events: Optional[List[torch.cuda.Event]],
234
+ ) -> torch.Tensor:
235
+ hidden_states = self.embed_tokens(input_ids)
236
+ for i in range(len(self.layers)):
237
+ if cache_events is None:
238
+ cache_event = None
239
+ else:
240
+ cache_event = cache_events[i]
241
+ layer = self.layers[i]
242
+ hidden_states = layer(
243
+ positions,
244
+ hidden_states,
245
+ kv_caches[i],
246
+ input_metadata,
247
+ cache_event,
248
+ )
249
+ hidden_states = self.norm(hidden_states)
250
+
251
+ return hidden_states
252
+
253
+
254
+class AquilaForCausalLM(nn.Module):
255
+
256
+ def __init__(self, config):
257
+ super().__init__()
258
+ self.config = config
259
+ self.model = AquilaModel(config)
260
+ vocab_size = ((config.vocab_size + 63) // 64) * 64
261
+ self.lm_head = ColumnParallelLinear(config.hidden_size,
262
+ vocab_size,
263
+ bias=False,
264
+ gather_output=False,
265
+ perform_initialization=False)
266
+ self.sampler = Sampler(config.vocab_size)
267
+
268
+ def forward(
269
+ self,
270
+ input_ids: torch.Tensor,
271
+ positions: torch.Tensor,
272
+ kv_caches: List[KVCache],
273
+ input_metadata: InputMetadata,
274
+ cache_events: Optional[List[torch.cuda.Event]],
275
+ ) -> Dict[int, SequenceOutputs]:
276
+ hidden_states = self.model(input_ids, positions, kv_caches,
277
+ input_metadata, cache_events)
278
+ next_tokens = self.sampler(self.lm_head.weight, hidden_states,
279
+ input_metadata)
280
+ return next_tokens
281
+
282
+ _column_parallel_weights = [
283
+ "embed_tokens.weight", "lm_head.weight", "qkv_proj.weight",
284
+ "gate_proj.weight", "up_proj.weight"
285
+ ]
286
+ _row_parallel_weights = ["o_proj.weight", "down_proj.weight"]
287
+
288
+ def load_weights(self,
289
+ model_name_or_path: str,
290
+ cache_dir: Optional[str] = None,
291
+ use_np_cache: bool = False):
292
+ tp_size = get_tensor_model_parallel_world_size()
293
+ tensor_model_parallel_rank = get_tensor_model_parallel_rank()
294
+ q_proj_shard_size = (self.config.hidden_size // tp_size)
295
+ kv_proj_shard_size = (self.config.hidden_size //
296
+ self.config.num_attention_heads *
297
+ self.config.num_attention_heads // tp_size)
298
+ attention_weight_specs = [
299
+ # (weight_name, shard_size, offset)
300
+ ("q_proj", q_proj_shard_size, 0),
301
+ ("k_proj", kv_proj_shard_size, q_proj_shard_size),
302
+ ("v_proj", kv_proj_shard_size,
303
+ q_proj_shard_size + kv_proj_shard_size),
304
+ ]
305
+ state_dict = self.state_dict()
306
+
307
+ for name, loaded_weight in hf_model_weights_iterator(
308
+ model_name_or_path, cache_dir, use_np_cache):
309
+ if "rotary_emb.inv_freq" in name:
310
+ continue
311
+
312
+ if "embed_tokens" in name or "lm_head" in name:
313
+ param = state_dict[name]
314
+ # Consider padding in the vocab size.
315
+ padded_vocab_size = (param.shape[0] * tp_size)
316
+ num_extra_rows = padded_vocab_size - self.config.vocab_size
317
+ extra_rows = torch.empty(num_extra_rows,
318
+ loaded_weight.shape[1])
319
+ extra_rows = extra_rows.to(loaded_weight)
320
+ loaded_weight = torch.cat([loaded_weight, extra_rows], dim=0)
321
+
322
+ is_attention_weight = False
323
+ for weight_name, shard_size, offset in attention_weight_specs:
324
+ if weight_name not in name:
325
+ continue
326
+ param = state_dict[name.replace(weight_name, "qkv_proj")]
327
+
328
+ loaded_weight = loaded_weight[
329
+ shard_size * tensor_model_parallel_rank:shard_size *
330
+ (tensor_model_parallel_rank + 1)]
331
+ param_slice = param.data[offset:offset + shard_size]
332
+ assert param_slice.shape == loaded_weight.shape
333
+
334
+ param_slice.copy_(loaded_weight)
335
+ is_attention_weight = True
336
+ break
337
+ if is_attention_weight:
338
+ continue
339
+
340
+ is_gate_up_weight = False
341
+ for stride_id, weight_name in enumerate(["gate_proj", "up_proj"]):
342
+ if weight_name not in name:
343
+ continue
344
+ param = state_dict[name.replace(weight_name, "gate_up_proj")]
345
+ shard_size = param.shape[0] // 2
346
+ loaded_weight = loaded_weight[
347
+ shard_size * tensor_model_parallel_rank:shard_size *
348
+ (tensor_model_parallel_rank + 1)]
349
+ param_slice = param.data[shard_size * stride_id:shard_size *
350
+ (stride_id + 1)]
351
+ assert param_slice.shape == loaded_weight.shape
352
+ param_slice.copy_(loaded_weight)
353
+ is_gate_up_weight = True
354
+ break
355
+ if is_gate_up_weight:
356
+ continue
357
+
358
+ param = state_dict[name]
359
+ load_tensor_parallel_weights(param, loaded_weight, name,
360
+ self._column_parallel_weights,
361
+ self._row_parallel_weights,
362
+ tensor_model_parallel_rank)