A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Migrate AquilaForCausalLM to LlamaForCausalLM (#2867)
Roy committed
Feb 15, 2024 at 04:30 UTC
4efbac6d3593ed35fd5b6ccb3958bd96b2c9b4da
5 files changed
+2
-417
vllm/model_executor/models/__init__.py
+2
-3
@@ -10,8 +10,8 @@ logger = init_logger(__name__)
10
11
# Architecture -> (module, class).
12
_MODELS = {
13
- "AquilaModel": ("aquila", "AquilaForCausalLM"),
14
- "AquilaForCausalLM": ("aquila", "AquilaForCausalLM"), # AquilaChat2
13
+ "AquilaModel": ("llama", "LlamaForCausalLM"),
14
+ "AquilaForCausalLM": ("llama", "LlamaForCausalLM"), # AquilaChat2
15
"BaiChuanForCausalLM": ("baichuan", "BaiChuanForCausalLM"), # baichuan-7b
16
"BaichuanForCausalLM": ("baichuan", "BaichuanForCausalLM"), # baichuan-13b
17
"BloomForCausalLM": ("bloom", "BloomForCausalLM"),
@@ -41,7 +41,6 @@ _MODELS = {
41
"Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
42
"RWForCausalLM": ("falcon", "FalconForCausalLM"),
43
"StableLMEpochForCausalLM": ("stablelm", "StablelmForCausalLM"),
44
- "YiForCausalLM": ("yi", "YiForCausalLM")
44
}
45
46
# Models not supported by ROCm.
vllm/model_executor/models/aquila.py
deleted
-342
@@ -1,342 +0,0 @@
1
-# coding=utf-8
2
-# Adapted from
3
-# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
4
-# Copyright 2023 The vLLM team.
5
-# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
6
-#
7
-# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
8
-# and OPT implementations in this library. It has been modified from its
9
-# original forms to accommodate minor architectural differences compared
10
-# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
11
-#
12
-# Licensed under the Apache License, Version 2.0 (the "License");
13
-# you may not use this file except in compliance with the License.
14
-# You may obtain a copy of the License at
15
-#
16
-# http://www.apache.org/licenses/LICENSE-2.0
17
-#
18
-# Unless required by applicable law or agreed to in writing, software
19
-# distributed under the License is distributed on an "AS IS" BASIS,
20
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21
-# See the License for the specific language governing permissions and
22
-# limitations under the License.
23
-"""Inference-only LLaMA model compatible with HuggingFace weights."""
24
-from typing import Any, Dict, List, Optional, Tuple
25
-
26
-import torch
27
-from torch import nn
28
-
29
-from vllm.model_executor.input_metadata import InputMetadata
30
-from vllm.model_executor.layers.activation import SiluAndMul
31
-from vllm.model_executor.layers.attention import PagedAttention
32
-from vllm.model_executor.layers.linear import (LinearMethodBase,
33
- MergedColumnParallelLinear,
34
- QKVParallelLinear,
35
- RowParallelLinear)
36
-from vllm.model_executor.layers.rotary_embedding import get_rope
37
-from vllm.model_executor.layers.sampler import Sampler
38
-from vllm.model_executor.layers.vocab_parallel_embedding import (
39
- VocabParallelEmbedding, ParallelLMHead)
40
-from vllm.model_executor.parallel_utils.parallel_state import (
41
- get_tensor_model_parallel_world_size)
42
-from vllm.model_executor.sampling_metadata import SamplingMetadata
43
-from vllm.model_executor.weight_utils import (default_weight_loader,
44
- hf_model_weights_iterator)
45
-from vllm.sequence import SamplerOutput
46
-from vllm.transformers_utils.configs.aquila import AquilaConfig
47
-
48
-KVCache = Tuple[torch.Tensor, torch.Tensor]
49
-
50
-
51
-class AquilaMLP(nn.Module):
52
-
53
- def __init__(
54
- self,
55
- hidden_size: int,
56
- intermediate_size: int,
57
- hidden_act: str,
58
- linear_method: Optional[LinearMethodBase] = None,
59
- ):
60
- super().__init__()
61
- self.gate_up_proj = MergedColumnParallelLinear(
62
- hidden_size, [intermediate_size] * 2,
63
- bias=False,
64
- linear_method=linear_method)
65
- self.down_proj = RowParallelLinear(intermediate_size,
66
- hidden_size,
67
- bias=False,
68
- linear_method=linear_method)
69
- if hidden_act != "silu":
70
- raise ValueError(f"Unsupported activation: {hidden_act}. "
71
- "Only silu is supported for now.")
72
- self.act_fn = SiluAndMul()
73
-
74
- def forward(self, x):
75
- gate_up, _ = self.gate_up_proj(x)
76
- x = self.act_fn(gate_up)
77
- x, _ = self.down_proj(x)
78
- return x
79
-
80
-
81
-class AquilaRMSNorm(nn.Module):
82
-
83
- def __init__(self, hidden_size, eps=1e-6):
84
- """
85
- AquilaRMSNorm is equivalent to T5LayerNorm
86
- """
87
- super().__init__()
88
- self.weight = nn.Parameter(torch.ones(hidden_size))
89
- self.variance_epsilon = eps
90
-
91
- def forward(self, hidden_states):
92
- input_dtype = hidden_states.dtype
93
- variance = hidden_states.to(torch.float32).pow(2).mean(-1,
94
- keepdim=True)
95
- hidden_states = hidden_states * torch.rsqrt(variance +
96
- self.variance_epsilon)
97
-
98
- return (self.weight * hidden_states).to(input_dtype)
99
-
100
-
101
-class AquilaAttention(nn.Module):
102
-
103
- def __init__(
104
- self,
105
- hidden_size: int,
106
- num_heads: int,
107
- num_kv_heads: int,
108
- rope_theta: float = 10000,
109
- max_position_embeddings: int = 8192,
110
- rope_scaling: Optional[Dict[str, Any]] = None,
111
- linear_method: Optional[LinearMethodBase] = None,
112
- ):
113
- super().__init__()
114
- self.hidden_size = hidden_size
115
- tp_size = get_tensor_model_parallel_world_size()
116
- self.total_num_heads = num_heads
117
- assert self.total_num_heads % tp_size == 0
118
- self.num_heads = self.total_num_heads // tp_size
119
- self.total_num_kv_heads = num_kv_heads
120
- assert self.total_num_kv_heads % tp_size == 0
121
- self.num_kv_heads = self.total_num_kv_heads // tp_size
122
- self.head_dim = hidden_size // self.total_num_heads
123
- self.q_size = self.num_heads * self.head_dim
124
- self.kv_size = self.num_kv_heads * self.head_dim
125
- self.scaling = self.head_dim**-0.5
126
- self.rope_theta = rope_theta
127
- self.max_position_embeddings = max_position_embeddings
128
-
129
- self.qkv_proj = QKVParallelLinear(
130
- hidden_size,
131
- self.head_dim,
132
- self.total_num_heads,
133
- self.total_num_kv_heads,
134
- bias=False,
135
- linear_method=linear_method,
136
- )
137
- self.o_proj = RowParallelLinear(
138
- self.total_num_heads * self.head_dim,
139
- hidden_size,
140
- bias=False,
141
- linear_method=linear_method,
142
- )
143
- self.rotary_emb = get_rope(
144
- self.head_dim,
145
- rotary_dim=self.head_dim,
146
- max_position=self.max_position_embeddings,
147
- base=self.rope_theta,
148
- rope_scaling=rope_scaling,
149
- )
150
- self.attn = PagedAttention(self.num_heads,
151
- self.head_dim,
152
- self.scaling,
153
- num_kv_heads=self.num_kv_heads)
154
-
155
- def forward(
156
- self,
157
- positions: torch.Tensor,
158
- hidden_states: torch.Tensor,
159
- kv_cache: KVCache,
160
- input_metadata: InputMetadata,
161
- ) -> torch.Tensor:
162
- qkv, _ = self.qkv_proj(hidden_states)
163
- q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
164
- q, k = self.rotary_emb(positions, q, k)
165
- k_cache, v_cache = kv_cache
166
- attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
167
- output, _ = self.o_proj(attn_output)
168
- return output
169
-
170
-
171
-class AquilaDecoderLayer(nn.Module):
172
-
173
- def __init__(
174
- self,
175
- config: AquilaConfig,
176
- linear_method: Optional[LinearMethodBase] = None,
177
- ):
178
- super().__init__()
179
- self.hidden_size = config.hidden_size
180
- rope_theta = getattr(config, "rope_theta", 10000)
181
- rope_scaling = getattr(config, "rope_scaling", None)
182
- max_position_embeddings = getattr(config, "max_position_embeddings",
183
- 8192)
184
- self.self_attn = AquilaAttention(
185
- hidden_size=self.hidden_size,
186
- num_heads=config.num_attention_heads,
187
- num_kv_heads=config.num_key_value_heads,
188
- rope_theta=rope_theta,
189
- max_position_embeddings=max_position_embeddings,
190
- rope_scaling=rope_scaling,
191
- linear_method=linear_method,
192
- )
193
- self.mlp = AquilaMLP(
194
- hidden_size=self.hidden_size,
195
- intermediate_size=config.intermediate_size,
196
- hidden_act=config.hidden_act,
197
- linear_method=linear_method,
198
- )
199
- self.input_layernorm = AquilaRMSNorm(config.hidden_size,
200
- eps=config.rms_norm_eps)
201
- self.post_attention_layernorm = AquilaRMSNorm(config.hidden_size,
202
- eps=config.rms_norm_eps)
203
-
204
- def forward(
205
- self,
206
- positions: torch.Tensor,
207
- hidden_states: torch.Tensor,
208
- kv_cache: KVCache,
209
- input_metadata: InputMetadata,
210
- ) -> torch.Tensor:
211
- # Self Attention
212
- residual = hidden_states
213
- hidden_states = self.input_layernorm(hidden_states)
214
- hidden_states = self.self_attn(
215
- positions=positions,
216
- hidden_states=hidden_states,
217
- kv_cache=kv_cache,
218
- input_metadata=input_metadata,
219
- )
220
- hidden_states = residual + hidden_states
221
-
222
- # Fully Connected
223
- residual = hidden_states
224
- hidden_states = self.post_attention_layernorm(hidden_states)
225
- hidden_states = self.mlp(hidden_states)
226
- hidden_states = residual + hidden_states
227
- return hidden_states
228
-
229
-
230
-class AquilaModel(nn.Module):
231
-
232
- def __init__(
233
- self,
234
- config: AquilaConfig,
235
- linear_method: Optional[LinearMethodBase] = None,
236
- ):
237
- super().__init__()
238
- self.config = config
239
- self.padding_idx = config.pad_token_id
240
- self.vocab_size = config.vocab_size
241
- self.embed_tokens = VocabParallelEmbedding(
242
- config.vocab_size,
243
- config.hidden_size,
244
- )
245
- self.layers = nn.ModuleList([
246
- AquilaDecoderLayer(config, linear_method)
247
- for _ in range(config.num_hidden_layers)
248
- ])
249
- self.norm = AquilaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
250
-
251
- def forward(
252
- self,
253
- input_ids: torch.Tensor,
254
- positions: torch.Tensor,
255
- kv_caches: List[KVCache],
256
- input_metadata: InputMetadata,
257
- ) -> torch.Tensor:
258
- hidden_states = self.embed_tokens(input_ids)
259
- for i in range(len(self.layers)):
260
- layer = self.layers[i]
261
- hidden_states = layer(
262
- positions,
263
- hidden_states,
264
- kv_caches[i],
265
- input_metadata,
266
- )
267
- hidden_states = self.norm(hidden_states)
268
-
269
- return hidden_states
270
-
271
-
272
-class AquilaForCausalLM(nn.Module):
273
-
274
- def __init__(
275
- self,
276
- config,
277
- linear_method: Optional[LinearMethodBase] = None,
278
- ):
279
- super().__init__()
280
- self.config = config
281
- self.linear_method = linear_method
282
- self.model = AquilaModel(config, linear_method)
283
- self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
284
- self.sampler = Sampler(config.vocab_size)
285
-
286
- def forward(
287
- self,
288
- input_ids: torch.Tensor,
289
- positions: torch.Tensor,
290
- kv_caches: List[KVCache],
291
- input_metadata: InputMetadata,
292
- ) -> torch.Tensor:
293
- hidden_states = self.model(input_ids, positions, kv_caches,
294
- input_metadata)
295
- return hidden_states
296
-
297
- def sample(
298
- self,
299
- hidden_states: torch.Tensor,
300
- sampling_metadata: SamplingMetadata,
301
- ) -> Optional[SamplerOutput]:
302
- next_tokens = self.sampler(self.lm_head.weight, hidden_states,
303
- sampling_metadata)
304
- return next_tokens
305
-
306
- def load_weights(self,
307
- model_name_or_path: str,
308
- cache_dir: Optional[str] = None,
309
- load_format: str = "auto",
310
- revision: Optional[str] = None):
311
- stacked_params_mapping = [
312
- # (param_name, shard_name, shard_id)
313
- ("qkv_proj", "q_proj", "q"),
314
- ("qkv_proj", "k_proj", "k"),
315
- ("qkv_proj", "v_proj", "v"),
316
- ("gate_up_proj", "gate_proj", 0),
317
- ("gate_up_proj", "up_proj", 1),
318
- ]
319
- params_dict = dict(self.named_parameters())
320
- for name, loaded_weight in hf_model_weights_iterator(
321
- model_name_or_path, cache_dir, load_format, revision):
322
- if "rotary_emb.inv_freq" in name:
323
- continue
324
- for (param_name, weight_name, shard_id) in stacked_params_mapping:
325
- if weight_name not in name:
326
- continue
327
- name = name.replace(weight_name, param_name)
328
- # Skip loading extra bias for GPTQ models.
329
- if name.endswith(".bias") and name not in params_dict:
330
- continue
331
- param = params_dict[name]
332
- weight_loader = param.weight_loader
333
- weight_loader(param, loaded_weight, shard_id)
334
- break
335
- else:
336
- # Skip loading extra bias for GPTQ models.
337
- if name.endswith(".bias") and name not in params_dict:
338
- continue
339
- param = params_dict[name]
340
- weight_loader = getattr(param, "weight_loader",
341
- default_weight_loader)
342
- weight_loader(param, loaded_weight)
vllm/transformers_utils/config.py
-1
@@ -5,7 +5,6 @@ from transformers import AutoConfig, PretrainedConfig
5
from vllm.transformers_utils.configs import *
6
7
_CONFIG_REGISTRY = {
8
- "aquila": AquilaConfig,
8
"baichuan": BaiChuanConfig,
9
"chatglm": ChatGLMConfig,
10
"mpt": MPTConfig,
vllm/transformers_utils/configs/__init__.py
-2
@@ -1,4 +1,3 @@
1
-from vllm.transformers_utils.configs.aquila import AquilaConfig
1
from vllm.transformers_utils.configs.baichuan import BaiChuanConfig
2
from vllm.transformers_utils.configs.chatglm import ChatGLMConfig
3
from vllm.transformers_utils.configs.mpt import MPTConfig
@@ -9,7 +8,6 @@ from vllm.transformers_utils.configs.qwen import QWenConfig
8
from vllm.transformers_utils.configs.falcon import RWConfig
9
10
__all__ = [
12
- "AquilaConfig",
11
"BaiChuanConfig",
12
"ChatGLMConfig",
13
"MPTConfig",
vllm/transformers_utils/configs/aquila.py
deleted
-69
@@ -1,69 +0,0 @@
1
-# coding=utf-8
2
-# Copyright 2023 EleutherAI and the HuggingFace Inc. team. All rights reserved.
3
-#
4
-# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
5
-# and OPT implementations in this library. It has been modified from its
6
-# original forms to accommodate minor architectural differences compared
7
-# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
8
-#
9
-# Licensed under the Apache License, Version 2.0 (the "License");
10
-# you may not use this file except in compliance with the License.
11
-# You may obtain a copy of the License at
12
-#
13
-# http://www.apache.org/licenses/LICENSE-2.0
14
-#
15
-# Unless required by applicable law or agreed to in writing, software
16
-# distributed under the License is distributed on an "AS IS" BASIS,
17
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18
-# See the License for the specific language governing permissions and
19
-# limitations under the License.
20
-""" Aquila model configuration"""
21
-
22
-from transformers import PretrainedConfig
23
-
24
-
25
-class AquilaConfig(PretrainedConfig):
26
- model_type = "aquila"
27
- keys_to_ignore_at_inference = ["past_key_values"]
28
-
29
- def __init__(
30
- self,
31
- vocab_size=100008,
32
- hidden_size=4096,
33
- intermediate_size=11008,
34
- num_hidden_layers=32,
35
- num_attention_heads=32,
36
- num_key_value_heads=None,
37
- hidden_act="silu",
38
- max_position_embeddings=2048,
39
- initializer_range=0.006,
40
- rms_norm_eps=1e-5,
41
- use_cache=True,
42
- pad_token_id=0,
43
- bos_token_id=1,
44
- eos_token_id=2,
45
- tie_word_embeddings=False,
46
- **kwargs,
47
- ):
48
- self.vocab_size = vocab_size
49
- self.max_position_embeddings = max_position_embeddings
50
- self.hidden_size = hidden_size
51
- self.intermediate_size = intermediate_size
52
- self.num_hidden_layers = num_hidden_layers
53
- # for backward compatibility
54
- if num_key_value_heads is None:
55
- num_key_value_heads = num_attention_heads
56
-
57
- self.num_key_value_heads = num_key_value_heads
58
- self.num_attention_heads = num_attention_heads
59
- self.hidden_act = hidden_act
60
- self.initializer_range = initializer_range
61
- self.rms_norm_eps = rms_norm_eps
62
- self.use_cache = use_cache
63
- super().__init__(
64
- pad_token_id=pad_token_id,
65
- bos_token_id=bos_token_id,
66
- eos_token_id=eos_token_id,
67
- tie_word_embeddings=tie_word_embeddings,
68
- **kwargs,
69
- )