A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Remove Yi model definition, please use `LlamaForCausalLM` instead (#2854)
Co-authored-by: Roy <jasonailu87@gmail.com>
Philipp Moritz committed
Feb 13, 2024 at 14:22 UTC
317b29de0f16428610e2e4d6a6953bee5a2d0ec2
5 files changed
+2
-402
docs/source/models/supported_models.rst
+2
-5
@@ -51,8 +51,8 @@ Alongside each architecture, we include some popular models that use it.
51
- InternLM2
52
- :code:`internlm/internlm2-7b`, :code:`internlm/internlm2-chat-7b`, etc.
53
* - :code:`LlamaForCausalLM`
54
- - LLaMA, LLaMA-2, Vicuna, Alpaca, Koala, Guanaco
55
- - :code:`meta-llama/Llama-2-13b-hf`, :code:`meta-llama/Llama-2-70b-hf`, :code:`openlm-research/open_llama_13b`, :code:`lmsys/vicuna-13b-v1.3`, :code:`young-geng/koala`, etc.
54
+ - LLaMA, LLaMA-2, Vicuna, Alpaca, Yi
55
+ - :code:`meta-llama/Llama-2-13b-hf`, :code:`meta-llama/Llama-2-70b-hf`, :code:`openlm-research/open_llama_13b`, :code:`lmsys/vicuna-13b-v1.3`, :code:`01-ai/Yi-6B`, :code:`01-ai/Yi-34B`, etc.
56
* - :code:`MistralForCausalLM`
57
- Mistral, Mistral-Instruct
58
- :code:`mistralai/Mistral-7B-v0.1`, :code:`mistralai/Mistral-7B-Instruct-v0.1`, etc.
@@ -77,9 +77,6 @@ Alongside each architecture, we include some popular models that use it.
77
* - :code:`StableLMEpochForCausalLM`
78
- StableLM
79
- :code:`stabilityai/stablelm-3b-4e1t/` , :code:`stabilityai/stablelm-base-alpha-7b-v2`, etc.
80
- * - :code:`YiForCausalLM`
81
- - Yi
82
- - :code:`01-ai/Yi-6B`, :code:`01-ai/Yi-34B`, etc.
80
81
If your model uses one of the above model architectures, you can seamlessly run your model with vLLM.
82
Otherwise, please refer to :ref:`Adding a New Model <adding_a_new_model>` for instructions on how to implement support for your model.
vllm/model_executor/models/yi.py
deleted
-330
@@ -1,330 +0,0 @@
1
-# coding=utf-8
2
-# Adapted from
3
-# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
4
-# Copyright 2023 The vLLM team.
5
-# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
6
-#
7
-# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
8
-# and OPT implementations in this library. It has been modified from its
9
-# original forms to accommodate minor architectural differences compared
10
-# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
11
-#
12
-# Licensed under the Apache License, Version 2.0 (the "License");
13
-# you may not use this file except in compliance with the License.
14
-# You may obtain a copy of the License at
15
-#
16
-# http://www.apache.org/licenses/LICENSE-2.0
17
-#
18
-# Unless required by applicable law or agreed to in writing, software
19
-# distributed under the License is distributed on an "AS IS" BASIS,
20
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21
-# See the License for the specific language governing permissions and
22
-# limitations under the License.
23
-"""Inference-only Yi model (https://01.ai) compatible with HuggingFace weights."""
24
-from typing import Any, Dict, List, Optional, Tuple
25
-
26
-import torch
27
-from torch import nn
28
-from vllm.transformers_utils.configs.yi import YiConfig
29
-
30
-from vllm.model_executor.input_metadata import InputMetadata
31
-from vllm.model_executor.layers.activation import SiluAndMul
32
-from vllm.model_executor.layers.attention import PagedAttention
33
-from vllm.model_executor.layers.layernorm import RMSNorm
34
-from vllm.model_executor.layers.linear import (LinearMethodBase,
35
- MergedColumnParallelLinear,
36
- QKVParallelLinear,
37
- RowParallelLinear)
38
-from vllm.model_executor.layers.rotary_embedding import get_rope
39
-from vllm.model_executor.layers.sampler import Sampler
40
-from vllm.model_executor.layers.vocab_parallel_embedding import (
41
- VocabParallelEmbedding, ParallelLMHead)
42
-from vllm.model_executor.parallel_utils.parallel_state import (
43
- get_tensor_model_parallel_world_size)
44
-from vllm.model_executor.sampling_metadata import SamplingMetadata
45
-from vllm.model_executor.weight_utils import (default_weight_loader,
46
- hf_model_weights_iterator)
47
-from vllm.sequence import SamplerOutput
48
-
49
-KVCache = Tuple[torch.Tensor, torch.Tensor]
50
-
51
-
52
-class YiMLP(nn.Module):
53
-
54
- def __init__(
55
- self,
56
- hidden_size: int,
57
- intermediate_size: int,
58
- hidden_act: str,
59
- linear_method: Optional[LinearMethodBase] = None,
60
- ) -> None:
61
- super().__init__()
62
- self.gate_up_proj = MergedColumnParallelLinear(
63
- hidden_size, [intermediate_size] * 2,
64
- bias=False,
65
- linear_method=linear_method)
66
- self.down_proj = RowParallelLinear(intermediate_size,
67
- hidden_size,
68
- bias=False,
69
- linear_method=linear_method)
70
- if hidden_act != "silu":
71
- raise ValueError(f"Unsupported activation: {hidden_act}. "
72
- "Only silu is supported for now.")
73
- self.act_fn = SiluAndMul()
74
-
75
- def forward(self, x):
76
- gate_up, _ = self.gate_up_proj(x)
77
- x = self.act_fn(gate_up)
78
- x, _ = self.down_proj(x)
79
- return x
80
-
81
-
82
-class YiAttention(nn.Module):
83
-
84
- def __init__(
85
- self,
86
- hidden_size: int,
87
- num_heads: int,
88
- num_kv_heads: int,
89
- rope_theta: float = 10000,
90
- rope_scaling: Optional[Dict[str, Any]] = None,
91
- max_position_embeddings: int = 8192,
92
- linear_method: Optional[LinearMethodBase] = None,
93
- ) -> None:
94
- super().__init__()
95
- self.hidden_size = hidden_size
96
- tp_size = get_tensor_model_parallel_world_size()
97
- self.total_num_heads = num_heads
98
- assert self.total_num_heads % tp_size == 0
99
- self.num_heads = self.total_num_heads // tp_size
100
- self.total_num_kv_heads = num_kv_heads
101
- if self.total_num_kv_heads >= tp_size:
102
- # Number of KV heads is greater than TP size, so we partition
103
- # the KV heads across multiple tensor parallel GPUs.
104
- assert self.total_num_kv_heads % tp_size == 0
105
- else:
106
- # Number of KV heads is less than TP size, so we replicate
107
- # the KV heads across multiple tensor parallel GPUs.
108
- assert tp_size % self.total_num_kv_heads == 0
109
- self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
110
- self.head_dim = hidden_size // self.total_num_heads
111
- self.q_size = self.num_heads * self.head_dim
112
- self.kv_size = self.num_kv_heads * self.head_dim
113
- self.scaling = self.head_dim**-0.5
114
- self.rope_theta = rope_theta
115
- self.max_position_embeddings = max_position_embeddings
116
-
117
- self.qkv_proj = QKVParallelLinear(
118
- hidden_size,
119
- self.head_dim,
120
- self.total_num_heads,
121
- self.total_num_kv_heads,
122
- bias=False,
123
- linear_method=linear_method,
124
- )
125
- self.o_proj = RowParallelLinear(
126
- self.total_num_heads * self.head_dim,
127
- hidden_size,
128
- bias=False,
129
- linear_method=linear_method,
130
- )
131
- self.rotary_emb = get_rope(
132
- self.head_dim,
133
- rotary_dim=self.head_dim,
134
- max_position=max_position_embeddings,
135
- base=self.rope_theta,
136
- rope_scaling=rope_scaling,
137
- )
138
- self.attn = PagedAttention(self.num_heads,
139
- self.head_dim,
140
- self.scaling,
141
- num_kv_heads=self.num_kv_heads)
142
-
143
- def forward(
144
- self,
145
- positions: torch.Tensor,
146
- hidden_states: torch.Tensor,
147
- kv_cache: KVCache,
148
- input_metadata: InputMetadata,
149
- ) -> torch.Tensor:
150
- qkv, _ = self.qkv_proj(hidden_states)
151
- q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
152
- q, k = self.rotary_emb(positions, q, k)
153
- k_cache, v_cache = kv_cache
154
- attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
155
- output, _ = self.o_proj(attn_output)
156
- return output
157
-
158
-
159
-class YiDecoderLayer(nn.Module):
160
-
161
- def __init__(
162
- self,
163
- config: YiConfig,
164
- linear_method: Optional[LinearMethodBase] = None,
165
- ) -> None:
166
- super().__init__()
167
- self.hidden_size = config.hidden_size
168
- rope_theta = getattr(config, "rope_theta", 10000)
169
- rope_scaling = getattr(config, "rope_scaling", None)
170
- max_position_embeddings = getattr(config, "max_position_embeddings",
171
- 8192)
172
- self.self_attn = YiAttention(
173
- hidden_size=self.hidden_size,
174
- num_heads=config.num_attention_heads,
175
- num_kv_heads=config.num_key_value_heads,
176
- rope_theta=rope_theta,
177
- rope_scaling=rope_scaling,
178
- max_position_embeddings=max_position_embeddings,
179
- linear_method=linear_method,
180
- )
181
- self.mlp = YiMLP(
182
- hidden_size=self.hidden_size,
183
- intermediate_size=config.intermediate_size,
184
- hidden_act=config.hidden_act,
185
- linear_method=linear_method,
186
- )
187
- self.ln1 = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
188
- self.ln2 = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
189
-
190
- def forward(
191
- self,
192
- positions: torch.Tensor,
193
- hidden_states: torch.Tensor,
194
- kv_cache: KVCache,
195
- input_metadata: InputMetadata,
196
- residual: Optional[torch.Tensor],
197
- ) -> Tuple[torch.Tensor, torch.Tensor]:
198
- # Self Attention
199
- if residual is None:
200
- residual = hidden_states
201
- hidden_states = self.ln1(hidden_states)
202
- else:
203
- hidden_states, residual = self.ln1(hidden_states, residual)
204
- hidden_states = self.self_attn(
205
- positions=positions,
206
- hidden_states=hidden_states,
207
- kv_cache=kv_cache,
208
- input_metadata=input_metadata,
209
- )
210
-
211
- # Fully Connected
212
- hidden_states, residual = self.ln2(hidden_states, residual)
213
- hidden_states = self.mlp(hidden_states)
214
- return hidden_states, residual
215
-
216
-
217
-class YiModel(nn.Module):
218
-
219
- def __init__(
220
- self,
221
- config: YiConfig,
222
- linear_method: Optional[LinearMethodBase] = None,
223
- ) -> None:
224
- super().__init__()
225
- self.config = config
226
- self.padding_idx = config.pad_token_id
227
- self.vocab_size = config.vocab_size
228
- self.embed_tokens = VocabParallelEmbedding(
229
- config.vocab_size,
230
- config.hidden_size,
231
- )
232
- self.layers = nn.ModuleList([
233
- YiDecoderLayer(config, linear_method)
234
- for _ in range(config.num_hidden_layers)
235
- ])
236
- self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
237
-
238
- def forward(
239
- self,
240
- input_ids: torch.Tensor,
241
- positions: torch.Tensor,
242
- kv_caches: List[KVCache],
243
- input_metadata: InputMetadata,
244
- ) -> torch.Tensor:
245
- hidden_states = self.embed_tokens(input_ids)
246
- residual = None
247
- for i in range(len(self.layers)):
248
- layer = self.layers[i]
249
- hidden_states, residual = layer(
250
- positions,
251
- hidden_states,
252
- kv_caches[i],
253
- input_metadata,
254
- residual,
255
- )
256
- hidden_states, _ = self.norm(hidden_states, residual)
257
- return hidden_states
258
-
259
-
260
-class YiForCausalLM(nn.Module):
261
-
262
- def __init__(
263
- self,
264
- config: YiConfig,
265
- linear_method: Optional[LinearMethodBase] = None,
266
- ) -> None:
267
- super().__init__()
268
- self.config = config
269
- self.linear_method = linear_method
270
- self.model = YiModel(config, linear_method)
271
- self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
272
- self.sampler = Sampler(config.vocab_size)
273
-
274
- def forward(
275
- self,
276
- input_ids: torch.Tensor,
277
- positions: torch.Tensor,
278
- kv_caches: List[KVCache],
279
- input_metadata: InputMetadata,
280
- ) -> torch.Tensor:
281
- hidden_states = self.model(input_ids, positions, kv_caches,
282
- input_metadata)
283
- return hidden_states
284
-
285
- def sample(
286
- self,
287
- hidden_states: torch.Tensor,
288
- sampling_metadata: SamplingMetadata,
289
- ) -> Optional[SamplerOutput]:
290
- next_tokens = self.sampler(self.lm_head.weight, hidden_states,
291
- sampling_metadata)
292
- return next_tokens
293
-
294
- def load_weights(self,
295
- model_name_or_path: str,
296
- cache_dir: Optional[str] = None,
297
- load_format: str = "auto",
298
- revision: Optional[str] = None):
299
- stacked_params_mapping = [
300
- # (param_name, shard_name, shard_id)
301
- ("qkv_proj", "q_proj", "q"),
302
- ("qkv_proj", "k_proj", "k"),
303
- ("qkv_proj", "v_proj", "v"),
304
- ("gate_up_proj", "gate_proj", 0),
305
- ("gate_up_proj", "up_proj", 1),
306
- ]
307
- params_dict = dict(self.named_parameters())
308
- for name, loaded_weight in hf_model_weights_iterator(
309
- model_name_or_path, cache_dir, load_format, revision):
310
- if "rotary_emb.inv_freq" in name:
311
- continue
312
- for (param_name, weight_name, shard_id) in stacked_params_mapping:
313
- if weight_name not in name:
314
- continue
315
- name = name.replace(weight_name, param_name)
316
- # Skip loading extra bias for GPTQ models.
317
- if name.endswith(".bias") and name not in params_dict:
318
- continue
319
- param = params_dict[name]
320
- weight_loader = param.weight_loader
321
- weight_loader(param, loaded_weight, shard_id)
322
- break
323
- else:
324
- # Skip loading extra bias for GPTQ models.
325
- if name.endswith(".bias") and name not in params_dict:
326
- continue
327
- param = params_dict[name]
328
- weight_loader = getattr(param, "weight_loader",
329
- default_weight_loader)
330
- weight_loader(param, loaded_weight)
vllm/transformers_utils/config.py
-1
@@ -12,7 +12,6 @@ _CONFIG_REGISTRY = {
12
"qwen": QWenConfig,
13
"RefinedWeb": RWConfig, # For tiiuae/falcon-40b(-instruct)
14
"RefinedWebModel": RWConfig, # For tiiuae/falcon-7b(-instruct)
15
- "yi": YiConfig,
15
}
16
17
vllm/transformers_utils/configs/__init__.py
-2
@@ -7,7 +7,6 @@ from vllm.transformers_utils.configs.qwen import QWenConfig
7
# tiiuae/falcon-7b(-instruct) models. Newer Falcon models will use the
8
# `FalconConfig` class from the official HuggingFace transformers library.
9
from vllm.transformers_utils.configs.falcon import RWConfig
10
-from vllm.transformers_utils.configs.yi import YiConfig
10
11
__all__ = [
12
"AquilaConfig",
@@ -16,5 +15,4 @@ __all__ = [
15
"MPTConfig",
16
"QWenConfig",
17
"RWConfig",
19
- "YiConfig",
18
]
vllm/transformers_utils/configs/yi.py
deleted
-64
@@ -1,64 +0,0 @@
1
-""" Yi model configuration"""
2
-from transformers.configuration_utils import PretrainedConfig
3
-from transformers.utils import logging
4
-
5
-logger = logging.get_logger(__name__)
6
-
7
-Yi_PRETRAINED_CONFIG_ARCHIVE_MAP = {}
8
-
9
-
10
-class YiConfig(PretrainedConfig):
11
- r"""
12
- Reference:
13
- https://huggingface.co/01-ai/Yi-6B/blob/main/configuration_yi.py
14
- """
15
- model_type = "Yi"
16
- keys_to_ignore_at_inference = ["past_key_values"]
17
-
18
- def __init__(
19
- self,
20
- vocab_size=64000,
21
- hidden_size=4096,
22
- intermediate_size=11008,
23
- num_hidden_layers=32,
24
- num_attention_heads=32,
25
- num_key_value_heads=4,
26
- hidden_act="silu",
27
- max_position_embeddings=4096,
28
- initializer_range=0.02,
29
- rms_norm_eps=1e-5,
30
- use_cache=True,
31
- pad_token_id=0,
32
- bos_token_id=1,
33
- eos_token_id=2,
34
- tie_word_embeddings=False,
35
- output_attentions=False,
36
- rope_theta=5000000.0,
37
- **kwargs,
38
- ):
39
- self.vocab_size = vocab_size
40
- self.max_position_embeddings = max_position_embeddings
41
- self.hidden_size = hidden_size
42
- self.intermediate_size = intermediate_size
43
- self.num_hidden_layers = num_hidden_layers
44
- self.num_attention_heads = num_attention_heads
45
-
46
- # for backward compatibility
47
- if num_key_value_heads is None:
48
- num_key_value_heads = num_attention_heads
49
-
50
- self.num_key_value_heads = num_key_value_heads
51
- self.hidden_act = hidden_act
52
- self.initializer_range = initializer_range
53
- self.rms_norm_eps = rms_norm_eps
54
- self.use_cache = use_cache
55
- self.output_attentions = output_attentions
56
- self.rope_theta = rope_theta
57
-
58
- super().__init__(
59
- pad_token_id=pad_token_id,
60
- bos_token_id=bos_token_id,
61
- eos_token_id=eos_token_id,
62
- tie_word_embeddings=tie_word_embeddings,
63
- **kwargs,
64
- )