A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Support Yi model (#1567)
Roy committed
Nov 7, 2023 at 07:26 UTC
e7f579eb97f8129acec28804fb74d61f4f9b1ddf
6 files changed
+496
vllm/model_executor/model_loader.py
+1
@@ -33,6 +33,7 @@ _MODEL_REGISTRY = {
33
"OPTForCausalLM": OPTForCausalLM,
34
"QWenLMHeadModel": QWenLMHeadModel,
35
"RWForCausalLM": FalconForCausalLM,
36
+ "YiForCausalLM": YiForCausalLM,
37
}
38
39
# FIXME(woosuk): Remove this once all models support quantization.
vllm/model_executor/models/__init__.py
+2
@@ -13,6 +13,7 @@ from vllm.model_executor.models.mistral import MistralForCausalLM
13
from vllm.model_executor.models.mpt import MptForCausalLM
14
from vllm.model_executor.models.opt import OPTForCausalLM
15
from vllm.model_executor.models.qwen import QWenLMHeadModel
16
+from vllm.model_executor.models.yi import YiForCausalLM
17
18
__all__ = [
19
"AquilaForCausalLM",
@@ -30,4 +31,5 @@ __all__ = [
31
"OPTForCausalLM",
32
"QWenLMHeadModel",
33
"MistralForCausalLM",
34
+ "YiForCausalLM",
35
]
vllm/model_executor/models/yi.py
new
+426
@@ -0,0 +1,426 @@
1
+# coding=utf-8
2
+# Adapted from
3
+# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
4
+# Copyright 2023 The vLLM team.
5
+# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
6
+#
7
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
8
+# and OPT implementations in this library. It has been modified from its
9
+# original forms to accommodate minor architectural differences compared
10
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
11
+#
12
+# Licensed under the Apache License, Version 2.0 (the "License");
13
+# you may not use this file except in compliance with the License.
14
+# You may obtain a copy of the License at
15
+#
16
+# http://www.apache.org/licenses/LICENSE-2.0
17
+#
18
+# Unless required by applicable law or agreed to in writing, software
19
+# distributed under the License is distributed on an "AS IS" BASIS,
20
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21
+# See the License for the specific language governing permissions and
22
+# limitations under the License.
23
+"""Inference-only Yi model (https://01.ai) compatible with HuggingFace weights.
24
+
25
+The input of the model is flattened to a 1D tensor of tokens. The model uses
26
+InputMetadata to extract the original 2D shape of the input.
27
+"""
28
+from typing import Any, Dict, List, Optional, Tuple
29
+
30
+import torch
31
+from torch import nn
32
+from vllm.transformers_utils.configs.yi import YiConfig
33
+
34
+from vllm.model_executor.input_metadata import InputMetadata
35
+from vllm.model_executor.layers.activation import SiluAndMul
36
+from vllm.model_executor.layers.layernorm import RMSNorm
37
+from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
38
+from vllm.model_executor.layers.sampler import Sampler
39
+from vllm.model_executor.layers.quantized_linear import ParallelLinear
40
+from vllm.model_executor.parallel_utils.parallel_state import (
41
+ get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
42
+from vllm.model_executor.parallel_utils.layers import VocabParallelEmbedding
43
+from vllm.model_executor.quantization_utils import QuantizationConfig
44
+from vllm.model_executor.weight_utils import (
45
+ convert_pyslice_to_tensor, hf_model_weights_iterator,
46
+ load_tensor_parallel_weights, load_padded_tensor_parallel_vocab)
47
+from vllm.sequence import SamplerOutput
48
+
49
+KVCache = Tuple[torch.Tensor, torch.Tensor]
50
+
51
+
52
+class YiMLP(nn.Module):
53
+
54
+ def __init__(
55
+ self,
56
+ hidden_size: int,
57
+ intermediate_size: int,
58
+ hidden_act: str,
59
+ quant_config: Optional[QuantizationConfig] = None,
60
+ ) -> None:
61
+ super().__init__()
62
+ self.gate_up_proj = ParallelLinear.column(hidden_size,
63
+ 2 * intermediate_size,
64
+ bias=False,
65
+ gather_output=False,
66
+ quant_config=quant_config)
67
+ self.down_proj = ParallelLinear.row(intermediate_size,
68
+ hidden_size,
69
+ bias=False,
70
+ input_is_parallel=True,
71
+ quant_config=quant_config)
72
+ if hidden_act != "silu":
73
+ raise ValueError(f"Unsupported activation: {hidden_act}. "
74
+ "Only silu is supported for now.")
75
+ self.act_fn = SiluAndMul()
76
+
77
+ def forward(self, x):
78
+ gate_up, _ = self.gate_up_proj(x)
79
+ x = self.act_fn(gate_up)
80
+ x, _ = self.down_proj(x)
81
+ return x
82
+
83
+
84
+class YiAttention(nn.Module):
85
+
86
+ def __init__(
87
+ self,
88
+ hidden_size: int,
89
+ num_heads: int,
90
+ num_kv_heads: int,
91
+ rope_theta: float = 10000,
92
+ rope_scaling: Optional[Dict[str, Any]] = None,
93
+ max_position_embeddings: int = 8192,
94
+ quant_config: Optional[QuantizationConfig] = None,
95
+ ) -> None:
96
+ super().__init__()
97
+ self.hidden_size = hidden_size
98
+ tp_size = get_tensor_model_parallel_world_size()
99
+ self.total_num_heads = num_heads
100
+ assert self.total_num_heads % tp_size == 0
101
+ self.num_heads = self.total_num_heads // tp_size
102
+ self.total_num_kv_heads = num_kv_heads
103
+ if self.total_num_kv_heads >= tp_size:
104
+ # Number of KV heads is greater than TP size, so we partition
105
+ # the KV heads across multiple tensor parallel GPUs.
106
+ assert self.total_num_kv_heads % tp_size == 0
107
+ else:
108
+ # Number of KV heads is less than TP size, so we replicate
109
+ # the KV heads across multiple tensor parallel GPUs.
110
+ assert tp_size % self.total_num_kv_heads == 0
111
+ self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
112
+ num_kv_heads_replicas = max(1, tp_size // self.total_num_kv_heads)
113
+ self.head_dim = hidden_size // self.total_num_heads
114
+ self.q_size = self.num_heads * self.head_dim
115
+ self.kv_size = self.num_kv_heads * self.head_dim
116
+ self.scaling = self.head_dim**-0.5
117
+ self.rope_theta = rope_theta
118
+ self.max_position_embeddings = max_position_embeddings
119
+
120
+ self.qkv_proj = ParallelLinear.column(
121
+ hidden_size,
122
+ (self.total_num_heads +
123
+ 2 * self.total_num_kv_heads * num_kv_heads_replicas) *
124
+ self.head_dim,
125
+ bias=False,
126
+ gather_output=False,
127
+ quant_config=quant_config,
128
+ )
129
+ self.o_proj = ParallelLinear.row(
130
+ self.total_num_heads * self.head_dim,
131
+ hidden_size,
132
+ bias=False,
133
+ input_is_parallel=True,
134
+ quant_config=quant_config,
135
+ )
136
+ self.attn = PagedAttentionWithRoPE(
137
+ self.num_heads,
138
+ self.head_dim,
139
+ self.scaling,
140
+ base=self.rope_theta,
141
+ max_position=self.max_position_embeddings,
142
+ rotary_dim=self.head_dim,
143
+ num_kv_heads=self.num_kv_heads,
144
+ rope_scaling=rope_scaling)
145
+
146
+ def forward(
147
+ self,
148
+ positions: torch.Tensor,
149
+ hidden_states: torch.Tensor,
150
+ kv_cache: KVCache,
151
+ input_metadata: InputMetadata,
152
+ cache_event: Optional[torch.cuda.Event],
153
+ ) -> torch.Tensor:
154
+ qkv, _ = self.qkv_proj(hidden_states)
155
+ q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
156
+ k_cache, v_cache = kv_cache
157
+ attn_output = self.attn(positions, q, k, v, k_cache, v_cache,
158
+ input_metadata, cache_event)
159
+ output, _ = self.o_proj(attn_output)
160
+ return output
161
+
162
+
163
+class YiDecoderLayer(nn.Module):
164
+
165
+ def __init__(
166
+ self,
167
+ config: YiConfig,
168
+ quant_config: Optional[QuantizationConfig] = None,
169
+ ) -> None:
170
+ super().__init__()
171
+ self.hidden_size = config.hidden_size
172
+ # Requires transformers > 4.32.0
173
+ rope_theta = getattr(config, "rope_theta", 10000)
174
+ rope_scaling = getattr(config, "rope_scaling", None)
175
+ max_position_embeddings = getattr(config, "max_position_embeddings",
176
+ 8192)
177
+ self.self_attn = YiAttention(
178
+ hidden_size=self.hidden_size,
179
+ num_heads=config.num_attention_heads,
180
+ num_kv_heads=config.num_key_value_heads,
181
+ rope_theta=rope_theta,
182
+ rope_scaling=rope_scaling,
183
+ max_position_embeddings=max_position_embeddings,
184
+ quant_config=quant_config,
185
+ )
186
+ self.mlp = YiMLP(
187
+ hidden_size=self.hidden_size,
188
+ intermediate_size=config.intermediate_size,
189
+ hidden_act=config.hidden_act,
190
+ quant_config=quant_config,
191
+ )
192
+ self.ln1 = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
193
+ self.ln2 = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
194
+
195
+ def forward(
196
+ self,
197
+ positions: torch.Tensor,
198
+ hidden_states: torch.Tensor,
199
+ kv_cache: KVCache,
200
+ input_metadata: InputMetadata,
201
+ cache_event: Optional[torch.cuda.Event],
202
+ ) -> torch.Tensor:
203
+ # Self Attention
204
+ residual = hidden_states
205
+ hidden_states = self.ln1(hidden_states)
206
+ hidden_states = self.self_attn(
207
+ positions=positions,
208
+ hidden_states=hidden_states,
209
+ kv_cache=kv_cache,
210
+ input_metadata=input_metadata,
211
+ cache_event=cache_event,
212
+ )
213
+ hidden_states = residual + hidden_states
214
+
215
+ # Fully Connected
216
+ residual = hidden_states
217
+ hidden_states = self.ln2(hidden_states)
218
+ hidden_states = self.mlp(hidden_states)
219
+ hidden_states = residual + hidden_states
220
+ return hidden_states
221
+
222
+
223
+class YiModel(nn.Module):
224
+
225
+ def __init__(
226
+ self,
227
+ config: YiConfig,
228
+ quant_config: Optional[QuantizationConfig] = None,
229
+ ) -> None:
230
+ super().__init__()
231
+ self.config = config
232
+ self.padding_idx = config.pad_token_id
233
+ self.vocab_size = config.vocab_size
234
+
235
+ vocab_size = ((config.vocab_size + 63) // 64) * 64
236
+ self.embed_tokens = VocabParallelEmbedding(
237
+ vocab_size,
238
+ config.hidden_size,
239
+ )
240
+ self.layers = nn.ModuleList([
241
+ YiDecoderLayer(config, quant_config)
242
+ for _ in range(config.num_hidden_layers)
243
+ ])
244
+ self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
245
+
246
+ def forward(
247
+ self,
248
+ input_ids: torch.Tensor,
249
+ positions: torch.Tensor,
250
+ kv_caches: List[KVCache],
251
+ input_metadata: InputMetadata,
252
+ cache_events: Optional[List[torch.cuda.Event]],
253
+ ) -> torch.Tensor:
254
+ hidden_states = self.embed_tokens(input_ids)
255
+ for i in range(len(self.layers)):
256
+ if cache_events is None:
257
+ cache_event = None
258
+ else:
259
+ cache_event = cache_events[i]
260
+ layer = self.layers[i]
261
+ hidden_states = layer(
262
+ positions,
263
+ hidden_states,
264
+ kv_caches[i],
265
+ input_metadata,
266
+ cache_event,
267
+ )
268
+ hidden_states = self.norm(hidden_states)
269
+ return hidden_states
270
+
271
+
272
+class YiForCausalLM(nn.Module):
273
+
274
+ def __init__(
275
+ self,
276
+ config: YiConfig,
277
+ quant_config: Optional[QuantizationConfig] = None,
278
+ ) -> None:
279
+ super().__init__()
280
+ self.config = config
281
+ self.quant_config = quant_config
282
+ self.model = YiModel(config, quant_config)
283
+ vocab_size = ((config.vocab_size + 63) // 64) * 64
284
+ # NOTE: The LM head is not quantized.
285
+ self.lm_head = ParallelLinear.column(config.hidden_size,
286
+ vocab_size,
287
+ bias=False,
288
+ gather_output=False,
289
+ quant_config=None)
290
+ self.sampler = Sampler(config.vocab_size)
291
+
292
+ def forward(
293
+ self,
294
+ input_ids: torch.Tensor,
295
+ positions: torch.Tensor,
296
+ kv_caches: List[KVCache],
297
+ input_metadata: InputMetadata,
298
+ cache_events: Optional[List[torch.cuda.Event]],
299
+ ) -> SamplerOutput:
300
+ hidden_states = self.model(input_ids, positions, kv_caches,
301
+ input_metadata, cache_events)
302
+ next_tokens = self.sampler(self.lm_head.weight, hidden_states,
303
+ input_metadata)
304
+ return next_tokens
305
+
306
+ _column_parallel_layers = []
307
+ _row_parallel_layers = ["o_proj", "down_proj"]
308
+
309
+ def load_weights(self,
310
+ model_name_or_path: str,
311
+ cache_dir: Optional[str] = None,
312
+ load_format: str = "auto",
313
+ revision: Optional[str] = None):
314
+ if self.quant_config is None:
315
+ col_weight_suffixes = ["weight"]
316
+ row_weight_suffixes = ["weight"]
317
+ else:
318
+ col_weight_suffixes = (
319
+ self.quant_config.get_col_parallel_tensor_names())
320
+ row_weight_suffixes = (
321
+ self.quant_config.get_row_parallel_tensor_names())
322
+
323
+ column_parallel_weights: List[str] = []
324
+ for layer in self._column_parallel_layers:
325
+ for suffix in col_weight_suffixes:
326
+ column_parallel_weights.append(f"{layer}.{suffix}")
327
+ row_parallel_weights: List[str] = []
328
+ for layer in self._row_parallel_layers:
329
+ for suffix in row_weight_suffixes:
330
+ row_parallel_weights.append(f"{layer}.{suffix}")
331
+
332
+ tp_size = get_tensor_model_parallel_world_size()
333
+ tp_rank = get_tensor_model_parallel_rank()
334
+ q_proj_shard_size = (self.config.hidden_size // tp_size)
335
+ num_kv_heads_replicas = max(1,
336
+ tp_size // self.config.num_key_value_heads)
337
+ num_kv_heads_per_gpu = max(1,
338
+ self.config.num_key_value_heads // tp_size)
339
+ kv_proj_shard_size = (self.config.hidden_size //
340
+ self.config.num_attention_heads *
341
+ num_kv_heads_per_gpu)
342
+ attention_weight_specs = [
343
+ # (weight_name, shard_size, offset)
344
+ ("q_proj", q_proj_shard_size, 0),
345
+ ("k_proj", kv_proj_shard_size, q_proj_shard_size),
346
+ ("v_proj", kv_proj_shard_size,
347
+ q_proj_shard_size + kv_proj_shard_size),
348
+ ]
349
+ state_dict = self.state_dict()
350
+
351
+ for name, loaded_weight in hf_model_weights_iterator(
352
+ model_name_or_path, cache_dir, load_format, revision):
353
+ if "rotary_emb.inv_freq" in name:
354
+ continue
355
+
356
+ packed_dim = None
357
+ is_transposed = False
358
+ if self.quant_config is not None:
359
+ packed_dim = self.quant_config.get_packed_dim(name)
360
+ is_transposed = self.quant_config.is_transposed(name)
361
+ if is_transposed:
362
+ loaded_weight = convert_pyslice_to_tensor(loaded_weight)
363
+ loaded_weight = loaded_weight.T
364
+
365
+ is_attention_weight = False
366
+ for weight_name, shard_size, offset in attention_weight_specs:
367
+ if weight_name not in name:
368
+ continue
369
+ param = state_dict[name.replace(weight_name, "qkv_proj")]
370
+ if is_transposed:
371
+ param = param.T
372
+
373
+ if packed_dim is not None:
374
+ shard_dim = 0 if not is_transposed else 1
375
+ if packed_dim == shard_dim:
376
+ shard_size //= self.quant_config.pack_factor
377
+ offset //= self.quant_config.pack_factor
378
+
379
+ if weight_name in ["k_proj", "v_proj"]:
380
+ shard_id = tp_rank // num_kv_heads_replicas
381
+ else:
382
+ shard_id = tp_rank
383
+ loaded_weight = loaded_weight[shard_size *
384
+ shard_id:shard_size *
385
+ (shard_id + 1)]
386
+ param_slice = param.data[offset:offset + shard_size]
387
+ assert param_slice.shape == loaded_weight.shape
388
+
389
+ param_slice.copy_(loaded_weight)
390
+ is_attention_weight = True
391
+ break
392
+ if is_attention_weight:
393
+ continue
394
+
395
+ is_gate_up_weight = False
396
+ for stride_id, weight_name in enumerate(["gate_proj", "up_proj"]):
397
+ if weight_name not in name:
398
+ continue
399
+ param = state_dict[name.replace(weight_name, "gate_up_proj")]
400
+ if is_transposed:
401
+ param = param.T
402
+
403
+ shard_size = param.shape[0] // 2
404
+ loaded_weight = loaded_weight[shard_size * tp_rank:shard_size *
405
+ (tp_rank + 1)]
406
+ param_slice = param.data[shard_size * stride_id:shard_size *
407
+ (stride_id + 1)]
408
+ assert param_slice.shape == loaded_weight.shape
409
+ param_slice.copy_(loaded_weight)
410
+ is_gate_up_weight = True
411
+ break
412
+ if is_gate_up_weight:
413
+ continue
414
+
415
+ param = state_dict[name]
416
+ if is_transposed:
417
+ param = param.T
418
+
419
+ if "embed_tokens" in name or "lm_head" in name:
420
+ load_padded_tensor_parallel_vocab(param, loaded_weight,
421
+ tp_rank)
422
+ continue
423
+
424
+ load_tensor_parallel_weights(param, loaded_weight, name,
425
+ column_parallel_weights,
426
+ row_parallel_weights, tp_rank)
vllm/transformers_utils/config.py
+1
@@ -11,6 +11,7 @@ _CONFIG_REGISTRY = {
11
"qwen": QWenConfig,
12
"RefinedWeb": RWConfig, # For tiiuae/falcon-40b(-instruct)
13
"RefinedWebModel": RWConfig, # For tiiuae/falcon-7b(-instruct)
14
+ "yi": YiConfig,
15
}
16
17
vllm/transformers_utils/configs/__init__.py
+2
@@ -5,10 +5,12 @@ from vllm.transformers_utils.configs.qwen import QWenConfig
5
# tiiuae/falcon-7b(-instruct) models. Newer Falcon models will use the
6
# `FalconConfig` class from the official HuggingFace transformers library.
7
from vllm.transformers_utils.configs.falcon import RWConfig
8
+from vllm.transformers_utils.configs.yi import YiConfig
9
10
__all__ = [
11
"BaiChuanConfig",
12
"AquilaConfig",
13
"QWenConfig",
14
"RWConfig",
15
+ "YiConfig",
16
]
vllm/transformers_utils/configs/yi.py
new
+64
@@ -0,0 +1,64 @@
1
+""" Yi model configuration"""
2
+from transformers.configuration_utils import PretrainedConfig
3
+from transformers.utils import logging
4
+
5
+logger = logging.get_logger(__name__)
6
+
7
+Yi_PRETRAINED_CONFIG_ARCHIVE_MAP = {}
8
+
9
+
10
+class YiConfig(PretrainedConfig):
11
+ r"""
12
+ Reference:
13
+ https://huggingface.co/01-ai/Yi-6B/blob/main/configuration_yi.py
14
+ """
15
+ model_type = "Yi"
16
+ keys_to_ignore_at_inference = ["past_key_values"]
17
+
18
+ def __init__(
19
+ self,
20
+ vocab_size=64000,
21
+ hidden_size=4096,
22
+ intermediate_size=11008,
23
+ num_hidden_layers=32,
24
+ num_attention_heads=32,
25
+ num_key_value_heads=4,
26
+ hidden_act="silu",
27
+ max_position_embeddings=4096,
28
+ initializer_range=0.02,
29
+ rms_norm_eps=1e-5,
30
+ use_cache=True,
31
+ pad_token_id=0,
32
+ bos_token_id=1,
33
+ eos_token_id=2,
34
+ tie_word_embeddings=False,
35
+ output_attentions=False,
36
+ rope_theta=5000000.0,
37
+ **kwargs,
38
+ ):
39
+ self.vocab_size = vocab_size
40
+ self.max_position_embeddings = max_position_embeddings
41
+ self.hidden_size = hidden_size
42
+ self.intermediate_size = intermediate_size
43
+ self.num_hidden_layers = num_hidden_layers
44
+ self.num_attention_heads = num_attention_heads
45
+
46
+ # for backward compatibility
47
+ if num_key_value_heads is None:
48
+ num_key_value_heads = num_attention_heads
49
+
50
+ self.num_key_value_heads = num_key_value_heads
51
+ self.hidden_act = hidden_act
52
+ self.initializer_range = initializer_range
53
+ self.rms_norm_eps = rms_norm_eps
54
+ self.use_cache = use_cache
55
+ self.output_attentions = output_attentions
56
+ self.rope_theta = rope_theta
57
+
58
+ super().__init__(
59
+ pad_token_id=pad_token_id,
60
+ bos_token_id=bos_token_id,
61
+ eos_token_id=eos_token_id,
62
+ tie_word_embeddings=tie_word_embeddings,
63
+ **kwargs,
64
+ )