A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Remove `MPTConfig` (#1529)
Woosuk Kwon committed
Nov 1, 2023 at 15:29 UTC
1fe09900238870a0477fa638dbd2d2bd2f251e1d
6 files changed
+26
-102
vllm/model_executor/model_loader.py
+2
-2
@@ -28,8 +28,8 @@ _MODEL_REGISTRY = {
28
"LLaMAForCausalLM": LlamaForCausalLM, # For decapoda-research/llama-*
29
"MistralForCausalLM": MistralForCausalLM,
30
# transformers's mpt class has lower case
31
- "MptForCausalLM": MPTForCausalLM,
32
- "MPTForCausalLM": MPTForCausalLM,
31
+ "MptForCausalLM": MptForCausalLM,
32
+ "MPTForCausalLM": MptForCausalLM,
33
"OPTForCausalLM": OPTForCausalLM,
34
"QWenLMHeadModel": QWenLMHeadModel,
35
"RWForCausalLM": FalconForCausalLM,
vllm/model_executor/models/__init__.py
+2
-2
@@ -10,7 +10,7 @@ from vllm.model_executor.models.gpt_neox import GPTNeoXForCausalLM
10
from vllm.model_executor.models.internlm import InternLMForCausalLM
11
from vllm.model_executor.models.llama import LlamaForCausalLM
12
from vllm.model_executor.models.mistral import MistralForCausalLM
13
-from vllm.model_executor.models.mpt import MPTForCausalLM
13
+from vllm.model_executor.models.mpt import MptForCausalLM
14
from vllm.model_executor.models.opt import OPTForCausalLM
15
from vllm.model_executor.models.qwen import QWenLMHeadModel
16
@@ -26,7 +26,7 @@ __all__ = [
26
"GPTNeoXForCausalLM",
27
"InternLMForCausalLM",
28
"LlamaForCausalLM",
29
- "MPTForCausalLM",
29
+ "MptForCausalLM",
30
"OPTForCausalLM",
31
"QWenLMHeadModel",
32
"MistralForCausalLM",
vllm/model_executor/models/mpt.py
+20
-20
@@ -5,6 +5,7 @@ from typing import List, Optional, Tuple
5
6
import torch
7
import torch.nn as nn
8
+from transformers import MptConfig
9
10
from vllm.model_executor.input_metadata import InputMetadata
11
from vllm.model_executor.layers.activation import get_act_fn
@@ -19,7 +20,6 @@ from vllm.model_executor.parallel_utils.layers import (VocabParallelEmbedding,
20
ColumnParallelLinear,
21
RowParallelLinear)
22
from vllm.sequence import SamplerOutput
22
-from vllm.transformers_utils.configs.mpt import MPTConfig
23
24
KVCache = Tuple[torch.Tensor, torch.Tensor]
25
@@ -37,17 +37,17 @@ def _get_alibi_slopes(
37
return slopes
38
39
40
-class MPTAttention(nn.Module):
40
+class MptAttention(nn.Module):
41
42
- def __init__(self, config: MPTConfig):
42
+ def __init__(self, config: MptConfig):
43
super().__init__()
44
self.d_model = config.d_model
45
self.total_num_heads = config.n_heads
46
- self.clip_qkv = config.attn_config["clip_qkv"]
47
- self.qk_ln = config.attn_config["qk_ln"]
48
- self.alibi_bias_max = config.attn_config["alibi_bias_max"]
49
- assert not config.attn_config["prefix_lm"]
50
- assert config.attn_config["alibi"]
46
+ self.clip_qkv = config.attn_config.clip_qkv
47
+ self.qk_ln = config.attn_config.qk_ln
48
+ self.alibi_bias_max = config.attn_config.alibi_bias_max
49
+ assert not config.attn_config.prefix_lm
50
+ assert config.attn_config.alibi
51
52
self.qkv_proj = ColumnParallelLinear(
53
self.d_model,
@@ -105,9 +105,9 @@ class MPTAttention(nn.Module):
105
return output
106
107
108
-class MPTMLP(nn.Module):
108
+class MptMLP(nn.Module):
109
110
- def __init__(self, config: MPTConfig):
110
+ def __init__(self, config: MptConfig):
111
super().__init__()
112
hidden_size = config.d_model
113
expansion_ratio = config.expansion_ratio
@@ -133,15 +133,15 @@ class MPTMLP(nn.Module):
133
return x
134
135
136
-class MPTBlock(nn.Module):
136
+class MptBlock(nn.Module):
137
138
- def __init__(self, config: MPTConfig):
138
+ def __init__(self, config: MptConfig):
139
super().__init__()
140
hidden_size = config.d_model
141
self.norm_1 = nn.LayerNorm(hidden_size)
142
- self.attn = MPTAttention(config)
142
+ self.attn = MptAttention(config)
143
self.norm_2 = nn.LayerNorm(hidden_size)
144
- self.ffn = MPTMLP(config)
144
+ self.ffn = MptMLP(config)
145
146
def forward(
147
self,
@@ -166,9 +166,9 @@ class MPTBlock(nn.Module):
166
return hidden_states
167
168
169
-class MPTModel(nn.Module):
169
+class MptModel(nn.Module):
170
171
- def __init__(self, config: MPTConfig):
171
+ def __init__(self, config: MptConfig):
172
super().__init__()
173
assert config.embedding_fraction == 1.0
174
assert config.norm_type == "low_precision_layernorm"
@@ -178,7 +178,7 @@ class MPTModel(nn.Module):
178
config.d_model,
179
)
180
self.blocks = nn.ModuleList(
181
- [MPTBlock(config) for _ in range(config.n_layers)])
181
+ [MptBlock(config) for _ in range(config.n_layers)])
182
self.norm_f = nn.LayerNorm(config.d_model)
183
if config.no_bias:
184
for module in self.modules():
@@ -213,14 +213,14 @@ class MPTModel(nn.Module):
213
return hidden_states
214
215
216
-class MPTForCausalLM(nn.Module):
216
+class MptForCausalLM(nn.Module):
217
218
- def __init__(self, config: MPTConfig):
218
+ def __init__(self, config: MptConfig):
219
super().__init__()
220
self.config = config
221
assert config.tie_word_embeddings
222
223
- self.transformer = MPTModel(config)
223
+ self.transformer = MptModel(config)
224
# TODO(zhuohan): create a new weight after implementing pipeline
225
# parallelism
226
self.lm_head_weight = self.transformer.wte.weight
vllm/transformers_utils/config.py
+2
-2
@@ -1,11 +1,11 @@
1
from typing import Optional
2
3
-from transformers import AutoConfig, PretrainedConfig
3
+from transformers import AutoConfig, MptConfig, PretrainedConfig
4
5
from vllm.transformers_utils.configs import * # pylint: disable=wildcard-import
6
7
_CONFIG_REGISTRY = {
8
- "mpt": MPTConfig,
8
+ "mpt": MptConfig,
9
"baichuan": BaiChuanConfig,
10
"aquila": AquilaConfig,
11
"qwen": QWenConfig,
vllm/transformers_utils/configs/__init__.py
-2
@@ -1,4 +1,3 @@
1
-from vllm.transformers_utils.configs.mpt import MPTConfig
1
from vllm.transformers_utils.configs.baichuan import BaiChuanConfig
2
from vllm.transformers_utils.configs.aquila import AquilaConfig
3
from vllm.transformers_utils.configs.qwen import QWenConfig
@@ -8,7 +7,6 @@ from vllm.transformers_utils.configs.qwen import QWenConfig
7
from vllm.transformers_utils.configs.falcon import RWConfig
8
9
__all__ = [
11
- "MPTConfig",
10
"BaiChuanConfig",
11
"AquilaConfig",
12
"QWenConfig",
vllm/transformers_utils/configs/mpt.py
deleted
-74
@@ -1,74 +0,0 @@
1
-# Adapted from
2
-# https://huggingface.co/mosaicml/mpt-7b/blob/main/configuration_mpt.py
3
-from typing import Any, Dict, Optional, Union
4
-
5
-from transformers import PretrainedConfig
6
-
7
-_ATTN_CONFIG_DEFAULTS = {
8
- "attn_type": "multihead_attention",
9
- "attn_pdrop": 0.0,
10
- "attn_impl": "triton",
11
- "qk_ln": False,
12
- "clip_qkv": None,
13
- "softmax_scale": None,
14
- "prefix_lm": False,
15
- "attn_uses_sequence_id": False,
16
- "alibi": False,
17
- "alibi_bias_max": 8,
18
-}
19
-
20
-
21
-class MPTConfig(PretrainedConfig):
22
- model_type = "mpt"
23
- attribute_map = {
24
- "hidden_size": "d_model",
25
- "num_attention_heads": "n_heads",
26
- "num_hidden_layers": "n_layers",
27
- }
28
-
29
- def __init__(
30
- self,
31
- d_model: int = 2048,
32
- n_heads: int = 16,
33
- n_layers: int = 24,
34
- expansion_ratio: int = 4,
35
- max_seq_len: int = 2048,
36
- vocab_size: int = 50368,
37
- resid_pdrop: float = 0.0,
38
- emb_pdrop: float = 0.0,
39
- learned_pos_emb: bool = True,
40
- attn_config: Optional[Dict[str, Any]] = None,
41
- init_device: str = "cpu",
42
- logit_scale: Optional[Union[float, str]] = None,
43
- no_bias: bool = False,
44
- verbose: int = 0,
45
- embedding_fraction: float = 1.0,
46
- norm_type: str = "low_precision_layernorm",
47
- use_cache: bool = False,
48
- **kwargs,
49
- ) -> None:
50
- self.d_model = d_model
51
- self.n_heads = n_heads
52
- self.n_layers = n_layers
53
- self.expansion_ratio = expansion_ratio
54
- self.max_seq_len = max_seq_len
55
- self.vocab_size = vocab_size
56
- self.resid_pdrop = resid_pdrop
57
- self.emb_pdrop = emb_pdrop
58
- self.learned_pos_emb = learned_pos_emb
59
- if attn_config is None:
60
- self.attn_config = _ATTN_CONFIG_DEFAULTS
61
- else:
62
- self.attn_config = attn_config
63
- self.init_device = init_device
64
- self.logit_scale = logit_scale
65
- self.no_bias = no_bias
66
- self.verbose = verbose
67
- self.embedding_fraction = embedding_fraction
68
- self.norm_type = norm_type
69
- self.use_cache = use_cache
70
- if "name" in kwargs:
71
- del kwargs["name"]
72
- if "loss_fn" in kwargs:
73
- del kwargs["loss_fn"]
74
- super().__init__(**kwargs)