A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
[Model] Add support for MPT (#334)
Woosuk Kwon committed
Jul 3, 2023 at 16:47 UTC
404422f42ed9c59ee816dacd9b54196a59ae65b2
11 files changed
+388
-4
README.md
+1
@@ -46,6 +46,7 @@ vLLM seamlessly supports many Huggingface models, including the following archit
46
- GPT BigCode (`bigcode/starcoder`, `bigcode/gpt_bigcode-santacoder`, etc.)
47
- GPT-NeoX (`EleutherAI/gpt-neox-20b`, `databricks/dolly-v2-12b`, `stabilityai/stablelm-tuned-alpha-7b`, etc.)
48
- LLaMA (`lmsys/vicuna-13b-v1.3`, `young-geng/koala`, `openlm-research/open_llama_13b`, etc.)
49
+- MPT (`mosaicml/mpt-7b`, `mosaicml/mpt-30b`, etc.)
50
- OPT (`facebook/opt-66b`, `facebook/opt-iml-max-30b`, etc.)
51
52
Install vLLM with pip or [from source](https://vllm.readthedocs.io/en/latest/getting_started/installation.html#build-from-source):
csrc/attention/attention_kernels.cu
+3
@@ -395,6 +395,9 @@ void single_query_cached_kv_attention_launcher(
395
case 96:
396
LAUNCH_ATTENTION_KERNEL(T, 96, BLOCK_SIZE, NUM_THREADS);
397
break;
398
+ case 112:
399
+ LAUNCH_ATTENTION_KERNEL(T, 112, BLOCK_SIZE, NUM_THREADS);
400
+ break;
401
case 128:
402
LAUNCH_ATTENTION_KERNEL(T, 128, BLOCK_SIZE, NUM_THREADS);
403
break;
docs/source/models/supported_models.rst
+3
@@ -29,6 +29,9 @@ Alongside each architecture, we include some popular models that use it.
29
* - :code:`LlamaForCausalLM`
30
- LLaMA, Vicuna, Alpaca, Koala, Guanaco
31
- :code:`openlm-research/open_llama_13b`, :code:`lmsys/vicuna-13b-v1.3`, :code:`young-geng/koala`, :code:`JosephusCheung/Guanaco`, etc.
32
+ * - :code: `MPTForCausalLM`
33
+ - MPT, MPT-Instruct, MPT-Chat, MPT-StoryWriter
34
+ - :code:`mosaicml/mpt-7b`, :code:`mosaicml/mpt-7b-storywriter`, :code:`mosaicml/mpt-30b`, etc.
35
* - :code:`OPTForCausalLM`
36
- OPT, OPT-IML
37
- :code:`facebook/opt-66b`, :code:`facebook/opt-iml-max-30b`, etc.
vllm/config.py
+3
-2
@@ -1,9 +1,10 @@
1
from typing import Optional
2
3
import torch
4
-from transformers import AutoConfig, PretrainedConfig
4
+from transformers import PretrainedConfig
5
6
from vllm.logger import init_logger
7
+from vllm.transformers_utils.config import get_config
8
from vllm.utils import get_cpu_memory
9
10
logger = init_logger(__name__)
@@ -49,7 +50,7 @@ class ModelConfig:
50
self.use_dummy_weights = use_dummy_weights
51
self.seed = seed
52
52
- self.hf_config: PretrainedConfig = AutoConfig.from_pretrained(model)
53
+ self.hf_config = get_config(model)
54
self.dtype = _get_and_verify_dtype(self.hf_config, dtype)
55
self._verify_tokenizer_mode()
56
vllm/model_executor/layers/attention.py
+1
-1
@@ -12,7 +12,7 @@ from vllm import cache_ops
12
from vllm import pos_encoding_ops
13
from vllm.model_executor.input_metadata import InputMetadata
14
15
-_SUPPORTED_HEAD_SIZES = [64, 80, 96, 128]
15
+_SUPPORTED_HEAD_SIZES = [64, 80, 96, 112, 128]
16
17
18
class PagedAttention(nn.Module):
vllm/model_executor/model_loader.py
+2
-1
@@ -16,7 +16,8 @@ _MODEL_REGISTRY = {
16
"GPTBigCodeForCausalLM": GPTBigCodeForCausalLM,
17
"GPTNeoXForCausalLM": GPTNeoXForCausalLM,
18
"LlamaForCausalLM": LlamaForCausalLM,
19
- "LLaMAForCausalLM": LlamaForCausalLM,
19
+ "LLaMAForCausalLM": LlamaForCausalLM, # For decapoda-research/llama-*
20
+ "MPTForCausalLM": MPTForCausalLM,
21
"OPTForCausalLM": OPTForCausalLM,
22
}
23
vllm/model_executor/models/__init__.py
+2
@@ -3,6 +3,7 @@ from vllm.model_executor.models.gpt2 import GPT2LMHeadModel
3
from vllm.model_executor.models.gpt_bigcode import GPTBigCodeForCausalLM
4
from vllm.model_executor.models.gpt_neox import GPTNeoXForCausalLM
5
from vllm.model_executor.models.llama import LlamaForCausalLM
6
+from vllm.model_executor.models.mpt import MPTForCausalLM
7
from vllm.model_executor.models.opt import OPTForCausalLM
8
9
__all__ = [
@@ -11,5 +12,6 @@ __all__ = [
12
"GPTBigCodeForCausalLM",
13
"GPTNeoXForCausalLM",
14
"LlamaForCausalLM",
15
+ "MPTForCausalLM",
16
"OPTForCausalLM",
17
]
vllm/model_executor/models/mpt.py
new
+279
@@ -0,0 +1,279 @@
1
+# Adapted from https://huggingface.co/mosaicml/mpt-7b/tree/main
2
+import math
3
+from typing import Dict, List, Optional, Tuple
4
+
5
+import torch
6
+import torch.nn as nn
7
+
8
+from vllm.model_executor.input_metadata import InputMetadata
9
+from vllm.model_executor.layers.activation import get_act_fn
10
+from vllm.model_executor.layers.attention import PagedAttentionWithALiBi
11
+from vllm.model_executor.layers.sampler import Sampler
12
+from vllm.model_executor.weight_utils import (hf_model_weights_iterator,
13
+ load_tensor_parallel_weights)
14
+from vllm.model_executor.parallel_utils.parallel_state import (
15
+ get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
16
+from vllm.model_executor.parallel_utils.tensor_parallel import (
17
+ VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
18
+from vllm.sequence import SequenceOutputs
19
+from vllm.transformers_utils.configs.mpt import MPTConfig
20
+
21
+KVCache = Tuple[torch.Tensor, torch.Tensor]
22
+
23
+
24
+def _get_alibi_slopes(
25
+ total_num_heads: int,
26
+ alibi_bias_max: int,
27
+) -> torch.Tensor:
28
+ next_power_of_2 = 2**math.ceil(math.log2(total_num_heads))
29
+ m = torch.arange(1, next_power_of_2 + 1, dtype=torch.float32)
30
+ m = m.mul(alibi_bias_max / next_power_of_2)
31
+ slopes = 1.0 / torch.pow(2, m)
32
+ if next_power_of_2 != total_num_heads:
33
+ slopes = torch.concat([slopes[1::2], slopes[::2]])[:total_num_heads]
34
+ return slopes
35
+
36
+
37
+class MPTAttention(nn.Module):
38
+
39
+ def __init__(self, config: MPTConfig):
40
+ super().__init__()
41
+ self.d_model = config.d_model
42
+ self.total_num_heads = config.n_heads
43
+ self.clip_qkv = config.attn_config["clip_qkv"]
44
+ self.qk_ln = config.attn_config["qk_ln"]
45
+ self.alibi_bias_max = config.attn_config["alibi_bias_max"]
46
+ assert not config.attn_config["prefix_lm"]
47
+ assert config.attn_config["alibi"]
48
+
49
+ self.qkv_proj = ColumnParallelLinear(
50
+ self.d_model,
51
+ 3 * self.d_model,
52
+ bias=not config.no_bias,
53
+ gather_output=False,
54
+ perform_initialization=False,
55
+ )
56
+ if self.qk_ln:
57
+ self.q_ln = nn.LayerNorm(self.d_model)
58
+ self.k_ln = nn.LayerNorm(self.d_model)
59
+ self.out_proj = RowParallelLinear(
60
+ self.d_model,
61
+ self.d_model,
62
+ bias=not config.no_bias,
63
+ input_is_parallel=True,
64
+ perform_initialization=False,
65
+ )
66
+
67
+ tp_world_size = get_tensor_model_parallel_world_size()
68
+ assert self.total_num_heads % tp_world_size == 0
69
+ self.num_heads = self.total_num_heads // tp_world_size
70
+
71
+ # Create the alibi slopes and slice them.
72
+ tp_rank = get_tensor_model_parallel_rank()
73
+ head_start = tp_rank * self.num_heads
74
+ head_end = (tp_rank + 1) * self.num_heads
75
+ alibi_slopes = _get_alibi_slopes(self.total_num_heads,
76
+ self.alibi_bias_max)
77
+ alibi_slopes = alibi_slopes[head_start:head_end].tolist()
78
+
79
+ self.head_dim = self.d_model // self.total_num_heads
80
+ scaling = self.head_dim**-0.5
81
+ self.attn = PagedAttentionWithALiBi(self.num_heads, self.head_dim,
82
+ scaling, alibi_slopes)
83
+
84
+ def forward(
85
+ self,
86
+ position_ids: torch.Tensor,
87
+ hidden_states: torch.Tensor,
88
+ kv_cache: KVCache,
89
+ input_metadata: InputMetadata,
90
+ cache_event: Optional[torch.cuda.Event],
91
+ ) -> torch.Tensor:
92
+ del position_ids # unused.
93
+ qkv, _ = self.qkv_proj(hidden_states)
94
+ if self.clip_qkv is not None:
95
+ qkv.clamp_(min=-self.clip_qkv, max=self.clip_qkv)
96
+ q, k, v = qkv.chunk(chunks=3, dim=-1)
97
+ if self.qk_ln:
98
+ q = self.q_ln(q)
99
+ k = self.k_ln(k)
100
+ k_cache, v_cache = kv_cache
101
+ attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
102
+ cache_event)
103
+ output, _ = self.out_proj(attn_output)
104
+ return output
105
+
106
+
107
+class MPTMLP(nn.Module):
108
+
109
+ def __init__(self, config: MPTConfig):
110
+ super().__init__()
111
+ hidden_size = config.d_model
112
+ expansion_ratio = config.expansion_ratio
113
+ intermediate_size = expansion_ratio * hidden_size
114
+ self.up_proj = ColumnParallelLinear(hidden_size,
115
+ intermediate_size,
116
+ bias=not config.no_bias,
117
+ gather_output=False,
118
+ perform_initialization=False)
119
+ self.act = get_act_fn("gelu")
120
+ self.down_proj = RowParallelLinear(intermediate_size,
121
+ hidden_size,
122
+ bias=not config.no_bias,
123
+ input_is_parallel=True,
124
+ perform_initialization=False)
125
+
126
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
127
+ x, _ = self.up_proj(x)
128
+ x = self.act(x)
129
+ x, _ = self.down_proj(x)
130
+ return x
131
+
132
+
133
+class MPTBlock(nn.Module):
134
+
135
+ def __init__(self, config: MPTConfig):
136
+ super().__init__()
137
+ hidden_size = config.d_model
138
+ self.norm_1 = nn.LayerNorm(hidden_size)
139
+ self.attn = MPTAttention(config)
140
+ self.norm_2 = nn.LayerNorm(hidden_size)
141
+ self.ffn = MPTMLP(config)
142
+
143
+ def forward(
144
+ self,
145
+ position_ids: torch.Tensor,
146
+ hidden_states: torch.Tensor,
147
+ kv_cache: KVCache,
148
+ input_metadata: InputMetadata,
149
+ cache_event: Optional[torch.cuda.Event],
150
+ ) -> torch.Tensor:
151
+ x = self.norm_1(hidden_states)
152
+ x = self.attn(
153
+ position_ids=position_ids,
154
+ hidden_states=x,
155
+ kv_cache=kv_cache,
156
+ input_metadata=input_metadata,
157
+ cache_event=cache_event,
158
+ )
159
+ hidden_states = hidden_states + x
160
+ x = self.norm_2(hidden_states)
161
+ x = self.ffn(x)
162
+ hidden_states = hidden_states + x
163
+ return hidden_states
164
+
165
+
166
+class MPTModel(nn.Module):
167
+
168
+ def __init__(self, config: MPTConfig):
169
+ super().__init__()
170
+ assert config.embedding_fraction == 1.0
171
+ assert config.norm_type == "low_precision_layernorm"
172
+
173
+ self.wte = VocabParallelEmbedding(config.vocab_size,
174
+ config.d_model,
175
+ perform_initialization=False)
176
+ self.blocks = nn.ModuleList(
177
+ [MPTBlock(config) for _ in range(config.n_layers)])
178
+ self.norm_f = nn.LayerNorm(config.d_model)
179
+ if config.no_bias:
180
+ for module in self.modules():
181
+ if hasattr(module, "bias"):
182
+ if isinstance(module.bias, nn.Parameter):
183
+ # Remove the bias term in Linear and LayerNorm.
184
+ module.register_parameter("bias", None)
185
+
186
+ def forward(
187
+ self,
188
+ input_ids: torch.Tensor,
189
+ position_ids: torch.Tensor,
190
+ kv_caches: List[KVCache],
191
+ input_metadata: InputMetadata,
192
+ cache_events: Optional[List[torch.cuda.Event]],
193
+ ) -> torch.Tensor:
194
+ hidden_states = self.wte(input_ids)
195
+ for i in range(len(self.blocks)):
196
+ if cache_events is None:
197
+ cache_event = None
198
+ else:
199
+ cache_event = cache_events[i]
200
+ block = self.blocks[i]
201
+ hidden_states = block(
202
+ position_ids,
203
+ hidden_states,
204
+ kv_caches[i],
205
+ input_metadata,
206
+ cache_event,
207
+ )
208
+ hidden_states = self.norm_f(hidden_states)
209
+ return hidden_states
210
+
211
+
212
+class MPTForCausalLM(nn.Module):
213
+
214
+ def __init__(self, config: MPTConfig):
215
+ super().__init__()
216
+ self.config = config
217
+ assert config.tie_word_embeddings
218
+
219
+ self.transformer = MPTModel(config)
220
+ # TODO(zhuohan): create a new weight after implementing pipeline
221
+ # parallelism
222
+ self.lm_head_weight = self.transformer.wte.weight
223
+ self.sampler = Sampler(config.vocab_size)
224
+
225
+ def forward(
226
+ self,
227
+ input_ids: torch.Tensor,
228
+ positions: torch.Tensor,
229
+ kv_caches: List[KVCache],
230
+ input_metadata: InputMetadata,
231
+ cache_events: Optional[List[torch.cuda.Event]],
232
+ ) -> Dict[int, SequenceOutputs]:
233
+ hidden_states = self.transformer(input_ids, positions, kv_caches,
234
+ input_metadata, cache_events)
235
+ next_tokens = self.sampler(self.lm_head_weight, hidden_states,
236
+ input_metadata)
237
+ return next_tokens
238
+
239
+ _column_parallel_weights = ["wte.weight", "up_proj.weight", "up_proj.bias"]
240
+ _row_parallel_weights = ["out_proj.weight", "down_proj.weight"]
241
+
242
+ def load_weights(self,
243
+ model_name_or_path: str,
244
+ cache_dir: Optional[str] = None,
245
+ use_np_cache: bool = False):
246
+ tp_world_size = get_tensor_model_parallel_world_size()
247
+ tp_rank = get_tensor_model_parallel_rank()
248
+ state_dict = self.state_dict()
249
+ for name, loaded_weight in hf_model_weights_iterator(
250
+ model_name_or_path, cache_dir, use_np_cache):
251
+ if "Wqkv" in name:
252
+ # NOTE(woosuk): MPT's fused QKV has the shape of
253
+ # [3 * num_heads * head_size, hidden_size].
254
+ # When tensor model parallelism is used, we need to shard
255
+ # the weight along the hidden dimension.
256
+ total_num_heads = self.config.num_attention_heads
257
+ hidden_size = self.config.hidden_size
258
+ head_size = hidden_size // total_num_heads
259
+ num_heads = total_num_heads // tp_world_size
260
+ head_start = tp_rank * num_heads
261
+ head_end = (tp_rank + 1) * num_heads
262
+
263
+ if name.endswith(".weight"):
264
+ loaded_weight = loaded_weight.view(3, total_num_heads,
265
+ head_size, hidden_size)
266
+ loaded_weight = loaded_weight[:, head_start:head_end, :, :]
267
+ loaded_weight = loaded_weight.reshape(-1, hidden_size)
268
+ elif name.endswith(".bias"):
269
+ loaded_weight = loaded_weight.view(3, total_num_heads,
270
+ head_size)
271
+ loaded_weight = loaded_weight[:, head_start:head_end, :]
272
+ loaded_weight = loaded_weight.reshape(-1)
273
+ else:
274
+ raise ValueError(f"Unexpected parameter name {name}")
275
+ name = name.replace("Wqkv", "qkv_proj")
276
+ param = state_dict[name]
277
+ load_tensor_parallel_weights(param, loaded_weight, name,
278
+ self._column_parallel_weights,
279
+ self._row_parallel_weights, tp_rank)
vllm/transformers_utils/config.py
new
+15
@@ -0,0 +1,15 @@
1
+from transformers import AutoConfig, PretrainedConfig
2
+
3
+from vllm.transformers_utils.configs import * # pylint: disable=wildcard-import
4
+
5
+_CONFIG_REGISTRY = {
6
+ "mpt": MPTConfig,
7
+}
8
+
9
+
10
+def get_config(model: str) -> PretrainedConfig:
11
+ config = AutoConfig.from_pretrained(model, trust_remote_code=True)
12
+ if config.model_type in _CONFIG_REGISTRY:
13
+ config_class = _CONFIG_REGISTRY[config.model_type]
14
+ config = config_class.from_pretrained(model)
15
+ return config
vllm/transformers_utils/configs/__init__.py
new
+5
@@ -0,0 +1,5 @@
1
+from vllm.transformers_utils.configs.mpt import MPTConfig
2
+
3
+__all__ = [
4
+ "MPTConfig",
5
+]
vllm/transformers_utils/configs/mpt.py
new
+74
@@ -0,0 +1,74 @@
1
+# Adapted from
2
+# https://huggingface.co/mosaicml/mpt-7b/blob/main/configuration_mpt.py
3
+from typing import Any, Dict, Optional, Union
4
+
5
+from transformers import PretrainedConfig
6
+
7
+_ATTN_CONFIG_DEFAULTS = {
8
+ "attn_type": "multihead_attention",
9
+ "attn_pdrop": 0.0,
10
+ "attn_impl": "triton",
11
+ "qk_ln": False,
12
+ "clip_qkv": None,
13
+ "softmax_scale": None,
14
+ "prefix_lm": False,
15
+ "attn_uses_sequence_id": False,
16
+ "alibi": False,
17
+ "alibi_bias_max": 8,
18
+}
19
+
20
+
21
+class MPTConfig(PretrainedConfig):
22
+ model_type = "mpt"
23
+ attribute_map = {
24
+ "hidden_size": "d_model",
25
+ "num_attention_heads": "n_heads",
26
+ "num_hidden_layers": "n_layers",
27
+ }
28
+
29
+ def __init__(
30
+ self,
31
+ d_model: int = 2048,
32
+ n_heads: int = 16,
33
+ n_layers: int = 24,
34
+ expansion_ratio: int = 4,
35
+ max_seq_len: int = 2048,
36
+ vocab_size: int = 50368,
37
+ resid_pdrop: float = 0.0,
38
+ emb_pdrop: float = 0.0,
39
+ learned_pos_emb: bool = True,
40
+ attn_config: Optional[Dict[str, Any]] = None,
41
+ init_device: str = "cpu",
42
+ logit_scale: Optional[Union[float, str]] = None,
43
+ no_bias: bool = False,
44
+ verbose: int = 0,
45
+ embedding_fraction: float = 1.0,
46
+ norm_type: str = "low_precision_layernorm",
47
+ use_cache: bool = False,
48
+ **kwargs,
49
+ ) -> None:
50
+ self.d_model = d_model
51
+ self.n_heads = n_heads
52
+ self.n_layers = n_layers
53
+ self.expansion_ratio = expansion_ratio
54
+ self.max_seq_len = max_seq_len
55
+ self.vocab_size = vocab_size
56
+ self.resid_pdrop = resid_pdrop
57
+ self.emb_pdrop = emb_pdrop
58
+ self.learned_pos_emb = learned_pos_emb
59
+ if attn_config is None:
60
+ self.attn_config = _ATTN_CONFIG_DEFAULTS
61
+ else:
62
+ self.attn_config = attn_config
63
+ self.init_device = init_device
64
+ self.logit_scale = logit_scale
65
+ self.no_bias = no_bias
66
+ self.verbose = verbose
67
+ self.embedding_fraction = embedding_fraction
68
+ self.norm_type = norm_type
69
+ self.use_cache = use_cache
70
+ if "name" in kwargs:
71
+ del kwargs["name"]
72
+ if "loss_fn" in kwargs:
73
+ del kwargs["loss_fn"]
74
+ super().__init__(**kwargs)