@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b514d3c496
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Revert `MptConfig` to `MPTConfig` (#1668)

Megha Agarwal committed Nov 16, 2023 at 01:19 UTC b514d3c4960eb10c9963a6a582540e2aafafcfbb
6 files changed +260 -26
vllm/model_executor/model_loader.py
+2 -2
@@ -29,8 +29,8 @@ _MODEL_REGISTRY = {
29 "LLaMAForCausalLM": LlamaForCausalLM, # For decapoda-research/llama-*
30 "MistralForCausalLM": MistralForCausalLM,
31 # transformers's mpt class has lower case
32 - "MptForCausalLM": MptForCausalLM,
33 - "MPTForCausalLM": MptForCausalLM,
32 + "MptForCausalLM": MPTForCausalLM,
33 + "MPTForCausalLM": MPTForCausalLM,
34 "OPTForCausalLM": OPTForCausalLM,
35 "QWenLMHeadModel": QWenLMHeadModel,
36 "RWForCausalLM": FalconForCausalLM,
vllm/model_executor/models/__init__.py
+2 -2
@@ -10,7 +10,7 @@ from vllm.model_executor.models.gpt_neox import GPTNeoXForCausalLM
10 from vllm.model_executor.models.internlm import InternLMForCausalLM
11 from vllm.model_executor.models.llama import LlamaForCausalLM
12 from vllm.model_executor.models.mistral import MistralForCausalLM
13 -from vllm.model_executor.models.mpt import MptForCausalLM
13 +from vllm.model_executor.models.mpt import MPTForCausalLM
14 from vllm.model_executor.models.opt import OPTForCausalLM
15 from vllm.model_executor.models.qwen import QWenLMHeadModel
16 from vllm.model_executor.models.chatglm import ChatGLMForCausalLM
@@ -29,7 +29,7 @@ __all__ = [
29 "GPTNeoXForCausalLM",
30 "InternLMForCausalLM",
31 "LlamaForCausalLM",
32 - "MptForCausalLM",
32 + "MPTForCausalLM",
33 "OPTForCausalLM",
34 "QWenLMHeadModel",
35 "MistralForCausalLM",
vllm/model_executor/models/mpt.py
+20 -20
@@ -5,7 +5,6 @@ from typing import List, Optional, Tuple
5
6 import torch
7 import torch.nn as nn
8 -from transformers import MptConfig
8
9 from vllm.model_executor.input_metadata import InputMetadata
10 from vllm.model_executor.layers.activation import get_act_fn
@@ -22,6 +21,7 @@ from vllm.model_executor.parallel_utils.parallel_state import (
21 from vllm.model_executor.weight_utils import (default_weight_loader,
22 hf_model_weights_iterator)
23 from vllm.sequence import SamplerOutput
24 +from vllm.transformers_utils.configs.mpt import MPTConfig
25
26 KVCache = Tuple[torch.Tensor, torch.Tensor]
27
@@ -39,21 +39,21 @@ def _get_alibi_slopes(
39 return slopes
40
41
42 -class MptAttention(nn.Module):
42 +class MPTAttention(nn.Module):
43
44 def __init__(
45 self,
46 - config: MptConfig,
46 + config: MPTConfig,
47 linear_method: Optional[LinearMethodBase] = None,
48 ):
49 super().__init__()
50 self.d_model = config.d_model
51 self.total_num_heads = config.n_heads
52 - self.clip_qkv = config.attn_config.clip_qkv
53 - self.qk_ln = config.attn_config.qk_ln
54 - self.alibi_bias_max = config.attn_config.alibi_bias_max
55 - assert not config.attn_config.prefix_lm
56 - assert config.attn_config.alibi
52 + self.clip_qkv = config.attn_config["clip_qkv"]
53 + self.qk_ln = config.attn_config["qk_ln"]
54 + self.alibi_bias_max = config.attn_config["alibi_bias_max"]
55 + assert not config.attn_config["prefix_lm"]
56 + assert config.attn_config["alibi"]
57
58 # pylint: disable=invalid-name
59 self.Wqkv = QKVParallelLinear(
@@ -113,11 +113,11 @@ class MptAttention(nn.Module):
113 return output
114
115
116 -class MptMLP(nn.Module):
116 +class MPTMLP(nn.Module):
117
118 def __init__(
119 self,
120 - config: MptConfig,
120 + config: MPTConfig,
121 linear_method: Optional[LinearMethodBase] = None,
122 ):
123 super().__init__()
@@ -145,19 +145,19 @@ class MptMLP(nn.Module):
145 return x
146
147
148 -class MptBlock(nn.Module):
148 +class MPTBlock(nn.Module):
149
150 def __init__(
151 self,
152 - config: MptConfig,
152 + config: MPTConfig,
153 linear_method: Optional[LinearMethodBase] = None,
154 ):
155 super().__init__()
156 hidden_size = config.d_model
157 self.norm_1 = nn.LayerNorm(hidden_size)
158 - self.attn = MptAttention(config, linear_method)
158 + self.attn = MPTAttention(config, linear_method)
159 self.norm_2 = nn.LayerNorm(hidden_size)
160 - self.ffn = MptMLP(config, linear_method)
160 + self.ffn = MPTMLP(config, linear_method)
161
162 def forward(
163 self,
@@ -182,11 +182,11 @@ class MptBlock(nn.Module):
182 return hidden_states
183
184
185 -class MptModel(nn.Module):
185 +class MPTModel(nn.Module):
186
187 def __init__(
188 self,
189 - config: MptConfig,
189 + config: MPTConfig,
190 linear_method: Optional[LinearMethodBase] = None,
191 ):
192 super().__init__()
@@ -198,7 +198,7 @@ class MptModel(nn.Module):
198 config.d_model,
199 )
200 self.blocks = nn.ModuleList(
201 - [MptBlock(config, linear_method) for _ in range(config.n_layers)])
201 + [MPTBlock(config, linear_method) for _ in range(config.n_layers)])
202 self.norm_f = nn.LayerNorm(config.d_model)
203 if config.no_bias:
204 for module in self.modules():
@@ -233,11 +233,11 @@ class MptModel(nn.Module):
233 return hidden_states
234
235
236 -class MptForCausalLM(nn.Module):
236 +class MPTForCausalLM(nn.Module):
237
238 def __init__(
239 self,
240 - config: MptConfig,
240 + config: MPTConfig,
241 linear_method: Optional[LinearMethodBase] = None,
242 ):
243 super().__init__()
@@ -245,7 +245,7 @@ class MptForCausalLM(nn.Module):
245 assert config.tie_word_embeddings
246 self.linear_method = linear_method
247
248 - self.transformer = MptModel(config, linear_method)
248 + self.transformer = MPTModel(config, linear_method)
249 self.lm_head_weight = self.transformer.wte.weight
250 self.sampler = Sampler(config.vocab_size)
251
vllm/transformers_utils/config.py
+2 -2
@@ -1,6 +1,6 @@
1 from typing import Optional
2
3 -from transformers import AutoConfig, MptConfig, PretrainedConfig
3 +from transformers import AutoConfig, PretrainedConfig
4
5 from vllm.transformers_utils.configs import * # pylint: disable=wildcard-import
6
@@ -8,7 +8,7 @@ _CONFIG_REGISTRY = {
8 "aquila": AquilaConfig,
9 "baichuan": BaiChuanConfig,
10 "chatglm": ChatGLMConfig,
11 - "mpt": MptConfig,
11 + "mpt": MPTConfig,
12 "qwen": QWenConfig,
13 "RefinedWeb": RWConfig, # For tiiuae/falcon-40b(-instruct)
14 "RefinedWebModel": RWConfig, # For tiiuae/falcon-7b(-instruct)
vllm/transformers_utils/configs/__init__.py
+2
@@ -1,6 +1,7 @@
1 from vllm.transformers_utils.configs.aquila import AquilaConfig
2 from vllm.transformers_utils.configs.baichuan import BaiChuanConfig
3 from vllm.transformers_utils.configs.chatglm import ChatGLMConfig
4 +from vllm.transformers_utils.configs.mpt import MPTConfig
5 from vllm.transformers_utils.configs.qwen import QWenConfig
6 # RWConfig is for the original tiiuae/falcon-40b(-instruct) and
7 # tiiuae/falcon-7b(-instruct) models. Newer Falcon models will use the
@@ -12,6 +13,7 @@ __all__ = [
13 "AquilaConfig",
14 "BaiChuanConfig",
15 "ChatGLMConfig",
16 + "MPTConfig",
17 "QWenConfig",
18 "RWConfig",
19 "YiConfig",
vllm/transformers_utils/configs/mpt.py new
+232
@@ -0,0 +1,232 @@
1 +# coding=utf-8
2 +# Copied from
3 +# https://huggingface.co/mosaicml/mpt-7b/blob/main/configuration_mpt.py
4 +"""A HuggingFace-style model configuration."""
5 +import warnings
6 +from typing import Any, Dict, Optional, Union
7 +from transformers import PretrainedConfig
8 +
9 +attn_config_defaults: Dict = {
10 + 'attn_type': 'multihead_attention',
11 + 'attn_pdrop': 0.0,
12 + 'attn_impl': 'triton',
13 + 'qk_ln': False,
14 + 'clip_qkv': None,
15 + 'softmax_scale': None,
16 + 'prefix_lm': False,
17 + 'attn_uses_sequence_id': False,
18 + 'alibi': False,
19 + 'alibi_bias_max': 8
20 +}
21 +ffn_config_defaults: Dict = {'ffn_type': 'mptmlp'}
22 +init_config_defaults: Dict = {
23 + 'name': 'kaiming_normal_',
24 + 'fan_mode': 'fan_in',
25 + 'init_nonlinearity': 'relu',
26 + 'init_div_is_residual': True,
27 + 'emb_init_std': None,
28 + 'emb_init_uniform_lim': None,
29 + 'init_std': None,
30 + 'init_gain': 0.0
31 +}
32 +
33 +
34 +class MPTConfig(PretrainedConfig):
35 + model_type = 'mpt'
36 + attribute_map = {
37 + 'num_attention_heads': 'n_heads',
38 + 'hidden_size': 'd_model',
39 + 'num_hidden_layers': 'n_layers',
40 + }
41 +
42 + # pylint: disable=dangerous-default-value
43 + def __init__(self,
44 + d_model: int = 2048,
45 + n_heads: int = 16,
46 + n_layers: int = 24,
47 + expansion_ratio: int = 4,
48 + max_seq_len: int = 2048,
49 + vocab_size: int = 50368,
50 + resid_pdrop: float = 0.0,
51 + emb_pdrop: float = 0.0,
52 + learned_pos_emb: bool = True,
53 + attn_config: Dict = attn_config_defaults,
54 + ffn_config: Dict = ffn_config_defaults,
55 + init_device: str = 'cpu',
56 + logit_scale: Optional[Union[float, str]] = None,
57 + no_bias: bool = False,
58 + embedding_fraction: float = 1.0,
59 + norm_type: str = 'low_precision_layernorm',
60 + use_cache: bool = False,
61 + init_config: Dict = init_config_defaults,
62 + fc_type: str = 'torch',
63 + verbose: Optional[int] = None,
64 + **kwargs: Any):
65 + # pylint: disable=line-too-long
66 + """The MPT configuration class.
67 + Args:
68 + d_model (int): The size of the embedding dimension of the model.
69 + n_heads (int): The number of attention heads.
70 + n_layers (int): The number of layers in the model.
71 + expansion_ratio (int): The ratio of the up/down scale in the ffn.
72 + max_seq_len (int): The maximum sequence length of the model.
73 + vocab_size (int): The size of the vocabulary.
74 + resid_pdrop (float): The dropout probability applied to the attention output before combining with residual.
75 + emb_pdrop (float): The dropout probability for the embedding layer.
76 + learned_pos_emb (bool): Whether to use learned positional embeddings
77 + attn_config (Dict): A dictionary used to configure the model's attention module:
78 + attn_type (str): type of attention to use. Options: multihead_attention, multiquery_attention, grouped_query_attention
79 + attn_pdrop (float): The dropout probability for the attention layers.
80 + attn_impl (str): The attention implementation to use. One of 'torch', 'flash', or 'triton'.
81 + qk_ln (bool): Whether to apply layer normalization to the queries and keys in the attention layer.
82 + clip_qkv (Optional[float]): If not None, clip the queries, keys, and values in the attention layer to
83 + this value.
84 + softmax_scale (Optional[float]): If not None, scale the softmax in the attention layer by this value. If None,
85 + use the default scale of ``1/sqrt(d_keys)``.
86 + prefix_lm (Optional[bool]): Whether the model should operate as a Prefix LM. This requires passing an
87 + extra `prefix_mask` argument which indicates which tokens belong to the prefix. Tokens in the prefix
88 + can attend to one another bi-directionally. Tokens outside the prefix use causal attention.
89 + attn_uses_sequence_id (Optional[bool]): Whether to restrict attention to tokens that have the same sequence_id.
90 + When the model is in `train` mode, this requires passing an extra `sequence_id` argument which indicates
91 + which sub-sequence each token belongs to.
92 + Defaults to ``False`` meaning any provided `sequence_id` will be ignored.
93 + alibi (bool): Whether to use the alibi bias instead of position embeddings.
94 + alibi_bias_max (int): The maximum value of the alibi bias.
95 + kv_n_heads (Optional[int]): For grouped_query_attention only, allow user to specify number of kv heads.
96 + ffn_config (Dict): A dictionary used to configure the model's ffn module:
97 + ffn_type (str): type of ffn to use. Options: mptmlp, te_ln_mlp
98 + init_device (str): The device to use for parameter initialization.
99 + logit_scale (Optional[Union[float, str]]): If not None, scale the logits by this value.
100 + no_bias (bool): Whether to use bias in all layers.
101 + verbose (int): The verbosity level. 0 is silent.
102 + embedding_fraction (float): The fraction to scale the gradients of the embedding layer by.
103 + norm_type (str): choose type of norm to use
104 + use_cache (bool): Whether or not the model should return the last key/values attentions
105 + init_config (Dict): A dictionary used to configure the model initialization:
106 + init_config.name: The parameter initialization scheme to use. Options: 'default_', 'baseline_',
107 + 'kaiming_uniform_', 'kaiming_normal_', 'neox_init_', 'small_init_', 'xavier_uniform_', or
108 + 'xavier_normal_'. These mimic the parameter initialization methods in PyTorch.
109 + init_div_is_residual (Union[int, float, str, bool]): Value to divide initial weights by if ``module._is_residual`` is True.
110 + emb_init_std (Optional[float]): The standard deviation of the normal distribution used to initialize the embedding layer.
111 + emb_init_uniform_lim (Optional[Union[Tuple[float, float], float]]): The lower and upper limits of the uniform distribution
112 + used to initialize the embedding layer. Mutually exclusive with ``emb_init_std``.
113 + init_std (float): The standard deviation of the normal distribution used to initialize the model,
114 + if using the baseline_ parameter initialization scheme.
115 + init_gain (float): The gain to use for parameter initialization with kaiming or xavier initialization schemes.
116 + fan_mode (str): The fan mode to use for parameter initialization with kaiming initialization schemes.
117 + init_nonlinearity (str): The nonlinearity to use for parameter initialization with kaiming initialization schemes.
118 + ---
119 + See llmfoundry.models.utils.param_init_fns.py for info on other param init config options
120 + fc_type (str): choose fc layer implementation. Options: torch and te. te layers support fp8 when using H100 GPUs.
121 + """
122 + self.d_model = d_model
123 + self.n_heads = n_heads
124 + self.n_layers = n_layers
125 + self.expansion_ratio = expansion_ratio
126 + self.max_seq_len = max_seq_len
127 + self.vocab_size = vocab_size
128 + self.resid_pdrop = resid_pdrop
129 + self.emb_pdrop = emb_pdrop
130 + self.learned_pos_emb = learned_pos_emb
131 + self.attn_config = attn_config
132 + self.ffn_config = ffn_config
133 + self.init_device = init_device
134 + self.logit_scale = logit_scale
135 + self.no_bias = no_bias
136 + self.embedding_fraction = embedding_fraction
137 + self.norm_type = norm_type
138 + self.use_cache = use_cache
139 + self.init_config = init_config
140 + self.fc_type = fc_type
141 + if verbose is not None:
142 + warnings.warn(
143 + DeprecationWarning(
144 + 'verbose argument for MPTConfig is now ignored and will be removed. Use python_log_level instead.'
145 + ))
146 + if 'name' in kwargs:
147 + del kwargs['name']
148 + if 'loss_fn' in kwargs:
149 + del kwargs['loss_fn']
150 + if self.attn_config.get('alibi', False):
151 + self.learned_pos_emb = False
152 + warnings.warn(
153 + f'alibi is turned on, setting `learned_pos_emb` to {self.learned_pos_emb}`'
154 + )
155 + super().__init__(**kwargs)
156 + self._validate_config()
157 +
158 + def _set_config_defaults(
159 + self, config: Dict[str, Any],
160 + config_defaults: Dict[str, Any]) -> Dict[str, Any]:
161 + for (k, v) in config_defaults.items():
162 + if k not in config:
163 + config[k] = v
164 + return config
165 +
166 + def _validate_config(self) -> None:
167 + self.attn_config = self._set_config_defaults(self.attn_config,
168 + attn_config_defaults)
169 + self.ffn_config = self._set_config_defaults(self.ffn_config,
170 + ffn_config_defaults)
171 + self.init_config = self._set_config_defaults(self.init_config,
172 + init_config_defaults)
173 + if self.d_model % self.n_heads != 0:
174 + raise ValueError('d_model must be divisible by n_heads')
175 + if any((
176 + prob < 0 or prob > 1 for prob in
177 + [self.attn_config['attn_pdrop'], self.resid_pdrop, self.emb_pdrop]
178 + )):
179 + raise ValueError(
180 + "self.attn_config['attn_pdrop'], resid_pdrop, emb_pdrop are probabilities and must be between 0 and 1" # pylint: disable=line-too-long
181 + )
182 + if self.attn_config['attn_impl'] not in ['torch', 'flash', 'triton']:
183 + raise ValueError(
184 + f"Unknown attn_impl={self.attn_config['attn_impl']}")
185 + if self.attn_config['prefix_lm'] and self.attn_config[
186 + 'attn_impl'] not in ['torch', 'triton']:
187 + raise NotImplementedError(
188 + 'prefix_lm only implemented with torch and triton attention.')
189 + if self.attn_config['alibi'] and self.attn_config['attn_impl'] not in [
190 + 'torch', 'triton'
191 + ]:
192 + raise NotImplementedError(
193 + 'alibi only implemented with torch and triton attention.')
194 + if self.attn_config['attn_uses_sequence_id'] and self.attn_config[
195 + 'attn_impl'] not in ['torch', 'triton']:
196 + raise NotImplementedError(
197 + 'attn_uses_sequence_id only implemented with torch and triton attention.' # pylint: disable=line-too-long
198 + )
199 + if self.embedding_fraction > 1 or self.embedding_fraction <= 0:
200 + raise ValueError(
201 + 'model.embedding_fraction must be between 0 (exclusive) and 1 (inclusive)!' # pylint: disable=line-too-long
202 + )
203 + if isinstance(self.logit_scale,
204 + str) and self.logit_scale != 'inv_sqrt_d_model':
205 + raise ValueError(
206 + f"self.logit_scale={self.logit_scale!r} is not recognized as an option; use numeric value or 'inv_sqrt_d_model'." # pylint: disable=line-too-long
207 + )
208 + if self.init_config.get('name', None) is None:
209 + raise ValueError(
210 + f"self.init_config={self.init_config!r} 'name' needs to be set."
211 + )
212 + if not self.learned_pos_emb and (not self.attn_config['alibi']):
213 + warnings.warn(
214 + 'Positional information not being provided to the model.')
215 + if self.fc_type == 'te' or self.ffn_config['ffn_type'] == 'te_ln_mlp':
216 + try:
217 + # pylint: disable=import-outside-toplevel
218 + import transformer_engine.pytorch as te
219 + del te
220 + except Exception as exc:
221 + raise ImportError(
222 + # pylint: disable=line-too-long
223 + 'TransformerEngine import fail. `fc_type: te` requires TransformerEngine be installed. '
224 + +
225 + 'The required version of transformer_engine also requires FlashAttention v1.0.6 is installed:\n'
226 + + 'pip install flash-attn==1.0.6 --no-build-isolation \n' +
227 + 'pip install git+https://github.com/NVIDIA/TransformerEngine.git@144e4888b2cdd60bd52e706d5b7a79cb9c1a7156'
228 + ) from exc
229 + if self.ffn_config['ffn_type'] == 'mptmlp':
230 + self.ffn_config['fc_type'] = self.fc_type
231 + elif self.ffn_config['ffn_type'] == 'te_ln_mlp':
232 + self.ffn_config['bias'] = not self.no_bias