@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1fe0990023
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Remove `MPTConfig` (#1529)

Woosuk Kwon committed Nov 1, 2023 at 15:29 UTC 1fe09900238870a0477fa638dbd2d2bd2f251e1d
6 files changed +26 -102
vllm/model_executor/model_loader.py
+2 -2
@@ -28,8 +28,8 @@ _MODEL_REGISTRY = {
28 "LLaMAForCausalLM": LlamaForCausalLM, # For decapoda-research/llama-*
29 "MistralForCausalLM": MistralForCausalLM,
30 # transformers's mpt class has lower case
31 - "MptForCausalLM": MPTForCausalLM,
32 - "MPTForCausalLM": MPTForCausalLM,
31 + "MptForCausalLM": MptForCausalLM,
32 + "MPTForCausalLM": MptForCausalLM,
33 "OPTForCausalLM": OPTForCausalLM,
34 "QWenLMHeadModel": QWenLMHeadModel,
35 "RWForCausalLM": FalconForCausalLM,
vllm/model_executor/models/__init__.py
+2 -2
@@ -10,7 +10,7 @@ from vllm.model_executor.models.gpt_neox import GPTNeoXForCausalLM
10 from vllm.model_executor.models.internlm import InternLMForCausalLM
11 from vllm.model_executor.models.llama import LlamaForCausalLM
12 from vllm.model_executor.models.mistral import MistralForCausalLM
13 -from vllm.model_executor.models.mpt import MPTForCausalLM
13 +from vllm.model_executor.models.mpt import MptForCausalLM
14 from vllm.model_executor.models.opt import OPTForCausalLM
15 from vllm.model_executor.models.qwen import QWenLMHeadModel
16
@@ -26,7 +26,7 @@ __all__ = [
26 "GPTNeoXForCausalLM",
27 "InternLMForCausalLM",
28 "LlamaForCausalLM",
29 - "MPTForCausalLM",
29 + "MptForCausalLM",
30 "OPTForCausalLM",
31 "QWenLMHeadModel",
32 "MistralForCausalLM",
vllm/model_executor/models/mpt.py
+20 -20
@@ -5,6 +5,7 @@ from typing import List, Optional, Tuple
5
6 import torch
7 import torch.nn as nn
8 +from transformers import MptConfig
9
10 from vllm.model_executor.input_metadata import InputMetadata
11 from vllm.model_executor.layers.activation import get_act_fn
@@ -19,7 +20,6 @@ from vllm.model_executor.parallel_utils.layers import (VocabParallelEmbedding,
20 ColumnParallelLinear,
21 RowParallelLinear)
22 from vllm.sequence import SamplerOutput
22 -from vllm.transformers_utils.configs.mpt import MPTConfig
23
24 KVCache = Tuple[torch.Tensor, torch.Tensor]
25
@@ -37,17 +37,17 @@ def _get_alibi_slopes(
37 return slopes
38
39
40 -class MPTAttention(nn.Module):
40 +class MptAttention(nn.Module):
41
42 - def __init__(self, config: MPTConfig):
42 + def __init__(self, config: MptConfig):
43 super().__init__()
44 self.d_model = config.d_model
45 self.total_num_heads = config.n_heads
46 - self.clip_qkv = config.attn_config["clip_qkv"]
47 - self.qk_ln = config.attn_config["qk_ln"]
48 - self.alibi_bias_max = config.attn_config["alibi_bias_max"]
49 - assert not config.attn_config["prefix_lm"]
50 - assert config.attn_config["alibi"]
46 + self.clip_qkv = config.attn_config.clip_qkv
47 + self.qk_ln = config.attn_config.qk_ln
48 + self.alibi_bias_max = config.attn_config.alibi_bias_max
49 + assert not config.attn_config.prefix_lm
50 + assert config.attn_config.alibi
51
52 self.qkv_proj = ColumnParallelLinear(
53 self.d_model,
@@ -105,9 +105,9 @@ class MPTAttention(nn.Module):
105 return output
106
107
108 -class MPTMLP(nn.Module):
108 +class MptMLP(nn.Module):
109
110 - def __init__(self, config: MPTConfig):
110 + def __init__(self, config: MptConfig):
111 super().__init__()
112 hidden_size = config.d_model
113 expansion_ratio = config.expansion_ratio
@@ -133,15 +133,15 @@ class MPTMLP(nn.Module):
133 return x
134
135
136 -class MPTBlock(nn.Module):
136 +class MptBlock(nn.Module):
137
138 - def __init__(self, config: MPTConfig):
138 + def __init__(self, config: MptConfig):
139 super().__init__()
140 hidden_size = config.d_model
141 self.norm_1 = nn.LayerNorm(hidden_size)
142 - self.attn = MPTAttention(config)
142 + self.attn = MptAttention(config)
143 self.norm_2 = nn.LayerNorm(hidden_size)
144 - self.ffn = MPTMLP(config)
144 + self.ffn = MptMLP(config)
145
146 def forward(
147 self,
@@ -166,9 +166,9 @@ class MPTBlock(nn.Module):
166 return hidden_states
167
168
169 -class MPTModel(nn.Module):
169 +class MptModel(nn.Module):
170
171 - def __init__(self, config: MPTConfig):
171 + def __init__(self, config: MptConfig):
172 super().__init__()
173 assert config.embedding_fraction == 1.0
174 assert config.norm_type == "low_precision_layernorm"
@@ -178,7 +178,7 @@ class MPTModel(nn.Module):
178 config.d_model,
179 )
180 self.blocks = nn.ModuleList(
181 - [MPTBlock(config) for _ in range(config.n_layers)])
181 + [MptBlock(config) for _ in range(config.n_layers)])
182 self.norm_f = nn.LayerNorm(config.d_model)
183 if config.no_bias:
184 for module in self.modules():
@@ -213,14 +213,14 @@ class MPTModel(nn.Module):
213 return hidden_states
214
215
216 -class MPTForCausalLM(nn.Module):
216 +class MptForCausalLM(nn.Module):
217
218 - def __init__(self, config: MPTConfig):
218 + def __init__(self, config: MptConfig):
219 super().__init__()
220 self.config = config
221 assert config.tie_word_embeddings
222
223 - self.transformer = MPTModel(config)
223 + self.transformer = MptModel(config)
224 # TODO(zhuohan): create a new weight after implementing pipeline
225 # parallelism
226 self.lm_head_weight = self.transformer.wte.weight
vllm/transformers_utils/config.py
+2 -2
@@ -1,11 +1,11 @@
1 from typing import Optional
2
3 -from transformers import AutoConfig, PretrainedConfig
3 +from transformers import AutoConfig, MptConfig, PretrainedConfig
4
5 from vllm.transformers_utils.configs import * # pylint: disable=wildcard-import
6
7 _CONFIG_REGISTRY = {
8 - "mpt": MPTConfig,
8 + "mpt": MptConfig,
9 "baichuan": BaiChuanConfig,
10 "aquila": AquilaConfig,
11 "qwen": QWenConfig,
vllm/transformers_utils/configs/__init__.py
-2
@@ -1,4 +1,3 @@
1 -from vllm.transformers_utils.configs.mpt import MPTConfig
1 from vllm.transformers_utils.configs.baichuan import BaiChuanConfig
2 from vllm.transformers_utils.configs.aquila import AquilaConfig
3 from vllm.transformers_utils.configs.qwen import QWenConfig
@@ -8,7 +7,6 @@ from vllm.transformers_utils.configs.qwen import QWenConfig
7 from vllm.transformers_utils.configs.falcon import RWConfig
8
9 __all__ = [
11 - "MPTConfig",
10 "BaiChuanConfig",
11 "AquilaConfig",
12 "QWenConfig",
vllm/transformers_utils/configs/mpt.py deleted
-74
@@ -1,74 +0,0 @@
1 -# Adapted from
2 -# https://huggingface.co/mosaicml/mpt-7b/blob/main/configuration_mpt.py
3 -from typing import Any, Dict, Optional, Union
4 -
5 -from transformers import PretrainedConfig
6 -
7 -_ATTN_CONFIG_DEFAULTS = {
8 - "attn_type": "multihead_attention",
9 - "attn_pdrop": 0.0,
10 - "attn_impl": "triton",
11 - "qk_ln": False,
12 - "clip_qkv": None,
13 - "softmax_scale": None,
14 - "prefix_lm": False,
15 - "attn_uses_sequence_id": False,
16 - "alibi": False,
17 - "alibi_bias_max": 8,
18 -}
19 -
20 -
21 -class MPTConfig(PretrainedConfig):
22 - model_type = "mpt"
23 - attribute_map = {
24 - "hidden_size": "d_model",
25 - "num_attention_heads": "n_heads",
26 - "num_hidden_layers": "n_layers",
27 - }
28 -
29 - def __init__(
30 - self,
31 - d_model: int = 2048,
32 - n_heads: int = 16,
33 - n_layers: int = 24,
34 - expansion_ratio: int = 4,
35 - max_seq_len: int = 2048,
36 - vocab_size: int = 50368,
37 - resid_pdrop: float = 0.0,
38 - emb_pdrop: float = 0.0,
39 - learned_pos_emb: bool = True,
40 - attn_config: Optional[Dict[str, Any]] = None,
41 - init_device: str = "cpu",
42 - logit_scale: Optional[Union[float, str]] = None,
43 - no_bias: bool = False,
44 - verbose: int = 0,
45 - embedding_fraction: float = 1.0,
46 - norm_type: str = "low_precision_layernorm",
47 - use_cache: bool = False,
48 - **kwargs,
49 - ) -> None:
50 - self.d_model = d_model
51 - self.n_heads = n_heads
52 - self.n_layers = n_layers
53 - self.expansion_ratio = expansion_ratio
54 - self.max_seq_len = max_seq_len
55 - self.vocab_size = vocab_size
56 - self.resid_pdrop = resid_pdrop
57 - self.emb_pdrop = emb_pdrop
58 - self.learned_pos_emb = learned_pos_emb
59 - if attn_config is None:
60 - self.attn_config = _ATTN_CONFIG_DEFAULTS
61 - else:
62 - self.attn_config = attn_config
63 - self.init_device = init_device
64 - self.logit_scale = logit_scale
65 - self.no_bias = no_bias
66 - self.verbose = verbose
67 - self.embedding_fraction = embedding_fraction
68 - self.norm_type = norm_type
69 - self.use_cache = use_cache
70 - if "name" in kwargs:
71 - del kwargs["name"]
72 - if "loss_fn" in kwargs:
73 - del kwargs["loss_fn"]
74 - super().__init__(**kwargs)