A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Add OPT
Woosuk Kwon committed
Feb 9, 2023 at 11:25 UTC
39161c98a054756cb39edb9f634c6a466410c92d
2 files changed
+207
cacheflow/worker/models/__init__.py
new
+5
@@ -0,0 +1,5 @@
1
+from cacheflow.worker.models.opt import OPTForCausalLM
2
+
3
+__all__ = [
4
+ 'OPTForCausalLM',
5
+]
cacheflow/worker/models/opt.py
new
+202
@@ -0,0 +1,202 @@
1
+"""1D OPT model compatible with HuggingFace weights."""
2
+import torch
3
+from torch import nn
4
+from transformers import OPTConfig
5
+from transformers import PreTrainedModel
6
+
7
+
8
+class OPTLearnedPositionalEmbedding(nn.Embedding):
9
+
10
+ def __init__(self, num_embeddings: int, embedding_dim: int):
11
+ # OPT is set up so that if padding_idx is specified then offset the embedding ids by 2
12
+ # and adjust num_embeddings appropriately. Other models don't have this hack
13
+ self.offset = 2
14
+ super().__init__(num_embeddings + self.offset, embedding_dim)
15
+
16
+ def forward(self, positions: torch.LongTensor):
17
+ return super().forward(positions + self.offset)
18
+
19
+
20
+class OPTAttention(nn.Module):
21
+
22
+ def __init__(
23
+ self,
24
+ embed_dim: int,
25
+ num_heads: int,
26
+ bias: bool = True,
27
+ ) -> None:
28
+ super().__init__()
29
+ self.embed_dim = embed_dim
30
+ self.num_heads = num_heads
31
+ self.head_dim = embed_dim // num_heads
32
+ self.scaling = self.head_dim**-0.5
33
+
34
+ self.k_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
35
+ self.v_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
36
+ self.q_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
37
+ self.out_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
38
+
39
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
40
+ q = self.q_proj(hidden_states) * self.scaling
41
+ k = self.k_proj(hidden_states)
42
+ v = self.v_proj(hidden_states)
43
+ # TODO
44
+ attn_output = None
45
+ output = self.out_proj(attn_output)
46
+ return output
47
+
48
+
49
+class OPTDecoderLayer(nn.Module):
50
+
51
+ def __init__(self, config: OPTConfig):
52
+ super().__init__()
53
+ self.embed_dim = config.hidden_size
54
+ self.self_attn = OPTAttention(
55
+ embed_dim=self.embed_dim,
56
+ num_heads=config.num_attention_heads,
57
+ bias=config.enable_bias,
58
+ )
59
+ self.do_layer_norm_before = config.do_layer_norm_before
60
+ assert config.activation_function == 'relu'
61
+ self.activation_fn = nn.ReLU()
62
+
63
+ self.self_attn_layer_norm = nn.LayerNorm(
64
+ self.embed_dim, elementwise_affine=config.layer_norm_elementwise_affine)
65
+ self.fc1 = nn.Linear(self.embed_dim, config.ffn_dim, bias=config.enable_bias)
66
+ self.fc2 = nn.Linear(config.ffn_dim, self.embed_dim, bias=config.enable_bias)
67
+ self.final_layer_norm = nn.LayerNorm(self.embed_dim, elementwise_affine=config.layer_norm_elementwise_affine)
68
+
69
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
70
+ # Self Attention
71
+ residual = hidden_states
72
+ # 125m, 1.7B, ..., 175B applies layer norm BEFORE attention
73
+ if self.do_layer_norm_before:
74
+ hidden_states = self.self_attn_layer_norm(hidden_states)
75
+ hidden_states = self.self_attn(hidden_states=hidden_states)
76
+ hidden_states = residual + hidden_states
77
+ # 350m applies layer norm AFTER attention
78
+ if not self.do_layer_norm_before:
79
+ hidden_states = self.self_attn_layer_norm(hidden_states)
80
+
81
+ # Fully Connected
82
+ residual = hidden_states
83
+ # 125m, 1.7B, ..., 175B applies layer norm BEFORE attention
84
+ if self.do_layer_norm_before:
85
+ hidden_states = self.final_layer_norm(hidden_states)
86
+ hidden_states = self.fc1(hidden_states)
87
+ hidden_states = self.activation_fn(hidden_states)
88
+ hidden_states = self.fc2(hidden_states)
89
+ hidden_states = residual + hidden_states
90
+ # 350m applies layer norm AFTER attention
91
+ if not self.do_layer_norm_before:
92
+ hidden_states = self.final_layer_norm(hidden_states)
93
+ return hidden_states
94
+
95
+
96
+class OPTPreTrainedModel(PreTrainedModel):
97
+ config_class = OPTConfig
98
+ base_model_prefix = "model"
99
+ supports_gradient_checkpointing = True
100
+ _no_split_modules = ["OPTDecoderLayer"]
101
+ _keys_to_ignore_on_load_unexpected = [r"decoder\.version"]
102
+
103
+ def _init_weights(self, module) -> None:
104
+ del module # unused
105
+ return
106
+
107
+
108
+class OPTDecoder(OPTPreTrainedModel):
109
+
110
+ def __init__(self, config: OPTConfig):
111
+ super().__init__(config)
112
+ self.padding_idx = config.pad_token_id
113
+ self.max_target_positions = config.max_position_embeddings
114
+ self.vocab_size = config.vocab_size
115
+
116
+ self.embed_tokens = nn.Embedding(config.vocab_size, config.word_embed_proj_dim, self.padding_idx)
117
+ self.embed_positions = OPTLearnedPositionalEmbedding(config.max_position_embeddings, config.hidden_size)
118
+
119
+ if config.word_embed_proj_dim != config.hidden_size:
120
+ self.project_out = nn.Linear(config.hidden_size, config.word_embed_proj_dim, bias=False)
121
+ else:
122
+ self.project_out = None
123
+
124
+ if config.word_embed_proj_dim != config.hidden_size:
125
+ self.project_in = nn.Linear(config.word_embed_proj_dim, config.hidden_size, bias=False)
126
+ else:
127
+ self.project_in = None
128
+
129
+ # Note that the only purpose of `config._remove_final_layer_norm` is to keep backward compatibility
130
+ # with checkpoints that have been fine-tuned before transformers v4.20.1
131
+ # see https://github.com/facebookresearch/metaseq/pull/164
132
+ if config.do_layer_norm_before and not config._remove_final_layer_norm:
133
+ self.final_layer_norm = nn.LayerNorm(
134
+ config.hidden_size, elementwise_affine=config.layer_norm_elementwise_affine
135
+ )
136
+ else:
137
+ self.final_layer_norm = None
138
+
139
+ self.layers = nn.ModuleList([OPTDecoderLayer(config) for _ in range(config.num_hidden_layers)])
140
+
141
+ # Initialize weights and apply final processing
142
+ self.post_init()
143
+
144
+ def forward(
145
+ self,
146
+ input_ids: torch.LongTensor,
147
+ positions: torch.LongTensor,
148
+ ) -> torch.Tensor:
149
+ inputs_embeds = self.embed_tokens(input_ids)
150
+ pos_embeds = self.embed_positions(positions)
151
+ pos_embeds = None
152
+ if self.project_in is not None:
153
+ inputs_embeds = self.project_in(inputs_embeds)
154
+ hidden_states = inputs_embeds + pos_embeds
155
+
156
+ for layer in self.layers:
157
+ hidden_states = layer(hidden_states)
158
+
159
+ if self.final_layer_norm is not None:
160
+ hidden_states = self.final_layer_norm(hidden_states)
161
+ if self.project_out is not None:
162
+ hidden_states = self.project_out(hidden_states)
163
+ return hidden_states
164
+
165
+
166
+class OPTModel(OPTPreTrainedModel):
167
+
168
+ def __init__(self, config: OPTConfig):
169
+ super().__init__(config)
170
+ self.decoder = OPTDecoder(config)
171
+ # Initialize weights and apply final processing
172
+ self.post_init()
173
+
174
+ def forward(
175
+ self,
176
+ input_ids: torch.LongTensor,
177
+ positions: torch.LongTensor,
178
+ ) -> torch.Tensor:
179
+ return self.decoder(input_ids, positions)
180
+
181
+
182
+class OPTForCausalLM(OPTPreTrainedModel):
183
+ _keys_to_ignore_on_load_missing = [r"lm_head.weight"]
184
+
185
+ def __init__(self, config):
186
+ super().__init__(config)
187
+ self.model = OPTModel(config)
188
+
189
+ # the lm_head weight is automatically tied to the embed tokens weight
190
+ self.lm_head = nn.Linear(config.word_embed_proj_dim, config.vocab_size, bias=False)
191
+
192
+ # Initialize weights and apply final processing
193
+ self.post_init()
194
+
195
+ def forward(
196
+ self,
197
+ input_ids: torch.LongTensor,
198
+ positions: torch.LongTensor,
199
+ ) -> torch.Tensor:
200
+ hidden_states = self.model.decoder(input_ids, positions)
201
+ logits = self.lm_head(hidden_states).contiguous()
202
+ return logits