@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 39161c98a0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Add OPT

Woosuk Kwon committed Feb 9, 2023 at 11:25 UTC 39161c98a054756cb39edb9f634c6a466410c92d
2 files changed +207
cacheflow/worker/models/__init__.py new
+5
@@ -0,0 +1,5 @@
1 +from cacheflow.worker.models.opt import OPTForCausalLM
2 +
3 +__all__ = [
4 + 'OPTForCausalLM',
5 +]
cacheflow/worker/models/opt.py new
+202
@@ -0,0 +1,202 @@
1 +"""1D OPT model compatible with HuggingFace weights."""
2 +import torch
3 +from torch import nn
4 +from transformers import OPTConfig
5 +from transformers import PreTrainedModel
6 +
7 +
8 +class OPTLearnedPositionalEmbedding(nn.Embedding):
9 +
10 + def __init__(self, num_embeddings: int, embedding_dim: int):
11 + # OPT is set up so that if padding_idx is specified then offset the embedding ids by 2
12 + # and adjust num_embeddings appropriately. Other models don't have this hack
13 + self.offset = 2
14 + super().__init__(num_embeddings + self.offset, embedding_dim)
15 +
16 + def forward(self, positions: torch.LongTensor):
17 + return super().forward(positions + self.offset)
18 +
19 +
20 +class OPTAttention(nn.Module):
21 +
22 + def __init__(
23 + self,
24 + embed_dim: int,
25 + num_heads: int,
26 + bias: bool = True,
27 + ) -> None:
28 + super().__init__()
29 + self.embed_dim = embed_dim
30 + self.num_heads = num_heads
31 + self.head_dim = embed_dim // num_heads
32 + self.scaling = self.head_dim**-0.5
33 +
34 + self.k_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
35 + self.v_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
36 + self.q_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
37 + self.out_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
38 +
39 + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
40 + q = self.q_proj(hidden_states) * self.scaling
41 + k = self.k_proj(hidden_states)
42 + v = self.v_proj(hidden_states)
43 + # TODO
44 + attn_output = None
45 + output = self.out_proj(attn_output)
46 + return output
47 +
48 +
49 +class OPTDecoderLayer(nn.Module):
50 +
51 + def __init__(self, config: OPTConfig):
52 + super().__init__()
53 + self.embed_dim = config.hidden_size
54 + self.self_attn = OPTAttention(
55 + embed_dim=self.embed_dim,
56 + num_heads=config.num_attention_heads,
57 + bias=config.enable_bias,
58 + )
59 + self.do_layer_norm_before = config.do_layer_norm_before
60 + assert config.activation_function == 'relu'
61 + self.activation_fn = nn.ReLU()
62 +
63 + self.self_attn_layer_norm = nn.LayerNorm(
64 + self.embed_dim, elementwise_affine=config.layer_norm_elementwise_affine)
65 + self.fc1 = nn.Linear(self.embed_dim, config.ffn_dim, bias=config.enable_bias)
66 + self.fc2 = nn.Linear(config.ffn_dim, self.embed_dim, bias=config.enable_bias)
67 + self.final_layer_norm = nn.LayerNorm(self.embed_dim, elementwise_affine=config.layer_norm_elementwise_affine)
68 +
69 + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
70 + # Self Attention
71 + residual = hidden_states
72 + # 125m, 1.7B, ..., 175B applies layer norm BEFORE attention
73 + if self.do_layer_norm_before:
74 + hidden_states = self.self_attn_layer_norm(hidden_states)
75 + hidden_states = self.self_attn(hidden_states=hidden_states)
76 + hidden_states = residual + hidden_states
77 + # 350m applies layer norm AFTER attention
78 + if not self.do_layer_norm_before:
79 + hidden_states = self.self_attn_layer_norm(hidden_states)
80 +
81 + # Fully Connected
82 + residual = hidden_states
83 + # 125m, 1.7B, ..., 175B applies layer norm BEFORE attention
84 + if self.do_layer_norm_before:
85 + hidden_states = self.final_layer_norm(hidden_states)
86 + hidden_states = self.fc1(hidden_states)
87 + hidden_states = self.activation_fn(hidden_states)
88 + hidden_states = self.fc2(hidden_states)
89 + hidden_states = residual + hidden_states
90 + # 350m applies layer norm AFTER attention
91 + if not self.do_layer_norm_before:
92 + hidden_states = self.final_layer_norm(hidden_states)
93 + return hidden_states
94 +
95 +
96 +class OPTPreTrainedModel(PreTrainedModel):
97 + config_class = OPTConfig
98 + base_model_prefix = "model"
99 + supports_gradient_checkpointing = True
100 + _no_split_modules = ["OPTDecoderLayer"]
101 + _keys_to_ignore_on_load_unexpected = [r"decoder\.version"]
102 +
103 + def _init_weights(self, module) -> None:
104 + del module # unused
105 + return
106 +
107 +
108 +class OPTDecoder(OPTPreTrainedModel):
109 +
110 + def __init__(self, config: OPTConfig):
111 + super().__init__(config)
112 + self.padding_idx = config.pad_token_id
113 + self.max_target_positions = config.max_position_embeddings
114 + self.vocab_size = config.vocab_size
115 +
116 + self.embed_tokens = nn.Embedding(config.vocab_size, config.word_embed_proj_dim, self.padding_idx)
117 + self.embed_positions = OPTLearnedPositionalEmbedding(config.max_position_embeddings, config.hidden_size)
118 +
119 + if config.word_embed_proj_dim != config.hidden_size:
120 + self.project_out = nn.Linear(config.hidden_size, config.word_embed_proj_dim, bias=False)
121 + else:
122 + self.project_out = None
123 +
124 + if config.word_embed_proj_dim != config.hidden_size:
125 + self.project_in = nn.Linear(config.word_embed_proj_dim, config.hidden_size, bias=False)
126 + else:
127 + self.project_in = None
128 +
129 + # Note that the only purpose of `config._remove_final_layer_norm` is to keep backward compatibility
130 + # with checkpoints that have been fine-tuned before transformers v4.20.1
131 + # see https://github.com/facebookresearch/metaseq/pull/164
132 + if config.do_layer_norm_before and not config._remove_final_layer_norm:
133 + self.final_layer_norm = nn.LayerNorm(
134 + config.hidden_size, elementwise_affine=config.layer_norm_elementwise_affine
135 + )
136 + else:
137 + self.final_layer_norm = None
138 +
139 + self.layers = nn.ModuleList([OPTDecoderLayer(config) for _ in range(config.num_hidden_layers)])
140 +
141 + # Initialize weights and apply final processing
142 + self.post_init()
143 +
144 + def forward(
145 + self,
146 + input_ids: torch.LongTensor,
147 + positions: torch.LongTensor,
148 + ) -> torch.Tensor:
149 + inputs_embeds = self.embed_tokens(input_ids)
150 + pos_embeds = self.embed_positions(positions)
151 + pos_embeds = None
152 + if self.project_in is not None:
153 + inputs_embeds = self.project_in(inputs_embeds)
154 + hidden_states = inputs_embeds + pos_embeds
155 +
156 + for layer in self.layers:
157 + hidden_states = layer(hidden_states)
158 +
159 + if self.final_layer_norm is not None:
160 + hidden_states = self.final_layer_norm(hidden_states)
161 + if self.project_out is not None:
162 + hidden_states = self.project_out(hidden_states)
163 + return hidden_states
164 +
165 +
166 +class OPTModel(OPTPreTrainedModel):
167 +
168 + def __init__(self, config: OPTConfig):
169 + super().__init__(config)
170 + self.decoder = OPTDecoder(config)
171 + # Initialize weights and apply final processing
172 + self.post_init()
173 +
174 + def forward(
175 + self,
176 + input_ids: torch.LongTensor,
177 + positions: torch.LongTensor,
178 + ) -> torch.Tensor:
179 + return self.decoder(input_ids, positions)
180 +
181 +
182 +class OPTForCausalLM(OPTPreTrainedModel):
183 + _keys_to_ignore_on_load_missing = [r"lm_head.weight"]
184 +
185 + def __init__(self, config):
186 + super().__init__(config)
187 + self.model = OPTModel(config)
188 +
189 + # the lm_head weight is automatically tied to the embed tokens weight
190 + self.lm_head = nn.Linear(config.word_embed_proj_dim, config.vocab_size, bias=False)
191 +
192 + # Initialize weights and apply final processing
193 + self.post_init()
194 +
195 + def forward(
196 + self,
197 + input_ids: torch.LongTensor,
198 + positions: torch.LongTensor,
199 + ) -> torch.Tensor:
200 + hidden_states = self.model.decoder(input_ids, positions)
201 + logits = self.lm_head(hidden_states).contiguous()
202 + return logits