@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4c07dd28c0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[🚀 Ready to be merged] Added support for Jais models (#3183)

Lalit Pradhan committed Mar 21, 2024 at 13:45 UTC 4c07dd28c0ef8642735222e077935b55f4c98017
8 files changed +596 -3
README.md
+1
@@ -76,6 +76,7 @@ vLLM seamlessly supports many Hugging Face models, including the following archi
76 - GPT-NeoX (`EleutherAI/gpt-neox-20b`, `databricks/dolly-v2-12b`, `stabilityai/stablelm-tuned-alpha-7b`, etc.)
77 - InternLM (`internlm/internlm-7b`, `internlm/internlm-chat-7b`, etc.)
78 - InternLM2 (`internlm/internlm2-7b`, `internlm/internlm2-chat-7b`, etc.)
79 +- Jais (`core42/jais-13b`, `core42/jais-13b-chat`, `core42/jais-30b-v3`, `core42/jais-30b-chat-v3`, etc.)
80 - LLaMA & LLaMA-2 (`meta-llama/Llama-2-70b-hf`, `lmsys/vicuna-13b-v1.3`, `young-geng/koala`, `openlm-research/open_llama_13b`, etc.)
81 - Mistral (`mistralai/Mistral-7B-v0.1`, `mistralai/Mistral-7B-Instruct-v0.1`, etc.)
82 - Mixtral (`mistralai/Mixtral-8x7B-v0.1`, `mistralai/Mixtral-8x7B-Instruct-v0.1`, etc.)
docs/source/models/supported_models.rst
+5 -1
@@ -66,7 +66,11 @@ Alongside each architecture, we include some popular models that use it.
66 * - :code:`InternLM2ForCausalLM`
67 - InternLM2
68 - :code:`internlm/internlm2-7b`, :code:`internlm/internlm2-chat-7b`, etc.
69 - -
69 + -
70 + * - :code:`JAISLMHeadModel`
71 + - Jais
72 + - :code:`core42/jais-13b`, :code:`core42/jais-13b-chat`, :code:`core42/jais-30b-v3`, :code:`core42/jais-30b-chat-v3`, etc.
73 + -
74 * - :code:`LlamaForCausalLM`
75 - LLaMA, LLaMA-2, Vicuna, Alpaca, Yi
76 - :code:`meta-llama/Llama-2-13b-hf`, :code:`meta-llama/Llama-2-70b-hf`, :code:`openlm-research/open_llama_13b`, :code:`lmsys/vicuna-13b-v1.3`, :code:`01-ai/Yi-6B`, :code:`01-ai/Yi-34B`, etc.
vllm/model_executor/models/__init__.py
+1
@@ -27,6 +27,7 @@ _MODELS = {
27 "GPTNeoXForCausalLM": ("gpt_neox", "GPTNeoXForCausalLM"),
28 "InternLMForCausalLM": ("llama", "LlamaForCausalLM"),
29 "InternLM2ForCausalLM": ("internlm2", "InternLM2ForCausalLM"),
30 + "JAISLMHeadModel": ("jais", "JAISLMHeadModel"),
31 "LlamaForCausalLM": ("llama", "LlamaForCausalLM"),
32 # For decapoda-research/llama-*
33 "LLaMAForCausalLM": ("llama", "LlamaForCausalLM"),
vllm/model_executor/models/gpt2.py
+1 -2
@@ -242,8 +242,7 @@ class GPT2LMHeadModel(nn.Module):
242 logits: torch.Tensor,
243 sampling_metadata: SamplingMetadata,
244 ) -> Optional[SamplerOutput]:
245 - next_tokens = self.sampler(self.lm_head_weight, logits,
246 - sampling_metadata)
245 + next_tokens = self.sampler(logits, sampling_metadata)
246 return next_tokens
247
248 def load_weights(self,
vllm/model_executor/models/jais.py new
+351
@@ -0,0 +1,351 @@
1 +# coding=utf-8
2 +# Adapted from
3 +# https://huggingface.co/core42/jais-30b-chat-v3/blob/main/modeling_jais.py
4 +# Copyright 2023 The vLLM team.
5 +# Copyright 2023 the Jais authors and HuggingFace Inc. team. All rights
6 +# reserved.
7 +# Copyright 2023 Cerebras Systems.
8 +#
9 +# Licensed under the Apache License, Version 2.0 (the "License");
10 +# you may not use this file except in compliance with the License.
11 +# You may obtain a copy of the License at
12 +#
13 +# http://www.apache.org/licenses/LICENSE-2.0
14 +#
15 +# Unless required by applicable law or agreed to in writing, software
16 +# distributed under the License is distributed on an "AS IS" BASIS,
17 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18 +# See the License for the specific language governing permissions and
19 +# limitations under the License.
20 +"""Inference-only Jais model compatible with HuggingFace weights."""
21 +
22 +import math
23 +from typing import List, Optional, Tuple
24 +
25 +import torch
26 +from torch import nn
27 +from vllm.transformers_utils.configs import JAISConfig
28 +
29 +from vllm.model_executor.input_metadata import InputMetadata
30 +from vllm.model_executor.layers.attention import Attention
31 +from vllm.model_executor.layers.linear import (
32 + ColumnParallelLinear,
33 + LinearMethodBase,
34 + QKVParallelLinear,
35 + RowParallelLinear,
36 +)
37 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
38 +from vllm.model_executor.layers.sampler import Sampler
39 +from vllm.model_executor.layers.vocab_parallel_embedding import (
40 + VocabParallelEmbedding, )
41 +from vllm.model_executor.parallel_utils.parallel_state import (
42 + get_tensor_model_parallel_world_size,
43 + get_tensor_model_parallel_rank,
44 +)
45 +from vllm.model_executor.weight_utils import (
46 + default_weight_loader,
47 + hf_model_weights_iterator,
48 +)
49 +from vllm.sequence import SamplerOutput
50 +from vllm.model_executor.sampling_metadata import SamplingMetadata
51 +
52 +KVCache = Tuple[torch.Tensor, torch.Tensor]
53 +
54 +
55 +class SwiGLUActivation(nn.Module):
56 +
57 + def forward(self, x1: torch.Tensor, x2: torch.Tensor) -> torch.Tensor:
58 + return x1 * nn.functional.silu(x2)
59 +
60 +
61 +def _get_alibi_slopes(n):
62 +
63 + def get_slopes_power_of_2(n):
64 + start = 2**(-(2**-(math.log2(n) - 3)))
65 + ratio = start
66 + return [start * ratio**i for i in range(n)]
67 +
68 + if math.log2(n).is_integer():
69 + return get_slopes_power_of_2(n)
70 + else:
71 + closest_power_of_2 = 2**math.floor(math.log2(n))
72 + return (get_slopes_power_of_2(closest_power_of_2) + _get_alibi_slopes(
73 + 2 * closest_power_of_2)[0::2][:n - closest_power_of_2])
74 +
75 +
76 +class JAISAttention(nn.Module):
77 +
78 + def __init__(
79 + self,
80 + config: JAISConfig,
81 + linear_method: Optional[LinearMethodBase] = None,
82 + ):
83 + super().__init__()
84 + self.hidden_size = config.hidden_size
85 + total_num_heads = config.num_attention_heads
86 + tensor_model_parallel_world_size = (
87 + get_tensor_model_parallel_world_size())
88 + assert total_num_heads % tensor_model_parallel_world_size == 0
89 + self.num_heads = total_num_heads // tensor_model_parallel_world_size
90 + self.head_dim = self.hidden_size // total_num_heads
91 + if hasattr(config, "scale_qk_dot_by_d"):
92 + config.mup_scale_qk_dot_by_d = config.scale_qk_dot_by_d
93 + self.attn_scale_power = 1.0 if config.mup_scale_qk_dot_by_d else 0.5
94 + self.scale = self.head_dim**-self.attn_scale_power
95 +
96 + self.c_attn = QKVParallelLinear(
97 + self.hidden_size,
98 + self.head_dim,
99 + total_num_heads,
100 + bias=True,
101 + linear_method=linear_method,
102 + )
103 + self.c_proj = RowParallelLinear(
104 + self.hidden_size,
105 + self.hidden_size,
106 + bias=True,
107 + linear_method=linear_method,
108 + )
109 +
110 + tp_rank = get_tensor_model_parallel_rank()
111 + head_start = tp_rank * self.num_heads
112 + head_end = (tp_rank + 1) * self.num_heads
113 + alibi_slopes = _get_alibi_slopes(total_num_heads)
114 + alibi_slopes = alibi_slopes[head_start:head_end]
115 + self.attn = Attention(
116 + self.num_heads,
117 + self.head_dim,
118 + scale=self.scale,
119 + alibi_slopes=alibi_slopes,
120 + )
121 +
122 + def forward(
123 + self,
124 + hidden_states: torch.Tensor,
125 + kv_cache: KVCache,
126 + input_metadata: InputMetadata,
127 + ) -> torch.Tensor:
128 + qkv, _ = self.c_attn(hidden_states)
129 + q, k, v = qkv.chunk(chunks=3, dim=-1)
130 + key_cache, value_cache = kv_cache
131 + attn_output = self.attn(q, k, v, key_cache, value_cache,
132 + input_metadata)
133 + attn_output, _ = self.c_proj(attn_output)
134 + return attn_output
135 +
136 +
137 +class JAISMLP(nn.Module):
138 +
139 + def __init__(
140 + self,
141 + intermediate_size: int,
142 + config: JAISConfig,
143 + linear_method: Optional[LinearMethodBase] = None,
144 + ):
145 + super().__init__()
146 + hidden_size = config.hidden_size
147 + self.swiglu = config.activation_function == "swiglu"
148 + self.c_fc = ColumnParallelLinear(
149 + hidden_size,
150 + intermediate_size,
151 + bias=True,
152 + linear_method=linear_method,
153 + )
154 + self.c_fc2 = (ColumnParallelLinear(
155 + hidden_size,
156 + intermediate_size,
157 + bias=True,
158 + linear_method=linear_method,
159 + ) if self.swiglu else None)
160 + self.c_proj = RowParallelLinear(
161 + intermediate_size,
162 + hidden_size,
163 + bias=True,
164 + linear_method=linear_method,
165 + )
166 +
167 + self.act = SwiGLUActivation()
168 +
169 + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
170 + if self.swiglu:
171 + hidden_states2, _ = self.c_fc2(hidden_states)
172 + hidden_states, _ = self.c_fc(hidden_states)
173 + hidden_states = (self.act(hidden_states, hidden_states2)
174 + if self.swiglu else self.act(hidden_states))
175 + hidden_states, _ = self.c_proj(hidden_states)
176 + return hidden_states
177 +
178 +
179 +class JAISBlock(nn.Module):
180 +
181 + def __init__(
182 + self,
183 + config: JAISConfig,
184 + linear_method: Optional[LinearMethodBase] = None,
185 + ):
186 + super().__init__()
187 + hidden_size = config.hidden_size
188 + inner_dim = (config.n_inner if config.n_inner is not None else 4 *
189 + hidden_size)
190 +
191 + self.ln_1 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
192 + self.attn = JAISAttention(config, linear_method)
193 + self.ln_2 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
194 + self.mlp = JAISMLP(inner_dim, config, linear_method)
195 +
196 + def forward(
197 + self,
198 + hidden_states: torch.Tensor,
199 + kv_cache: KVCache,
200 + input_metadata: InputMetadata,
201 + ) -> torch.Tensor:
202 + residual = hidden_states
203 + hidden_states = self.ln_1(hidden_states)
204 + attn_output = self.attn(
205 + hidden_states=hidden_states,
206 + kv_cache=kv_cache,
207 + input_metadata=input_metadata,
208 + )
209 + # residual connection
210 + hidden_states = attn_output + residual
211 +
212 + residual = hidden_states
213 + hidden_states = self.ln_2(hidden_states)
214 + feed_forward_hidden_states = self.mlp(hidden_states)
215 + # residual connection
216 + hidden_states = residual + feed_forward_hidden_states
217 + return hidden_states
218 +
219 +
220 +class JAISModel(nn.Module):
221 +
222 + def __init__(
223 + self,
224 + config: JAISConfig,
225 + linear_method: Optional[LinearMethodBase] = None,
226 + ):
227 + super().__init__()
228 + self.config = config
229 + assert not config.add_cross_attention
230 + assert not config.scale_attn_by_inverse_layer_idx
231 + assert not config.reorder_and_upcast_attn
232 + self.embed_dim = config.hidden_size
233 + self.wte = VocabParallelEmbedding(config.vocab_size, self.embed_dim)
234 + self.wpe = (nn.Embedding(config.max_position_embeddings,
235 + self.embed_dim)
236 + if config.position_embedding_type != "alibi" else None)
237 + if hasattr(config, "embeddings_scale"):
238 + self.embeddings_scale = config.embeddings_scale
239 + else:
240 + self.embeddings_scale = config.mup_embeddings_scale
241 + self.h = nn.ModuleList([
242 + JAISBlock(config, linear_method)
243 + for _ in range(config.num_hidden_layers)
244 + ])
245 + self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
246 +
247 + def forward(
248 + self,
249 + input_ids: torch.Tensor,
250 + position_ids: torch.Tensor,
251 + kv_caches: List[KVCache],
252 + input_metadata: InputMetadata,
253 + ) -> torch.Tensor:
254 + inputs_embeds = self.wte(input_ids)
255 + if self.wpe is not None:
256 + position_embeds = self.wpe(position_ids)
257 + hidden_states = inputs_embeds + position_embeds
258 + else:
259 + hidden_states = inputs_embeds
260 + hidden_states *= torch.tensor(float(self.embeddings_scale),
261 + dtype=hidden_states.dtype)
262 +
263 + for i in range(len(self.h)):
264 + layer = self.h[i]
265 + hidden_states = layer(hidden_states, kv_caches[i], input_metadata)
266 +
267 + hidden_states = self.ln_f(hidden_states)
268 + return hidden_states
269 +
270 +
271 +class JAISLMHeadModel(nn.Module):
272 +
273 + def __init__(
274 + self,
275 + config: JAISConfig,
276 + linear_method: Optional[LinearMethodBase] = None,
277 + ):
278 + super().__init__()
279 + self.config = config
280 + self.linear_method = linear_method
281 + self.transformer = JAISModel(config, linear_method)
282 + self.lm_head_weight = self.transformer.wte.weight
283 + if hasattr(config, "width_scale"):
284 + self.output_logits_scale = config.width_scale
285 + else:
286 + self.output_logits_scale = (config.mup_output_alpha *
287 + config.mup_width_scale)
288 + self.logits_processor = LogitsProcessor(vocab_size=config.vocab_size,
289 + scale=self.output_logits_scale)
290 + self.sampler = Sampler()
291 +
292 + def forward(
293 + self,
294 + input_ids: torch.Tensor,
295 + positions: torch.Tensor,
296 + kv_caches: List[KVCache],
297 + input_metadata: InputMetadata,
298 + ) -> torch.Tensor:
299 + hidden_states = self.transformer(input_ids, positions, kv_caches,
300 + input_metadata)
301 + return hidden_states
302 +
303 + def compute_logits(self, hidden_states: torch.Tensor,
304 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
305 + logits = self.logits_processor(self.lm_head_weight, hidden_states,
306 + sampling_metadata)
307 + return logits
308 +
309 + def sample(
310 + self,
311 + logits: torch.Tensor,
312 + sampling_metadata: SamplingMetadata,
313 + ) -> Optional[SamplerOutput]:
314 + next_tokens = self.sampler(logits, sampling_metadata)
315 + return next_tokens
316 +
317 + def load_weights(
318 + self,
319 + model_name_or_path: str,
320 + cache_dir: Optional[str] = None,
321 + load_format: str = "auto",
322 + revision: Optional[str] = None,
323 + ):
324 + params_dict = dict(self.named_parameters(remove_duplicate=False))
325 + for name, loaded_weight in hf_model_weights_iterator(
326 + model_name_or_path, cache_dir, load_format, revision):
327 + if "lm_head.weight" in name:
328 + # GPT-2 ties the weights of the embedding layer and the final
329 + # linear layer.
330 + continue
331 + if ".attn.bias" in name or ".attn.masked_bias" in name:
332 + # Skip attention mask.
333 + # NOTE: "c_attn.bias" should not be skipped.
334 + continue
335 + if "relative_pe" in name:
336 + continue
337 + if not name.startswith("transformer."):
338 + name = "transformer." + name
339 + param = params_dict[name]
340 + # The HF's GPT-2 implementation uses Conv1D instead of Linear.
341 + # Because of this, we need to transpose the weights.
342 + # Note(zhuohan): the logic below might break quantized models.
343 + for conv1d_weight_name in ["c_attn", "c_proj", "c_fc"]:
344 + if conv1d_weight_name not in name:
345 + continue
346 + if not name.endswith(".weight"):
347 + continue
348 + loaded_weight = loaded_weight.t()
349 + weight_loader = getattr(param, "weight_loader",
350 + default_weight_loader)
351 + weight_loader(param, loaded_weight)
\ No newline at end of file
vllm/transformers_utils/config.py
+1
@@ -10,6 +10,7 @@ _CONFIG_REGISTRY = {
10 "RefinedWeb": RWConfig, # For tiiuae/falcon-40b(-instruct)
11 "RefinedWebModel": RWConfig, # For tiiuae/falcon-7b(-instruct)
12 "starcoder2": Starcoder2Config,
13 + "jais": JAISConfig,
14 }
15
16
vllm/transformers_utils/configs/__init__.py
+2
@@ -5,10 +5,12 @@ from vllm.transformers_utils.configs.mpt import MPTConfig
5 # `FalconConfig` class from the official HuggingFace transformers library.
6 from vllm.transformers_utils.configs.falcon import RWConfig
7 from vllm.transformers_utils.configs.starcoder2 import Starcoder2Config
8 +from vllm.transformers_utils.configs.jais import JAISConfig
9
10 __all__ = [
11 "ChatGLMConfig",
12 "MPTConfig",
13 "RWConfig",
14 "Starcoder2Config",
15 + "JAISConfig",
16 ]
vllm/transformers_utils/configs/jais.py new
+234
@@ -0,0 +1,234 @@
1 +# coding=utf-8
2 +# Copyright 2023 The OpenAI Team Authors and HuggingFace Inc. team.
3 +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved.
4 +# Copyright 2023 Cerebras Systems.
5 +#
6 +# Licensed under the Apache License, Version 2.0 (the "License");
7 +# you may not use this file except in compliance with the License.
8 +# You may obtain a copy of the License at
9 +#
10 +# http://www.apache.org/licenses/LICENSE-2.0
11 +#
12 +# Unless required by applicable law or agreed to in writing, software
13 +# distributed under the License is distributed on an "AS IS" BASIS,
14 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 +# See the License for the specific language governing permissions and
16 +# limitations under the License.
17 +"""JAIS configuration"""
18 +
19 +from transformers.configuration_utils import PretrainedConfig
20 +from transformers.utils import logging
21 +
22 +logger = logging.get_logger(__name__)
23 +
24 +
25 +class JAISConfig(PretrainedConfig):
26 + """
27 + This is the configuration class to store the configuration of a
28 + [`JAISModel`]. It is used to instantiate a JAIS model according to the
29 + specified arguments, defining the model architecture.
30 +
31 + Configuration objects inherit from [`PretrainedConfig`] and can be used
32 + to control the model outputs. Read the documentation from
33 + [`PretrainedConfig`] for more information.
34 +
35 +
36 + Args:
37 + vocab_size (`int`, *optional*, defaults to 50257):
38 + Vocabulary size of the JAIS model. Defines the number of different
39 + tokens that can be represented by the
40 + `inputs_ids` passed when calling [`JAISModel`].
41 + n_positions (`int`, *optional*, defaults to 1024):
42 + The maximum sequence length that this model might ever be used
43 + with. Typically set this to something large just in case
44 + (e.g., 512 or 1024 or 2048).
45 + n_embd (`int`, *optional*, defaults to 768):
46 + Dimensionality of the embeddings and hidden states.
47 + n_layer (`int`, *optional*, defaults to 12):
48 + Number of hidden layers in the Transformer encoder.
49 + n_head (`int`, *optional*, defaults to 12):
50 + Number of attention heads for each attention layer in the
51 + Transformer encoder.
52 + n_inner (`int`, *optional*, defaults to None):
53 + Dimensionality of the inner feed-forward layers. `None` will set
54 + it to 4 times n_embd
55 + activation_function (`str`, *optional*, defaults to `"gelu"`):
56 + Activation function, to be selected in the list
57 + `["relu", "silu", "gelu", "tanh", "gelu_new", "swiglu"]`.
58 + resid_pdrop (`float`, *optional*, defaults to 0.1):
59 + The dropout probability for all fully connected layers in
60 + the embeddings, encoder, and pooler.
61 + embd_pdrop (`float`, *optional*, defaults to 0.1):
62 + The dropout ratio for the embeddings.
63 + attn_pdrop (`float`, *optional*, defaults to 0.1):
64 + The dropout ratio for the attention.
65 + layer_norm_epsilon (`float`, *optional*, defaults to 1e-5):
66 + The epsilon to use in the layer normalization layers.
67 + initializer_range (`float`, *optional*, defaults to 0.02):
68 + The standard deviation of the truncated_normal_initializer for
69 + initializing all weight matrices.
70 + scale_attn_weights (`bool`, *optional*, defaults to `True`):
71 + Scale attention weights by dividing by sqrt(hidden_size)..
72 + use_cache (`bool`, *optional*, defaults to `True`):
73 + Whether or not the model should return the last key/values
74 + attentions (not used by all models).
75 + scale_attn_by_inverse_layer_idx (`bool`, *optional*,
76 + defaults to `False`):
77 + Whether to additionally scale attention weights by
78 + `1 / layer_idx + 1`.
79 + reorder_and_upcast_attn (`bool`, *optional*, defaults to `False`):
80 + Whether to scale keys (K) prior to computing attention
81 + (dot-product)
82 + and upcast attention dot-product/softmax to float() when training
83 + with mixed precision.
84 + position_embedding_type (`str`, *optional*, defaults to `"learned"`):
85 + Positional embedding can be either `"alibi"` or `"learned"`.
86 + mup_width_scale (`float`, *optional*, defaults to 1.0):
87 + muP parameter to scale learning rate and initializers. Calculated
88 + as (`d_model,0 / d_model`), where
89 + `d_model` is the model's width and `d_model,0` is the proxy
90 + model's width.
91 + mup_embeddings_scale (`float`, *optional*, defaults to 1.0):
92 + muP parameter to scale token and position embeddings.
93 + mup_output_alpha (`float`, *optional*, defaults to 1.0):
94 + muP parameter to scale output logits
95 + (`output_logits_scale = mup_output_alpha * mup_width_scale`).
96 + mup_scale_qk_dot_by_d (`bool`, *optional*, defaults to `False`):
97 + Scale attention weights by dividing by hidden_size instead of
98 + sqrt(hidden_size). Need to set scale_attn_weights to `True` as
99 + well.
100 + alibi_scaling (`Dict`, *optional*):
101 + Dictionary containing the scaling configuration for ALiBi
102 + embeddings. Currently only supports linear
103 + scaling strategy. Can specify either the scaling `factor` (must be
104 + a float greater than 1) for fixed scaling
105 + or `train_seq_len` for dynamic scaling on input samples with
106 + sequence length > `train_seq_len`. The expected
107 + formats are `{"type": strategy name, "factor": scaling factor}` or
108 + `{"type": strategy name,
109 + "train_seq_len": training sequence length}`.
110 + architectures (`List`, *optional*, defaults to ['JAISLMHeadModel']):
111 + architecture names for Jais.
112 +
113 + Example:
114 +
115 + ```python
116 + >>> from transformers import JAISConfig, JAISModel
117 +
118 + >>> # Initializing a JAIS configuration
119 + >>> configuration = JAISConfig()
120 +
121 + >>> # Initializing a model (with random weights) from the configuration
122 + >>> model = JAISModel(configuration)
123 +
124 + >>> # Accessing the model configuration
125 + >>> configuration = model.config
126 + ```"""
127 +
128 + model_type = "jais"
129 + keys_to_ignore_at_inference = ["past_key_values"]
130 + attribute_map = {
131 + "hidden_size": "n_embd",
132 + "max_position_embeddings": "n_positions",
133 + "num_attention_heads": "n_head",
134 + "num_hidden_layers": "n_layer",
135 + }
136 +
137 + def __init__(
138 + self,
139 + vocab_size=50257,
140 + n_positions=1024,
141 + n_embd=768,
142 + n_layer=12,
143 + n_head=12,
144 + n_inner=None,
145 + activation_function="gelu_new",
146 + resid_pdrop=0.1,
147 + embd_pdrop=0.1,
148 + attn_pdrop=0.1,
149 + layer_norm_epsilon=1e-5,
150 + initializer_range=0.02,
151 + scale_attn_weights=True,
152 + use_cache=True,
153 + bos_token_id=50256,
154 + eos_token_id=50256,
155 + scale_attn_by_inverse_layer_idx=False,
156 + reorder_and_upcast_attn=False,
157 + position_embedding_type="learned",
158 + mup_width_scale=1.0,
159 + mup_embeddings_scale=1.0,
160 + mup_output_alpha=1.0,
161 + mup_scale_qk_dot_by_d=False,
162 + alibi_scaling=None,
163 + architectures=None,
164 + **kwargs,
165 + ):
166 + self.vocab_size = vocab_size
167 + self.n_positions = n_positions
168 + self.n_embd = n_embd
169 + self.n_layer = n_layer
170 + self.n_head = n_head
171 + self.n_inner = n_inner
172 + self.activation_function = activation_function
173 + self.resid_pdrop = resid_pdrop
174 + self.embd_pdrop = embd_pdrop
175 + self.attn_pdrop = attn_pdrop
176 + self.layer_norm_epsilon = layer_norm_epsilon
177 + self.initializer_range = initializer_range
178 + self.scale_attn_weights = scale_attn_weights
179 + self.use_cache = use_cache
180 + self.scale_attn_by_inverse_layer_idx = scale_attn_by_inverse_layer_idx
181 + self.reorder_and_upcast_attn = reorder_and_upcast_attn
182 +
183 + self.bos_token_id = bos_token_id
184 + self.eos_token_id = eos_token_id
185 +
186 + self.position_embedding_type = position_embedding_type
187 + self.mup_width_scale = mup_width_scale
188 + self.mup_embeddings_scale = mup_embeddings_scale
189 + self.mup_output_alpha = mup_output_alpha
190 + self.mup_scale_qk_dot_by_d = mup_scale_qk_dot_by_d
191 +
192 + self.alibi_scaling = alibi_scaling
193 + self._alibi_scaling_validation()
194 + if architectures is None:
195 + architectures = ["JAISLMHeadModel"]
196 +
197 + super().__init__(
198 + bos_token_id=bos_token_id,
199 + eos_token_id=eos_token_id,
200 + architectures=architectures,
201 + **kwargs,
202 + )
203 +
204 + def _alibi_scaling_validation(self):
205 + """
206 + Validate the `alibi_scaling` configuration.
207 + """
208 + if self.alibi_scaling is None:
209 + return
210 +
211 + if (not isinstance(self.alibi_scaling, dict)
212 + or len(self.alibi_scaling) != 2):
213 + raise ValueError(
214 + "`alibi_scaling` must be a dictionary with two fields,"
215 + "`type` and `factor` or `type` and `train_seq_len`, "
216 + f"got {self.alibi_scaling}")
217 + alibi_scaling_type = self.alibi_scaling.get("type", None)
218 + alibi_scaling_factor = self.alibi_scaling.get("factor", None)
219 + alibi_dynamic_scaling = self.alibi_scaling.get("train_seq_len", None)
220 + if alibi_scaling_type is None or alibi_scaling_type != "linear":
221 + raise ValueError(f"`alibi_scaling`'s type field must be 'linear',"
222 + f"got {alibi_scaling_type}")
223 + if (alibi_scaling_factor is not None
224 + and not isinstance(alibi_scaling_factor, float)
225 + or alibi_scaling_factor <= 1.0):
226 + raise ValueError(
227 + f"`alibi_scaling`'s factor field must be a float > 1.0,"
228 + f"got {alibi_scaling_factor}")
229 + if (alibi_dynamic_scaling is not None
230 + and not isinstance(alibi_dynamic_scaling, int)
231 + or alibi_dynamic_scaling <= 1):
232 + raise ValueError(
233 + f"`alibi_scaling`'s `train_seq_len` field must be an"
234 + f"integer > 1, got {alibi_dynamic_scaling}")