@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e24336b5a7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[Model] Add support for DBRX (#3660)

Megha Agarwal committed Mar 27, 2024 at 13:01 UTC e24336b5a772ab3aa6ad83527b880f9e5050ea2a
9 files changed +713
README.md
+1
@@ -67,6 +67,7 @@ vLLM seamlessly supports many Hugging Face models, including the following archi
67 - Baichuan & Baichuan2 (`baichuan-inc/Baichuan2-13B-Chat`, `baichuan-inc/Baichuan-7B`, etc.)
68 - BLOOM (`bigscience/bloom`, `bigscience/bloomz`, etc.)
69 - ChatGLM (`THUDM/chatglm2-6b`, `THUDM/chatglm3-6b`, etc.)
70 +- DBRX (`databricks/dbrx-base`, `databricks/dbrx-instruct` etc.)
71 - DeciLM (`Deci/DeciLM-7B`, `Deci/DeciLM-7B-instruct`, etc.)
72 - Falcon (`tiiuae/falcon-7b`, `tiiuae/falcon-40b`, `tiiuae/falcon-rw-7b`, etc.)
73 - Gemma (`google/gemma-2b`, `google/gemma-7b`, etc.)
docs/source/models/supported_models.rst
+4
@@ -27,6 +27,10 @@ Alongside each architecture, we include some popular models that use it.
27 - ChatGLM
28 - :code:`THUDM/chatglm2-6b`, :code:`THUDM/chatglm3-6b`, etc.
29 - ✅︎
30 + * - :code:`DbrxForCausalLM`
31 + - DBRX
32 + - :code:`databricks/dbrx-base`, :code:`databricks/dbrx-instruct`, etc.
33 + -
34 * - :code:`DeciLMForCausalLM`
35 - DeciLM
36 - :code:`Deci/DeciLM-7B`, :code:`Deci/DeciLM-7B-instruct`, etc.
requirements.txt
+1
@@ -14,3 +14,4 @@ prometheus_client >= 0.18.0
14 pynvml == 11.5.0
15 triton >= 2.1.0
16 outlines == 0.0.34
17 +tiktoken == 0.6.0 # Required for DBRX tokenizer
vllm/config.py
+5
@@ -277,6 +277,11 @@ class ModelConfig:
277 # Currently, tensor parallelism is not supported in this case.
278 return 1
279
280 + # For DBRX and MPT
281 + if self.hf_config.model_type in ["dbrx", "mpt"]:
282 + return getattr(self.hf_config.attn_config, "kv_n_heads",
283 + self.hf_config.num_attention_heads)
284 +
285 attributes = [
286 # For Falcon:
287 "n_head_kv",
vllm/model_executor/models/__init__.py
+1
@@ -17,6 +17,7 @@ _MODELS = {
17 "BloomForCausalLM": ("bloom", "BloomForCausalLM"),
18 "ChatGLMModel": ("chatglm", "ChatGLMForCausalLM"),
19 "ChatGLMForConditionalGeneration": ("chatglm", "ChatGLMForCausalLM"),
20 + "DbrxForCausalLM": ("dbrx", "DbrxForCausalLM"),
21 "DeciLMForCausalLM": ("decilm", "DeciLMForCausalLM"),
22 "DeepseekForCausalLM": ("deepseek", "DeepseekForCausalLM"),
23 "FalconForCausalLM": ("falcon", "FalconForCausalLM"),
vllm/model_executor/models/dbrx.py new
+421
@@ -0,0 +1,421 @@
1 +# coding=utf-8
2 +from typing import List, Optional
3 +
4 +import torch
5 +import torch.nn as nn
6 +
7 +from vllm.attention import Attention, AttentionMetadata
8 +from vllm.model_executor.layers.fused_moe import fused_moe
9 +from vllm.model_executor.layers.linear import (LinearMethodBase,
10 + QKVParallelLinear,
11 + ReplicatedLinear,
12 + RowParallelLinear)
13 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
14 +from vllm.model_executor.layers.rotary_embedding import get_rope
15 +from vllm.model_executor.layers.sampler import Sampler
16 +from vllm.model_executor.layers.vocab_parallel_embedding import (
17 + DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
18 +from vllm.model_executor.parallel_utils.communication_op import (
19 + tensor_model_parallel_all_reduce)
20 +from vllm.model_executor.parallel_utils.parallel_state import (
21 + get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
22 +from vllm.model_executor.sampling_metadata import SamplingMetadata
23 +from vllm.model_executor.utils import set_weight_attrs
24 +from vllm.model_executor.weight_utils import (default_weight_loader,
25 + hf_model_weights_iterator)
26 +from vllm.sequence import SamplerOutput
27 +from vllm.transformers_utils.configs.dbrx import DbrxConfig
28 +
29 +
30 +class DbrxRouter(nn.Module):
31 + """A Router implementation for DBRX that returns logits for each expert
32 + per token.
33 + """
34 +
35 + def __init__(
36 + self,
37 + config: DbrxConfig,
38 + params_dtype: Optional[torch.dtype] = None,
39 + ):
40 + super().__init__()
41 + self.tp_size = get_tensor_model_parallel_world_size()
42 + self.num_total_experts = config.ffn_config.moe_num_experts
43 + self.d_model = config.d_model
44 + self.layer = ReplicatedLinear(
45 + self.d_model,
46 + self.num_total_experts,
47 + bias=False,
48 + params_dtype=params_dtype,
49 + linear_method=None,
50 + )
51 +
52 + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
53 + router_logits, _ = self.layer(hidden_states)
54 + return router_logits
55 +
56 +
57 +class DbrxExperts(nn.Module):
58 + """A tensor-parallel MoE implementation for DBRX.
59 +
60 + Each expert's weights are sharded across all ranks and a fused MoE
61 + kernel is used for the forward pass, and finally we reduce the outputs
62 + across ranks.
63 + """
64 +
65 + def __init__(
66 + self,
67 + config: DbrxConfig,
68 + linear_method: Optional[LinearMethodBase] = None,
69 + params_dtype: Optional[torch.dtype] = None,
70 + ):
71 + super().__init__()
72 + self.tp_size = get_tensor_model_parallel_world_size()
73 + self.num_total_experts = config.ffn_config.moe_num_experts
74 + self.top_k = config.ffn_config.moe_top_k
75 + self.d_model = config.d_model
76 + self.intermediate_size = (config.ffn_config.ffn_hidden_size //
77 + self.tp_size)
78 +
79 + if params_dtype is None:
80 + params_dtype = torch.get_default_dtype()
81 + self.params_dtype = params_dtype
82 +
83 + self.router = DbrxRouter(config, self.params_dtype)
84 + self.ws = nn.Parameter(
85 + torch.empty(
86 + self.num_total_experts,
87 + 2 * self.intermediate_size,
88 + self.d_model,
89 + device="cuda",
90 + dtype=self.params_dtype,
91 + ))
92 + self.w2s = nn.Parameter(
93 + torch.empty(
94 + self.num_total_experts,
95 + self.d_model,
96 + self.intermediate_size,
97 + device="cuda",
98 + dtype=self.params_dtype,
99 + ))
100 +
101 + set_weight_attrs(
102 + self.ws,
103 + {
104 + "weight_loader": self.weight_loader,
105 + },
106 + )
107 + set_weight_attrs(
108 + self.w2s,
109 + {
110 + "weight_loader": self.weight_loader,
111 + },
112 + )
113 +
114 + def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor,
115 + weight_name: str):
116 + tp_rank = get_tensor_model_parallel_rank()
117 + param_data = param.data
118 + shard_size = self.intermediate_size
119 + shard = slice(tp_rank * shard_size, (tp_rank + 1) * shard_size)
120 + # DBRX uses GLU for each experts.
121 + # GLU has 3 linear layers: w1, v1 and w2.
122 + if weight_name.endswith("w1"):
123 + loaded_weight = torch.reshape(
124 + loaded_weight,
125 + [-1, self.intermediate_size * self.tp_size, self.d_model],
126 + )
127 + param_data[:, 0:shard_size, :] = loaded_weight[:, shard, :]
128 + if weight_name.endswith("v1"):
129 + loaded_weight = torch.reshape(
130 + loaded_weight,
131 + [-1, self.intermediate_size * self.tp_size, self.d_model],
132 + )
133 + param_data[:,
134 + shard_size:2 * shard_size, :] = loaded_weight[:,
135 + shard, :]
136 + if weight_name.endswith("w2"):
137 + loaded_weight = torch.reshape(
138 + loaded_weight,
139 + [-1, self.intermediate_size * self.tp_size, self.d_model],
140 + ).transpose(1, 2)
141 + param_data[:] = loaded_weight[:, :, shard]
142 +
143 + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
144 + num_tokens, hidden_size = hidden_states.shape
145 + hidden_states = hidden_states.view(-1, self.d_model)
146 + # router_logits: (num_tokens, n_experts)
147 + router_logits = self.router(hidden_states)
148 + final_hidden_states = fused_moe(
149 + hidden_states,
150 + self.ws,
151 + self.w2s,
152 + router_logits,
153 + self.top_k,
154 + renormalize=True,
155 + inplace=True,
156 + )
157 +
158 + if self.tp_size > 1:
159 + final_hidden_states = tensor_model_parallel_all_reduce(
160 + final_hidden_states)
161 +
162 + return final_hidden_states.view(num_tokens, hidden_size)
163 +
164 +
165 +class DbrxAttention(nn.Module):
166 +
167 + def __init__(
168 + self,
169 + config: DbrxConfig,
170 + linear_method: Optional[LinearMethodBase] = None,
171 + ):
172 + super().__init__()
173 + self.d_model = config.d_model
174 + self.total_num_heads = config.n_heads
175 + self.head_dim = self.d_model // self.total_num_heads
176 + self.total_num_kv_heads = config.attn_config.kv_n_heads
177 + self.clip_qkv = config.attn_config.clip_qkv
178 + self.rope_theta = config.attn_config.rope_theta
179 + self.max_position = config.max_seq_len
180 +
181 + # pylint: disable=invalid-name
182 + self.Wqkv = QKVParallelLinear(
183 + self.d_model,
184 + self.head_dim,
185 + self.total_num_heads,
186 + self.total_num_kv_heads,
187 + bias=False,
188 + linear_method=linear_method,
189 + )
190 + self.out_proj = RowParallelLinear(
191 + self.d_model,
192 + self.d_model,
193 + bias=False,
194 + linear_method=linear_method,
195 + )
196 + self.rotary_emb = get_rope(
197 + self.head_dim,
198 + rotary_dim=self.head_dim,
199 + max_position=self.max_position,
200 + base=int(self.rope_theta),
201 + is_neox_style=True,
202 + )
203 +
204 + tp_world_size = get_tensor_model_parallel_world_size()
205 + self.tp_size = tp_world_size
206 + assert self.total_num_heads % tp_world_size == 0
207 + self.num_heads = self.total_num_heads // tp_world_size
208 + if self.total_num_kv_heads >= tp_world_size:
209 + # Number of KV heads is greater than TP size, so we partition
210 + # the KV heads across multiple tensor parallel GPUs.
211 + assert self.total_num_kv_heads % tp_world_size == 0
212 + else:
213 + # Number of KV heads is less than TP size, so we replicate
214 + # the KV heads across multiple tensor parallel GPUs.
215 + assert tp_world_size % self.total_num_kv_heads == 0
216 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_world_size)
217 + self.q_size = self.num_heads * self.head_dim
218 + self.kv_size = self.num_kv_heads * self.head_dim
219 + self.scaling = self.head_dim**-0.5
220 + self.attn = Attention(
221 + self.num_heads,
222 + self.head_dim,
223 + self.scaling,
224 + num_kv_heads=self.num_kv_heads,
225 + )
226 +
227 + def forward(
228 + self,
229 + position_ids: torch.Tensor,
230 + hidden_states: torch.Tensor,
231 + kv_cache: torch.Tensor,
232 + attn_metadata: AttentionMetadata,
233 + ) -> torch.Tensor:
234 + qkv, _ = self.Wqkv(hidden_states)
235 + if self.clip_qkv is not None:
236 + qkv.clamp_(min=-self.clip_qkv, max=self.clip_qkv)
237 + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
238 + q, k = self.rotary_emb(position_ids, q, k)
239 + attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
240 + hidden_states, _ = self.out_proj(attn_output)
241 + return hidden_states
242 +
243 +
244 +class DbrxFusedNormAttention(nn.Module):
245 +
246 + def __init__(
247 + self,
248 + config: DbrxConfig,
249 + linear_method: Optional[LinearMethodBase] = None,
250 + ):
251 + super().__init__()
252 + self.d_model = config.d_model
253 + self.attn = DbrxAttention(config, linear_method)
254 + self.norm_1 = nn.LayerNorm(self.d_model)
255 + self.norm_2 = nn.LayerNorm(self.d_model)
256 +
257 + def forward(
258 + self,
259 + position_ids: torch.Tensor,
260 + hidden_states: torch.Tensor,
261 + kv_cache: torch.Tensor,
262 + attn_metadata: AttentionMetadata,
263 + ) -> torch.Tensor:
264 + residual = hidden_states
265 + hidden_states = self.norm_1(hidden_states)
266 + x = self.attn(
267 + position_ids=position_ids,
268 + hidden_states=hidden_states,
269 + kv_cache=kv_cache,
270 + attn_metadata=attn_metadata,
271 + )
272 + hidden_states = residual + x
273 + residual = hidden_states
274 + hidden_states = self.norm_2(hidden_states)
275 + return hidden_states, residual
276 +
277 +
278 +class DbrxBlock(nn.Module):
279 +
280 + def __init__(
281 + self,
282 + config: DbrxConfig,
283 + linear_method: Optional[LinearMethodBase] = None,
284 + ):
285 + super().__init__()
286 + self.norm_attn_norm = DbrxFusedNormAttention(config, linear_method)
287 + self.ffn = DbrxExperts(config, linear_method)
288 +
289 + def forward(
290 + self,
291 + position_ids: torch.Tensor,
292 + hidden_states: torch.Tensor,
293 + kv_cache: torch.Tensor,
294 + attn_metadata: AttentionMetadata,
295 + ) -> torch.Tensor:
296 + hidden_states, residual = self.norm_attn_norm(
297 + position_ids=position_ids,
298 + hidden_states=hidden_states,
299 + kv_cache=kv_cache,
300 + attn_metadata=attn_metadata,
301 + )
302 + hidden_states = self.ffn(hidden_states)
303 + hidden_states = hidden_states + residual
304 + return hidden_states
305 +
306 +
307 +class DbrxModel(nn.Module):
308 +
309 + def __init__(
310 + self,
311 + config: DbrxConfig,
312 + linear_method: Optional[LinearMethodBase] = None,
313 + ):
314 + super().__init__()
315 + self.wte = VocabParallelEmbedding(
316 + config.vocab_size,
317 + config.d_model,
318 + )
319 + self.blocks = nn.ModuleList(
320 + [DbrxBlock(config, linear_method) for _ in range(config.n_layers)])
321 + self.norm_f = nn.LayerNorm(config.d_model, eps=1e-5)
322 + for module in self.modules():
323 + if hasattr(module, "bias") and isinstance(module.bias,
324 + nn.Parameter):
325 + # Remove the bias term in Linear and LayerNorm.
326 + module.register_parameter("bias", None)
327 +
328 + def forward(
329 + self,
330 + input_ids: torch.Tensor,
331 + position_ids: torch.Tensor,
332 + kv_caches: List[torch.Tensor],
333 + attn_metadata: AttentionMetadata,
334 + ) -> torch.Tensor:
335 + hidden_states = self.wte(input_ids)
336 + for i in range(len(self.blocks)):
337 + block = self.blocks[i]
338 + hidden_states = block(
339 + position_ids,
340 + hidden_states,
341 + kv_caches[i],
342 + attn_metadata,
343 + )
344 + hidden_states = self.norm_f(hidden_states)
345 + return hidden_states
346 +
347 +
348 +class DbrxForCausalLM(nn.Module):
349 +
350 + def __init__(
351 + self,
352 + config: DbrxConfig,
353 + linear_method: Optional[LinearMethodBase] = None,
354 + ):
355 + super().__init__()
356 + self.config = config
357 + self.linear_method = linear_method
358 + self.unpadded_vocab_size = config.vocab_size
359 + self.transformer = DbrxModel(config, linear_method)
360 + self.lm_head = ParallelLMHead(
361 + config.vocab_size,
362 + config.d_model,
363 + org_num_embeddings=config.vocab_size,
364 + padding_size=DEFAULT_VOCAB_PADDING_SIZE,
365 + )
366 + self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
367 + config.vocab_size)
368 + self.sampler = Sampler()
369 +
370 + def forward(
371 + self,
372 + input_ids: torch.Tensor,
373 + positions: torch.Tensor,
374 + kv_caches: List[torch.Tensor],
375 + attn_metadata: AttentionMetadata,
376 + ) -> torch.Tensor:
377 + hidden_states = self.transformer(input_ids, positions, kv_caches,
378 + attn_metadata)
379 + return hidden_states
380 +
381 + def compute_logits(self, hidden_states: torch.Tensor,
382 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
383 + logits = self.logits_processor(self.lm_head.weight, hidden_states,
384 + sampling_metadata)
385 + return logits
386 +
387 + def sample(
388 + self,
389 + logits: Optional[torch.Tensor],
390 + sampling_metadata: SamplingMetadata,
391 + ) -> Optional[SamplerOutput]:
392 + next_tokens = self.sampler(logits, sampling_metadata)
393 + return next_tokens
394 +
395 + def load_weights(
396 + self,
397 + model_name_or_path: str,
398 + cache_dir: Optional[str] = None,
399 + load_format: str = "auto",
400 + revision: Optional[str] = None,
401 + ):
402 + expert_params_mapping = [(
403 + "ws" if weight_name in ["w1", "v1"] else "w2s",
404 + f"experts.mlp.{weight_name}",
405 + ) for weight_name in ["w1", "v1", "w2"]]
406 + params_dict = dict(self.named_parameters(remove_duplicate=False))
407 + for name, loaded_weight in hf_model_weights_iterator(
408 + model_name_or_path, cache_dir, load_format, revision):
409 + for param_name, weight_name in expert_params_mapping:
410 + if weight_name not in name:
411 + continue
412 + name = name.replace(weight_name, param_name)
413 + param = params_dict[name]
414 + weight_loader = param.weight_loader
415 + weight_loader(param, loaded_weight, weight_name)
416 + break
417 + else:
418 + param = params_dict[name]
419 + weight_loader = getattr(param, "weight_loader",
420 + default_weight_loader)
421 + weight_loader(param, loaded_weight)
vllm/transformers_utils/config.py
+1
@@ -6,6 +6,7 @@ from vllm.transformers_utils.configs import *
6
7 _CONFIG_REGISTRY = {
8 "chatglm": ChatGLMConfig,
9 + "dbrx": DbrxConfig,
10 "mpt": MPTConfig,
11 "RefinedWeb": RWConfig, # For tiiuae/falcon-40b(-instruct)
12 "RefinedWebModel": RWConfig, # For tiiuae/falcon-7b(-instruct)
vllm/transformers_utils/configs/__init__.py
+2
@@ -1,4 +1,5 @@
1 from vllm.transformers_utils.configs.chatglm import ChatGLMConfig
2 +from vllm.transformers_utils.configs.dbrx import DbrxConfig
3 # RWConfig is for the original tiiuae/falcon-40b(-instruct) and
4 # tiiuae/falcon-7b(-instruct) models. Newer Falcon models will use the
5 # `FalconConfig` class from the official HuggingFace transformers library.
@@ -8,6 +9,7 @@ from vllm.transformers_utils.configs.mpt import MPTConfig
9
10 __all__ = [
11 "ChatGLMConfig",
12 + "DbrxConfig",
13 "MPTConfig",
14 "RWConfig",
15 "JAISConfig",
vllm/transformers_utils/configs/dbrx.py new
+277
@@ -0,0 +1,277 @@
1 +# yapf: disable
2 +# ruff: noqa: E501
3 +# coding=utf-8
4 +# Copied from
5 +# https://huggingface.co/databricks/dbrx-base/blob/main/configuration_dbrx.py
6 +"""Dbrx configuration."""
7 +
8 +from typing import Any, Optional
9 +
10 +from transformers.configuration_utils import PretrainedConfig
11 +from transformers.utils import logging
12 +
13 +logger = logging.get_logger(__name__)
14 +
15 +DBRX_PRETRAINED_CONFIG_ARCHIVE_MAP = {}
16 +
17 +
18 +class DbrxAttentionConfig(PretrainedConfig):
19 + """Configuration class for Dbrx Attention.
20 +
21 + [`DbrxAttention`] class. It is used to instantiate attention layers
22 + according to the specified arguments, defining the layers architecture.
23 +
24 + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
25 + documentation from [`PretrainedConfig`] for more information.
26 +
27 + Args:
28 + attn_pdrop (`float`, *optional*, defaults to 0.0):
29 + The dropout probability for the attention layers.
30 + clip_qkv (`float`, *optional*, defaults to None):
31 + If not `None`, clip the queries, keys, and values in the attention layer to this value.
32 + kv_n_heads (Optional[int]): For grouped_query_attention only, allow user to specify number of kv heads.
33 + rope_theta (float): The base frequency for rope.
34 + """
35 +
36 + def __init__(
37 + self,
38 + attn_pdrop: float = 0,
39 + clip_qkv: Optional[float] = None,
40 + kv_n_heads: int = 1,
41 + rope_theta: float = 10000.0,
42 + **kwargs: Any,
43 + ):
44 + super().__init__(**kwargs)
45 + self.attn_pdrop = attn_pdrop
46 + self.clip_qkv = clip_qkv
47 + self.kv_n_heads = kv_n_heads
48 + self.rope_theta = rope_theta
49 +
50 + for k in ["model_type"]:
51 + if k in kwargs:
52 + kwargs.pop(k)
53 + if len(kwargs) != 0:
54 + raise ValueError(f"Found unknown {kwargs=}")
55 +
56 + @classmethod
57 + def from_pretrained(
58 + cls, pretrained_model_name_or_path: str, **kwargs: Any
59 + ) -> "PretrainedConfig":
60 + cls._set_token_in_kwargs(kwargs)
61 +
62 + config_dict, kwargs = cls.get_config_dict(
63 + pretrained_model_name_or_path, **kwargs
64 + )
65 +
66 + if config_dict.get("model_type") == "dbrx":
67 + config_dict = config_dict["attn_config"]
68 +
69 + if (
70 + "model_type" in config_dict
71 + and hasattr(cls, "model_type")
72 + and config_dict["model_type"] != cls.model_type
73 + ):
74 + logger.warning(
75 + f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
76 + + f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."
77 + )
78 +
79 + return cls.from_dict(config_dict, **kwargs)
80 +
81 +
82 +class DbrxFFNConfig(PretrainedConfig):
83 + """Configuration class for Dbrx FFN.
84 +
85 + [`DbrxFFN`] class. It is used to instantiate feedforward layers according to
86 + the specified arguments, defining the layers architecture.
87 +
88 + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
89 + documentation from [`PretrainedConfig`] for more information.
90 +
91 + Args:
92 + ffn_act_fn (dict, optional): A dict specifying activation function for the FFN.
93 + The dict should have a key 'name' with the value being the name of
94 + the activation function along with any additional keyword arguments.
95 + ffn_hidden_size (int, optional): The hidden size of the feedforward network.
96 + moe_num_experts (int, optional): The number of experts in the mixture of experts layer.
97 + moe_top_k (int, optional): The number of experts to use in the mixture of experts layer.
98 + moe_jitter_eps (float, optional): The jitter epsilon for the mixture of experts layer.
99 + moe_loss_weight (float, optional): The loss weight for the mixture of experts layer.
100 + moe_normalize_expert_weights (float, optional): The normalization factor for the expert weights.
101 + uniform_expert_assignment (bool, optional): Whether to use uniform expert assignment.
102 + This should only be used for benchmarking purposes.
103 + """
104 +
105 + def __init__(
106 + self,
107 + ffn_act_fn: Optional[dict] = None,
108 + ffn_hidden_size: int = 3584,
109 + moe_num_experts: int = 4,
110 + moe_top_k: int = 1,
111 + moe_jitter_eps: Optional[float] = None,
112 + moe_loss_weight: float = 0.01,
113 + moe_normalize_expert_weights: Optional[float] = 1,
114 + uniform_expert_assignment: bool = False,
115 + **kwargs: Any,
116 + ):
117 + super().__init__()
118 + if ffn_act_fn is None:
119 + ffn_act_fn = {"name": "silu"}
120 + self.ffn_act_fn = ffn_act_fn
121 + self.ffn_hidden_size = ffn_hidden_size
122 + self.moe_num_experts = moe_num_experts
123 + self.moe_top_k = moe_top_k
124 + self.moe_jitter_eps = moe_jitter_eps
125 + self.moe_loss_weight = moe_loss_weight
126 + self.moe_normalize_expert_weights = moe_normalize_expert_weights
127 + self.uniform_expert_assignment = uniform_expert_assignment
128 +
129 + for k in ["model_type"]:
130 + if k in kwargs:
131 + kwargs.pop(k)
132 + if len(kwargs) != 0:
133 + raise ValueError(f"Found unknown {kwargs=}")
134 +
135 + @classmethod
136 + def from_pretrained(
137 + cls, pretrained_model_name_or_path: str, **kwargs: Any
138 + ) -> "PretrainedConfig":
139 + cls._set_token_in_kwargs(kwargs)
140 +
141 + config_dict, kwargs = cls.get_config_dict(
142 + pretrained_model_name_or_path, **kwargs
143 + )
144 +
145 + if config_dict.get("model_type") == "dbrx":
146 + config_dict = config_dict["ffn_config"]
147 +
148 + if (
149 + "model_type" in config_dict
150 + and hasattr(cls, "model_type")
151 + and config_dict["model_type"] != cls.model_type
152 + ):
153 + logger.warning(
154 + f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
155 + + f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."
156 + )
157 +
158 + return cls.from_dict(config_dict, **kwargs)
159 +
160 +
161 +class DbrxConfig(PretrainedConfig):
162 + """Configuration class for Dbrx.
163 +
164 + [`DbrxModel`]. It is used to instantiate a Dbrx model according to the
165 + specified arguments, defining the model architecture.
166 +
167 + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
168 + documentation from [`PretrainedConfig`] for more information.
169 +
170 +
171 + Args:
172 + d_model (`int`, *optional*, defaults to 6144):
173 + Dimensionality of the embeddings and hidden states.
174 + n_heads (`int`, *optional*, defaults to 48):
175 + Number of attention heads for each attention layer in the Transformer encoder.
176 + n_layers (`int`, *optional*, defaults to 40):
177 + Number of hidden layers in the Transformer encoder.
178 + max_seq_len (`int`, *optional*, defaults to 32768):
179 + The maximum sequence length of the model.
180 + vocab_size (`int`, *optional*, defaults to 100352):
181 + Vocabulary size of the Dbrx model. Defines the maximum number of different tokens that can be represented by
182 + the `inputs_ids` passed when calling [`DbrxModel`].
183 + resid_pdrop (`float`, *optional*, defaults to 0.0):
184 + The dropout probability applied to the attention output before combining with residual.
185 + emb_pdrop (`float`, *optional*, defaults to 0.0):
186 + The dropout probability for the embedding layer.
187 + attn_config (`dict`, *optional*):
188 + A dictionary used to configure the model's attention module.
189 + ffn_config (`dict`, *optional*):
190 + A dictionary used to configure the model's FFN module.
191 + use_cache (`bool`, *optional*, defaults to `False`):
192 + Whether or not the model should return the last key/values attentions (not used by all models).
193 + initializer_range (`float`, *optional*, defaults to 0.02):
194 + The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
195 + output_router_logits (`bool`, *optional*, defaults to `False`):
196 + Whether or not the router logits should be returned by the model. Enabling this will also
197 + allow the model to output the auxiliary loss. See [here]() for more details
198 + router_aux_loss_coef (`float`, *optional*, defaults to 0.001):
199 + The aux loss factor for the total loss.
200 +
201 +
202 + Example:
203 + ```python
204 + >>> from transformers import DbrxConfig, DbrxModel
205 +
206 + >>> # Initializing a Dbrx configuration
207 + >>> configuration = DbrxConfig()
208 +
209 + >>> # Initializing a model (with random weights) from the configuration
210 + >>> model = DbrxModel(configuration)
211 +
212 + >>> # Accessing the model configuration
213 + >>> configuration = model.config
214 + ```
215 + """
216 +
217 + model_type = "dbrx"
218 + attribute_map = {
219 + "num_attention_heads": "n_heads",
220 + "hidden_size": "d_model",
221 + "num_hidden_layers": "n_layers",
222 + "max_position_embeddings": "max_seq_len",
223 + }
224 +
225 + def __init__(
226 + self,
227 + d_model: int = 2048,
228 + n_heads: int = 16,
229 + n_layers: int = 24,
230 + max_seq_len: int = 2048,
231 + vocab_size: int = 32000,
232 + resid_pdrop: float = 0.0,
233 + emb_pdrop: float = 0.0,
234 + attn_config: Optional[DbrxAttentionConfig] = None,
235 + ffn_config: Optional[DbrxFFNConfig] = None,
236 + use_cache: bool = True,
237 + initializer_range: float = 0.02,
238 + output_router_logits: bool = False,
239 + router_aux_loss_coef: float = 0.05,
240 + **kwargs: Any,
241 + ):
242 + if attn_config is None:
243 + self.attn_config = DbrxAttentionConfig()
244 + elif isinstance(attn_config, dict):
245 + self.attn_config = DbrxAttentionConfig(**attn_config)
246 + else:
247 + self.attn_config = attn_config
248 +
249 + if ffn_config is None:
250 + self.ffn_config = DbrxFFNConfig()
251 + elif isinstance(ffn_config, dict):
252 + self.ffn_config = DbrxFFNConfig(**ffn_config)
253 + else:
254 + self.ffn_config = ffn_config
255 +
256 + self.d_model = d_model
257 + self.n_heads = n_heads
258 + self.n_layers = n_layers
259 + self.max_seq_len = max_seq_len
260 + self.vocab_size = vocab_size
261 + self.resid_pdrop = resid_pdrop
262 + self.emb_pdrop = emb_pdrop
263 + self.use_cache = use_cache
264 + self.initializer_range = initializer_range
265 + self.output_router_logits = output_router_logits
266 + self.router_aux_loss_coef = router_aux_loss_coef
267 +
268 + tie_word_embeddings = kwargs.pop("tie_word_embeddings", False)
269 + if tie_word_embeddings:
270 + raise ValueError(
271 + "tie_word_embeddings is not supported for Dbrx models."
272 + )
273 +
274 + super().__init__(
275 + tie_word_embeddings=tie_word_embeddings,
276 + **kwargs,
277 + )