A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[Model] Add support for DBRX (#3660)
Megha Agarwal committed
Mar 27, 2024 at 13:01 UTC
e24336b5a772ab3aa6ad83527b880f9e5050ea2a
9 files changed
+713
README.md
+1
@@ -67,6 +67,7 @@ vLLM seamlessly supports many Hugging Face models, including the following archi
67
- Baichuan & Baichuan2 (`baichuan-inc/Baichuan2-13B-Chat`, `baichuan-inc/Baichuan-7B`, etc.)
68
- BLOOM (`bigscience/bloom`, `bigscience/bloomz`, etc.)
69
- ChatGLM (`THUDM/chatglm2-6b`, `THUDM/chatglm3-6b`, etc.)
70
+- DBRX (`databricks/dbrx-base`, `databricks/dbrx-instruct` etc.)
71
- DeciLM (`Deci/DeciLM-7B`, `Deci/DeciLM-7B-instruct`, etc.)
72
- Falcon (`tiiuae/falcon-7b`, `tiiuae/falcon-40b`, `tiiuae/falcon-rw-7b`, etc.)
73
- Gemma (`google/gemma-2b`, `google/gemma-7b`, etc.)
docs/source/models/supported_models.rst
+4
@@ -27,6 +27,10 @@ Alongside each architecture, we include some popular models that use it.
27
- ChatGLM
28
- :code:`THUDM/chatglm2-6b`, :code:`THUDM/chatglm3-6b`, etc.
29
- ✅︎
30
+ * - :code:`DbrxForCausalLM`
31
+ - DBRX
32
+ - :code:`databricks/dbrx-base`, :code:`databricks/dbrx-instruct`, etc.
33
+ -
34
* - :code:`DeciLMForCausalLM`
35
- DeciLM
36
- :code:`Deci/DeciLM-7B`, :code:`Deci/DeciLM-7B-instruct`, etc.
requirements.txt
+1
@@ -14,3 +14,4 @@ prometheus_client >= 0.18.0
14
pynvml == 11.5.0
15
triton >= 2.1.0
16
outlines == 0.0.34
17
+tiktoken == 0.6.0 # Required for DBRX tokenizer
vllm/config.py
+5
@@ -277,6 +277,11 @@ class ModelConfig:
277
# Currently, tensor parallelism is not supported in this case.
278
return 1
279
280
+ # For DBRX and MPT
281
+ if self.hf_config.model_type in ["dbrx", "mpt"]:
282
+ return getattr(self.hf_config.attn_config, "kv_n_heads",
283
+ self.hf_config.num_attention_heads)
284
+
285
attributes = [
286
# For Falcon:
287
"n_head_kv",
vllm/model_executor/models/__init__.py
+1
@@ -17,6 +17,7 @@ _MODELS = {
17
"BloomForCausalLM": ("bloom", "BloomForCausalLM"),
18
"ChatGLMModel": ("chatglm", "ChatGLMForCausalLM"),
19
"ChatGLMForConditionalGeneration": ("chatglm", "ChatGLMForCausalLM"),
20
+ "DbrxForCausalLM": ("dbrx", "DbrxForCausalLM"),
21
"DeciLMForCausalLM": ("decilm", "DeciLMForCausalLM"),
22
"DeepseekForCausalLM": ("deepseek", "DeepseekForCausalLM"),
23
"FalconForCausalLM": ("falcon", "FalconForCausalLM"),
vllm/model_executor/models/dbrx.py
new
+421
@@ -0,0 +1,421 @@
1
+# coding=utf-8
2
+from typing import List, Optional
3
+
4
+import torch
5
+import torch.nn as nn
6
+
7
+from vllm.attention import Attention, AttentionMetadata
8
+from vllm.model_executor.layers.fused_moe import fused_moe
9
+from vllm.model_executor.layers.linear import (LinearMethodBase,
10
+ QKVParallelLinear,
11
+ ReplicatedLinear,
12
+ RowParallelLinear)
13
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
14
+from vllm.model_executor.layers.rotary_embedding import get_rope
15
+from vllm.model_executor.layers.sampler import Sampler
16
+from vllm.model_executor.layers.vocab_parallel_embedding import (
17
+ DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
18
+from vllm.model_executor.parallel_utils.communication_op import (
19
+ tensor_model_parallel_all_reduce)
20
+from vllm.model_executor.parallel_utils.parallel_state import (
21
+ get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
22
+from vllm.model_executor.sampling_metadata import SamplingMetadata
23
+from vllm.model_executor.utils import set_weight_attrs
24
+from vllm.model_executor.weight_utils import (default_weight_loader,
25
+ hf_model_weights_iterator)
26
+from vllm.sequence import SamplerOutput
27
+from vllm.transformers_utils.configs.dbrx import DbrxConfig
28
+
29
+
30
+class DbrxRouter(nn.Module):
31
+ """A Router implementation for DBRX that returns logits for each expert
32
+ per token.
33
+ """
34
+
35
+ def __init__(
36
+ self,
37
+ config: DbrxConfig,
38
+ params_dtype: Optional[torch.dtype] = None,
39
+ ):
40
+ super().__init__()
41
+ self.tp_size = get_tensor_model_parallel_world_size()
42
+ self.num_total_experts = config.ffn_config.moe_num_experts
43
+ self.d_model = config.d_model
44
+ self.layer = ReplicatedLinear(
45
+ self.d_model,
46
+ self.num_total_experts,
47
+ bias=False,
48
+ params_dtype=params_dtype,
49
+ linear_method=None,
50
+ )
51
+
52
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
53
+ router_logits, _ = self.layer(hidden_states)
54
+ return router_logits
55
+
56
+
57
+class DbrxExperts(nn.Module):
58
+ """A tensor-parallel MoE implementation for DBRX.
59
+
60
+ Each expert's weights are sharded across all ranks and a fused MoE
61
+ kernel is used for the forward pass, and finally we reduce the outputs
62
+ across ranks.
63
+ """
64
+
65
+ def __init__(
66
+ self,
67
+ config: DbrxConfig,
68
+ linear_method: Optional[LinearMethodBase] = None,
69
+ params_dtype: Optional[torch.dtype] = None,
70
+ ):
71
+ super().__init__()
72
+ self.tp_size = get_tensor_model_parallel_world_size()
73
+ self.num_total_experts = config.ffn_config.moe_num_experts
74
+ self.top_k = config.ffn_config.moe_top_k
75
+ self.d_model = config.d_model
76
+ self.intermediate_size = (config.ffn_config.ffn_hidden_size //
77
+ self.tp_size)
78
+
79
+ if params_dtype is None:
80
+ params_dtype = torch.get_default_dtype()
81
+ self.params_dtype = params_dtype
82
+
83
+ self.router = DbrxRouter(config, self.params_dtype)
84
+ self.ws = nn.Parameter(
85
+ torch.empty(
86
+ self.num_total_experts,
87
+ 2 * self.intermediate_size,
88
+ self.d_model,
89
+ device="cuda",
90
+ dtype=self.params_dtype,
91
+ ))
92
+ self.w2s = nn.Parameter(
93
+ torch.empty(
94
+ self.num_total_experts,
95
+ self.d_model,
96
+ self.intermediate_size,
97
+ device="cuda",
98
+ dtype=self.params_dtype,
99
+ ))
100
+
101
+ set_weight_attrs(
102
+ self.ws,
103
+ {
104
+ "weight_loader": self.weight_loader,
105
+ },
106
+ )
107
+ set_weight_attrs(
108
+ self.w2s,
109
+ {
110
+ "weight_loader": self.weight_loader,
111
+ },
112
+ )
113
+
114
+ def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor,
115
+ weight_name: str):
116
+ tp_rank = get_tensor_model_parallel_rank()
117
+ param_data = param.data
118
+ shard_size = self.intermediate_size
119
+ shard = slice(tp_rank * shard_size, (tp_rank + 1) * shard_size)
120
+ # DBRX uses GLU for each experts.
121
+ # GLU has 3 linear layers: w1, v1 and w2.
122
+ if weight_name.endswith("w1"):
123
+ loaded_weight = torch.reshape(
124
+ loaded_weight,
125
+ [-1, self.intermediate_size * self.tp_size, self.d_model],
126
+ )
127
+ param_data[:, 0:shard_size, :] = loaded_weight[:, shard, :]
128
+ if weight_name.endswith("v1"):
129
+ loaded_weight = torch.reshape(
130
+ loaded_weight,
131
+ [-1, self.intermediate_size * self.tp_size, self.d_model],
132
+ )
133
+ param_data[:,
134
+ shard_size:2 * shard_size, :] = loaded_weight[:,
135
+ shard, :]
136
+ if weight_name.endswith("w2"):
137
+ loaded_weight = torch.reshape(
138
+ loaded_weight,
139
+ [-1, self.intermediate_size * self.tp_size, self.d_model],
140
+ ).transpose(1, 2)
141
+ param_data[:] = loaded_weight[:, :, shard]
142
+
143
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
144
+ num_tokens, hidden_size = hidden_states.shape
145
+ hidden_states = hidden_states.view(-1, self.d_model)
146
+ # router_logits: (num_tokens, n_experts)
147
+ router_logits = self.router(hidden_states)
148
+ final_hidden_states = fused_moe(
149
+ hidden_states,
150
+ self.ws,
151
+ self.w2s,
152
+ router_logits,
153
+ self.top_k,
154
+ renormalize=True,
155
+ inplace=True,
156
+ )
157
+
158
+ if self.tp_size > 1:
159
+ final_hidden_states = tensor_model_parallel_all_reduce(
160
+ final_hidden_states)
161
+
162
+ return final_hidden_states.view(num_tokens, hidden_size)
163
+
164
+
165
+class DbrxAttention(nn.Module):
166
+
167
+ def __init__(
168
+ self,
169
+ config: DbrxConfig,
170
+ linear_method: Optional[LinearMethodBase] = None,
171
+ ):
172
+ super().__init__()
173
+ self.d_model = config.d_model
174
+ self.total_num_heads = config.n_heads
175
+ self.head_dim = self.d_model // self.total_num_heads
176
+ self.total_num_kv_heads = config.attn_config.kv_n_heads
177
+ self.clip_qkv = config.attn_config.clip_qkv
178
+ self.rope_theta = config.attn_config.rope_theta
179
+ self.max_position = config.max_seq_len
180
+
181
+ # pylint: disable=invalid-name
182
+ self.Wqkv = QKVParallelLinear(
183
+ self.d_model,
184
+ self.head_dim,
185
+ self.total_num_heads,
186
+ self.total_num_kv_heads,
187
+ bias=False,
188
+ linear_method=linear_method,
189
+ )
190
+ self.out_proj = RowParallelLinear(
191
+ self.d_model,
192
+ self.d_model,
193
+ bias=False,
194
+ linear_method=linear_method,
195
+ )
196
+ self.rotary_emb = get_rope(
197
+ self.head_dim,
198
+ rotary_dim=self.head_dim,
199
+ max_position=self.max_position,
200
+ base=int(self.rope_theta),
201
+ is_neox_style=True,
202
+ )
203
+
204
+ tp_world_size = get_tensor_model_parallel_world_size()
205
+ self.tp_size = tp_world_size
206
+ assert self.total_num_heads % tp_world_size == 0
207
+ self.num_heads = self.total_num_heads // tp_world_size
208
+ if self.total_num_kv_heads >= tp_world_size:
209
+ # Number of KV heads is greater than TP size, so we partition
210
+ # the KV heads across multiple tensor parallel GPUs.
211
+ assert self.total_num_kv_heads % tp_world_size == 0
212
+ else:
213
+ # Number of KV heads is less than TP size, so we replicate
214
+ # the KV heads across multiple tensor parallel GPUs.
215
+ assert tp_world_size % self.total_num_kv_heads == 0
216
+ self.num_kv_heads = max(1, self.total_num_kv_heads // tp_world_size)
217
+ self.q_size = self.num_heads * self.head_dim
218
+ self.kv_size = self.num_kv_heads * self.head_dim
219
+ self.scaling = self.head_dim**-0.5
220
+ self.attn = Attention(
221
+ self.num_heads,
222
+ self.head_dim,
223
+ self.scaling,
224
+ num_kv_heads=self.num_kv_heads,
225
+ )
226
+
227
+ def forward(
228
+ self,
229
+ position_ids: torch.Tensor,
230
+ hidden_states: torch.Tensor,
231
+ kv_cache: torch.Tensor,
232
+ attn_metadata: AttentionMetadata,
233
+ ) -> torch.Tensor:
234
+ qkv, _ = self.Wqkv(hidden_states)
235
+ if self.clip_qkv is not None:
236
+ qkv.clamp_(min=-self.clip_qkv, max=self.clip_qkv)
237
+ q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
238
+ q, k = self.rotary_emb(position_ids, q, k)
239
+ attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
240
+ hidden_states, _ = self.out_proj(attn_output)
241
+ return hidden_states
242
+
243
+
244
+class DbrxFusedNormAttention(nn.Module):
245
+
246
+ def __init__(
247
+ self,
248
+ config: DbrxConfig,
249
+ linear_method: Optional[LinearMethodBase] = None,
250
+ ):
251
+ super().__init__()
252
+ self.d_model = config.d_model
253
+ self.attn = DbrxAttention(config, linear_method)
254
+ self.norm_1 = nn.LayerNorm(self.d_model)
255
+ self.norm_2 = nn.LayerNorm(self.d_model)
256
+
257
+ def forward(
258
+ self,
259
+ position_ids: torch.Tensor,
260
+ hidden_states: torch.Tensor,
261
+ kv_cache: torch.Tensor,
262
+ attn_metadata: AttentionMetadata,
263
+ ) -> torch.Tensor:
264
+ residual = hidden_states
265
+ hidden_states = self.norm_1(hidden_states)
266
+ x = self.attn(
267
+ position_ids=position_ids,
268
+ hidden_states=hidden_states,
269
+ kv_cache=kv_cache,
270
+ attn_metadata=attn_metadata,
271
+ )
272
+ hidden_states = residual + x
273
+ residual = hidden_states
274
+ hidden_states = self.norm_2(hidden_states)
275
+ return hidden_states, residual
276
+
277
+
278
+class DbrxBlock(nn.Module):
279
+
280
+ def __init__(
281
+ self,
282
+ config: DbrxConfig,
283
+ linear_method: Optional[LinearMethodBase] = None,
284
+ ):
285
+ super().__init__()
286
+ self.norm_attn_norm = DbrxFusedNormAttention(config, linear_method)
287
+ self.ffn = DbrxExperts(config, linear_method)
288
+
289
+ def forward(
290
+ self,
291
+ position_ids: torch.Tensor,
292
+ hidden_states: torch.Tensor,
293
+ kv_cache: torch.Tensor,
294
+ attn_metadata: AttentionMetadata,
295
+ ) -> torch.Tensor:
296
+ hidden_states, residual = self.norm_attn_norm(
297
+ position_ids=position_ids,
298
+ hidden_states=hidden_states,
299
+ kv_cache=kv_cache,
300
+ attn_metadata=attn_metadata,
301
+ )
302
+ hidden_states = self.ffn(hidden_states)
303
+ hidden_states = hidden_states + residual
304
+ return hidden_states
305
+
306
+
307
+class DbrxModel(nn.Module):
308
+
309
+ def __init__(
310
+ self,
311
+ config: DbrxConfig,
312
+ linear_method: Optional[LinearMethodBase] = None,
313
+ ):
314
+ super().__init__()
315
+ self.wte = VocabParallelEmbedding(
316
+ config.vocab_size,
317
+ config.d_model,
318
+ )
319
+ self.blocks = nn.ModuleList(
320
+ [DbrxBlock(config, linear_method) for _ in range(config.n_layers)])
321
+ self.norm_f = nn.LayerNorm(config.d_model, eps=1e-5)
322
+ for module in self.modules():
323
+ if hasattr(module, "bias") and isinstance(module.bias,
324
+ nn.Parameter):
325
+ # Remove the bias term in Linear and LayerNorm.
326
+ module.register_parameter("bias", None)
327
+
328
+ def forward(
329
+ self,
330
+ input_ids: torch.Tensor,
331
+ position_ids: torch.Tensor,
332
+ kv_caches: List[torch.Tensor],
333
+ attn_metadata: AttentionMetadata,
334
+ ) -> torch.Tensor:
335
+ hidden_states = self.wte(input_ids)
336
+ for i in range(len(self.blocks)):
337
+ block = self.blocks[i]
338
+ hidden_states = block(
339
+ position_ids,
340
+ hidden_states,
341
+ kv_caches[i],
342
+ attn_metadata,
343
+ )
344
+ hidden_states = self.norm_f(hidden_states)
345
+ return hidden_states
346
+
347
+
348
+class DbrxForCausalLM(nn.Module):
349
+
350
+ def __init__(
351
+ self,
352
+ config: DbrxConfig,
353
+ linear_method: Optional[LinearMethodBase] = None,
354
+ ):
355
+ super().__init__()
356
+ self.config = config
357
+ self.linear_method = linear_method
358
+ self.unpadded_vocab_size = config.vocab_size
359
+ self.transformer = DbrxModel(config, linear_method)
360
+ self.lm_head = ParallelLMHead(
361
+ config.vocab_size,
362
+ config.d_model,
363
+ org_num_embeddings=config.vocab_size,
364
+ padding_size=DEFAULT_VOCAB_PADDING_SIZE,
365
+ )
366
+ self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
367
+ config.vocab_size)
368
+ self.sampler = Sampler()
369
+
370
+ def forward(
371
+ self,
372
+ input_ids: torch.Tensor,
373
+ positions: torch.Tensor,
374
+ kv_caches: List[torch.Tensor],
375
+ attn_metadata: AttentionMetadata,
376
+ ) -> torch.Tensor:
377
+ hidden_states = self.transformer(input_ids, positions, kv_caches,
378
+ attn_metadata)
379
+ return hidden_states
380
+
381
+ def compute_logits(self, hidden_states: torch.Tensor,
382
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
383
+ logits = self.logits_processor(self.lm_head.weight, hidden_states,
384
+ sampling_metadata)
385
+ return logits
386
+
387
+ def sample(
388
+ self,
389
+ logits: Optional[torch.Tensor],
390
+ sampling_metadata: SamplingMetadata,
391
+ ) -> Optional[SamplerOutput]:
392
+ next_tokens = self.sampler(logits, sampling_metadata)
393
+ return next_tokens
394
+
395
+ def load_weights(
396
+ self,
397
+ model_name_or_path: str,
398
+ cache_dir: Optional[str] = None,
399
+ load_format: str = "auto",
400
+ revision: Optional[str] = None,
401
+ ):
402
+ expert_params_mapping = [(
403
+ "ws" if weight_name in ["w1", "v1"] else "w2s",
404
+ f"experts.mlp.{weight_name}",
405
+ ) for weight_name in ["w1", "v1", "w2"]]
406
+ params_dict = dict(self.named_parameters(remove_duplicate=False))
407
+ for name, loaded_weight in hf_model_weights_iterator(
408
+ model_name_or_path, cache_dir, load_format, revision):
409
+ for param_name, weight_name in expert_params_mapping:
410
+ if weight_name not in name:
411
+ continue
412
+ name = name.replace(weight_name, param_name)
413
+ param = params_dict[name]
414
+ weight_loader = param.weight_loader
415
+ weight_loader(param, loaded_weight, weight_name)
416
+ break
417
+ else:
418
+ param = params_dict[name]
419
+ weight_loader = getattr(param, "weight_loader",
420
+ default_weight_loader)
421
+ weight_loader(param, loaded_weight)
vllm/transformers_utils/config.py
+1
@@ -6,6 +6,7 @@ from vllm.transformers_utils.configs import *
6
7
_CONFIG_REGISTRY = {
8
"chatglm": ChatGLMConfig,
9
+ "dbrx": DbrxConfig,
10
"mpt": MPTConfig,
11
"RefinedWeb": RWConfig, # For tiiuae/falcon-40b(-instruct)
12
"RefinedWebModel": RWConfig, # For tiiuae/falcon-7b(-instruct)
vllm/transformers_utils/configs/__init__.py
+2
@@ -1,4 +1,5 @@
1
from vllm.transformers_utils.configs.chatglm import ChatGLMConfig
2
+from vllm.transformers_utils.configs.dbrx import DbrxConfig
3
# RWConfig is for the original tiiuae/falcon-40b(-instruct) and
4
# tiiuae/falcon-7b(-instruct) models. Newer Falcon models will use the
5
# `FalconConfig` class from the official HuggingFace transformers library.
@@ -8,6 +9,7 @@ from vllm.transformers_utils.configs.mpt import MPTConfig
9
10
__all__ = [
11
"ChatGLMConfig",
12
+ "DbrxConfig",
13
"MPTConfig",
14
"RWConfig",
15
"JAISConfig",
vllm/transformers_utils/configs/dbrx.py
new
+277
@@ -0,0 +1,277 @@
1
+# yapf: disable
2
+# ruff: noqa: E501
3
+# coding=utf-8
4
+# Copied from
5
+# https://huggingface.co/databricks/dbrx-base/blob/main/configuration_dbrx.py
6
+"""Dbrx configuration."""
7
+
8
+from typing import Any, Optional
9
+
10
+from transformers.configuration_utils import PretrainedConfig
11
+from transformers.utils import logging
12
+
13
+logger = logging.get_logger(__name__)
14
+
15
+DBRX_PRETRAINED_CONFIG_ARCHIVE_MAP = {}
16
+
17
+
18
+class DbrxAttentionConfig(PretrainedConfig):
19
+ """Configuration class for Dbrx Attention.
20
+
21
+ [`DbrxAttention`] class. It is used to instantiate attention layers
22
+ according to the specified arguments, defining the layers architecture.
23
+
24
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
25
+ documentation from [`PretrainedConfig`] for more information.
26
+
27
+ Args:
28
+ attn_pdrop (`float`, *optional*, defaults to 0.0):
29
+ The dropout probability for the attention layers.
30
+ clip_qkv (`float`, *optional*, defaults to None):
31
+ If not `None`, clip the queries, keys, and values in the attention layer to this value.
32
+ kv_n_heads (Optional[int]): For grouped_query_attention only, allow user to specify number of kv heads.
33
+ rope_theta (float): The base frequency for rope.
34
+ """
35
+
36
+ def __init__(
37
+ self,
38
+ attn_pdrop: float = 0,
39
+ clip_qkv: Optional[float] = None,
40
+ kv_n_heads: int = 1,
41
+ rope_theta: float = 10000.0,
42
+ **kwargs: Any,
43
+ ):
44
+ super().__init__(**kwargs)
45
+ self.attn_pdrop = attn_pdrop
46
+ self.clip_qkv = clip_qkv
47
+ self.kv_n_heads = kv_n_heads
48
+ self.rope_theta = rope_theta
49
+
50
+ for k in ["model_type"]:
51
+ if k in kwargs:
52
+ kwargs.pop(k)
53
+ if len(kwargs) != 0:
54
+ raise ValueError(f"Found unknown {kwargs=}")
55
+
56
+ @classmethod
57
+ def from_pretrained(
58
+ cls, pretrained_model_name_or_path: str, **kwargs: Any
59
+ ) -> "PretrainedConfig":
60
+ cls._set_token_in_kwargs(kwargs)
61
+
62
+ config_dict, kwargs = cls.get_config_dict(
63
+ pretrained_model_name_or_path, **kwargs
64
+ )
65
+
66
+ if config_dict.get("model_type") == "dbrx":
67
+ config_dict = config_dict["attn_config"]
68
+
69
+ if (
70
+ "model_type" in config_dict
71
+ and hasattr(cls, "model_type")
72
+ and config_dict["model_type"] != cls.model_type
73
+ ):
74
+ logger.warning(
75
+ f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
76
+ + f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."
77
+ )
78
+
79
+ return cls.from_dict(config_dict, **kwargs)
80
+
81
+
82
+class DbrxFFNConfig(PretrainedConfig):
83
+ """Configuration class for Dbrx FFN.
84
+
85
+ [`DbrxFFN`] class. It is used to instantiate feedforward layers according to
86
+ the specified arguments, defining the layers architecture.
87
+
88
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
89
+ documentation from [`PretrainedConfig`] for more information.
90
+
91
+ Args:
92
+ ffn_act_fn (dict, optional): A dict specifying activation function for the FFN.
93
+ The dict should have a key 'name' with the value being the name of
94
+ the activation function along with any additional keyword arguments.
95
+ ffn_hidden_size (int, optional): The hidden size of the feedforward network.
96
+ moe_num_experts (int, optional): The number of experts in the mixture of experts layer.
97
+ moe_top_k (int, optional): The number of experts to use in the mixture of experts layer.
98
+ moe_jitter_eps (float, optional): The jitter epsilon for the mixture of experts layer.
99
+ moe_loss_weight (float, optional): The loss weight for the mixture of experts layer.
100
+ moe_normalize_expert_weights (float, optional): The normalization factor for the expert weights.
101
+ uniform_expert_assignment (bool, optional): Whether to use uniform expert assignment.
102
+ This should only be used for benchmarking purposes.
103
+ """
104
+
105
+ def __init__(
106
+ self,
107
+ ffn_act_fn: Optional[dict] = None,
108
+ ffn_hidden_size: int = 3584,
109
+ moe_num_experts: int = 4,
110
+ moe_top_k: int = 1,
111
+ moe_jitter_eps: Optional[float] = None,
112
+ moe_loss_weight: float = 0.01,
113
+ moe_normalize_expert_weights: Optional[float] = 1,
114
+ uniform_expert_assignment: bool = False,
115
+ **kwargs: Any,
116
+ ):
117
+ super().__init__()
118
+ if ffn_act_fn is None:
119
+ ffn_act_fn = {"name": "silu"}
120
+ self.ffn_act_fn = ffn_act_fn
121
+ self.ffn_hidden_size = ffn_hidden_size
122
+ self.moe_num_experts = moe_num_experts
123
+ self.moe_top_k = moe_top_k
124
+ self.moe_jitter_eps = moe_jitter_eps
125
+ self.moe_loss_weight = moe_loss_weight
126
+ self.moe_normalize_expert_weights = moe_normalize_expert_weights
127
+ self.uniform_expert_assignment = uniform_expert_assignment
128
+
129
+ for k in ["model_type"]:
130
+ if k in kwargs:
131
+ kwargs.pop(k)
132
+ if len(kwargs) != 0:
133
+ raise ValueError(f"Found unknown {kwargs=}")
134
+
135
+ @classmethod
136
+ def from_pretrained(
137
+ cls, pretrained_model_name_or_path: str, **kwargs: Any
138
+ ) -> "PretrainedConfig":
139
+ cls._set_token_in_kwargs(kwargs)
140
+
141
+ config_dict, kwargs = cls.get_config_dict(
142
+ pretrained_model_name_or_path, **kwargs
143
+ )
144
+
145
+ if config_dict.get("model_type") == "dbrx":
146
+ config_dict = config_dict["ffn_config"]
147
+
148
+ if (
149
+ "model_type" in config_dict
150
+ and hasattr(cls, "model_type")
151
+ and config_dict["model_type"] != cls.model_type
152
+ ):
153
+ logger.warning(
154
+ f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
155
+ + f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."
156
+ )
157
+
158
+ return cls.from_dict(config_dict, **kwargs)
159
+
160
+
161
+class DbrxConfig(PretrainedConfig):
162
+ """Configuration class for Dbrx.
163
+
164
+ [`DbrxModel`]. It is used to instantiate a Dbrx model according to the
165
+ specified arguments, defining the model architecture.
166
+
167
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
168
+ documentation from [`PretrainedConfig`] for more information.
169
+
170
+
171
+ Args:
172
+ d_model (`int`, *optional*, defaults to 6144):
173
+ Dimensionality of the embeddings and hidden states.
174
+ n_heads (`int`, *optional*, defaults to 48):
175
+ Number of attention heads for each attention layer in the Transformer encoder.
176
+ n_layers (`int`, *optional*, defaults to 40):
177
+ Number of hidden layers in the Transformer encoder.
178
+ max_seq_len (`int`, *optional*, defaults to 32768):
179
+ The maximum sequence length of the model.
180
+ vocab_size (`int`, *optional*, defaults to 100352):
181
+ Vocabulary size of the Dbrx model. Defines the maximum number of different tokens that can be represented by
182
+ the `inputs_ids` passed when calling [`DbrxModel`].
183
+ resid_pdrop (`float`, *optional*, defaults to 0.0):
184
+ The dropout probability applied to the attention output before combining with residual.
185
+ emb_pdrop (`float`, *optional*, defaults to 0.0):
186
+ The dropout probability for the embedding layer.
187
+ attn_config (`dict`, *optional*):
188
+ A dictionary used to configure the model's attention module.
189
+ ffn_config (`dict`, *optional*):
190
+ A dictionary used to configure the model's FFN module.
191
+ use_cache (`bool`, *optional*, defaults to `False`):
192
+ Whether or not the model should return the last key/values attentions (not used by all models).
193
+ initializer_range (`float`, *optional*, defaults to 0.02):
194
+ The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
195
+ output_router_logits (`bool`, *optional*, defaults to `False`):
196
+ Whether or not the router logits should be returned by the model. Enabling this will also
197
+ allow the model to output the auxiliary loss. See [here]() for more details
198
+ router_aux_loss_coef (`float`, *optional*, defaults to 0.001):
199
+ The aux loss factor for the total loss.
200
+
201
+
202
+ Example:
203
+ ```python
204
+ >>> from transformers import DbrxConfig, DbrxModel
205
+
206
+ >>> # Initializing a Dbrx configuration
207
+ >>> configuration = DbrxConfig()
208
+
209
+ >>> # Initializing a model (with random weights) from the configuration
210
+ >>> model = DbrxModel(configuration)
211
+
212
+ >>> # Accessing the model configuration
213
+ >>> configuration = model.config
214
+ ```
215
+ """
216
+
217
+ model_type = "dbrx"
218
+ attribute_map = {
219
+ "num_attention_heads": "n_heads",
220
+ "hidden_size": "d_model",
221
+ "num_hidden_layers": "n_layers",
222
+ "max_position_embeddings": "max_seq_len",
223
+ }
224
+
225
+ def __init__(
226
+ self,
227
+ d_model: int = 2048,
228
+ n_heads: int = 16,
229
+ n_layers: int = 24,
230
+ max_seq_len: int = 2048,
231
+ vocab_size: int = 32000,
232
+ resid_pdrop: float = 0.0,
233
+ emb_pdrop: float = 0.0,
234
+ attn_config: Optional[DbrxAttentionConfig] = None,
235
+ ffn_config: Optional[DbrxFFNConfig] = None,
236
+ use_cache: bool = True,
237
+ initializer_range: float = 0.02,
238
+ output_router_logits: bool = False,
239
+ router_aux_loss_coef: float = 0.05,
240
+ **kwargs: Any,
241
+ ):
242
+ if attn_config is None:
243
+ self.attn_config = DbrxAttentionConfig()
244
+ elif isinstance(attn_config, dict):
245
+ self.attn_config = DbrxAttentionConfig(**attn_config)
246
+ else:
247
+ self.attn_config = attn_config
248
+
249
+ if ffn_config is None:
250
+ self.ffn_config = DbrxFFNConfig()
251
+ elif isinstance(ffn_config, dict):
252
+ self.ffn_config = DbrxFFNConfig(**ffn_config)
253
+ else:
254
+ self.ffn_config = ffn_config
255
+
256
+ self.d_model = d_model
257
+ self.n_heads = n_heads
258
+ self.n_layers = n_layers
259
+ self.max_seq_len = max_seq_len
260
+ self.vocab_size = vocab_size
261
+ self.resid_pdrop = resid_pdrop
262
+ self.emb_pdrop = emb_pdrop
263
+ self.use_cache = use_cache
264
+ self.initializer_range = initializer_range
265
+ self.output_router_logits = output_router_logits
266
+ self.router_aux_loss_coef = router_aux_loss_coef
267
+
268
+ tie_word_embeddings = kwargs.pop("tie_word_embeddings", False)
269
+ if tie_word_embeddings:
270
+ raise ValueError(
271
+ "tie_word_embeddings is not supported for Dbrx models."
272
+ )
273
+
274
+ super().__init__(
275
+ tie_word_embeddings=tie_word_embeddings,
276
+ **kwargs,
277
+ )