A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Model] Snowflake arctic model implementation (#4652)
Co-authored-by: Dash Desai <1723932+iamontheinet@users.noreply.github.com> Co-authored-by: Aurick Qiao <qiao@aurick.net> Co-authored-by: Aurick Qiao <aurick.qiao@snowflake.com> Co-authored-by: Aurick Qiao <aurickq@users.noreply.github.com> Co-authored-by: Cody Yu <hao.yu.cody@gmail.com>
Hao Zhang committed
May 9, 2024 at 15:37 UTC
ebce310b7433e050086f52ca48571807df467f50
8 files changed
+1042
-48
examples/offline_inference_arctic.py
new
+26
@@ -0,0 +1,26 @@
1
+from vllm import LLM, SamplingParams
2
+
3
+# Sample prompts.
4
+prompts = [
5
+ "Hello, my name is",
6
+ "The president of the United States is",
7
+ "The capital of France is",
8
+ "The future of AI is",
9
+]
10
+# Create a sampling params object.
11
+sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
12
+
13
+# Create an LLM.
14
+llm = LLM(model="snowflake/snowflake-arctic-instruct",
15
+ quantization="deepspeedfp",
16
+ tensor_parallel_size=8,
17
+ trust_remote_code=True)
18
+# Generate texts from the prompts. The output is a list of RequestOutput objects
19
+# that contain the prompt, generated text, and other information.
20
+
21
+outputs = llm.generate(prompts, sampling_params)
22
+# Print the outputs.
23
+for output in outputs:
24
+ prompt = output.prompt
25
+ generated_text = output.outputs[0].text
26
+ print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
vllm/model_executor/layers/fused_moe/__init__.py
+3
-1
@@ -1,7 +1,9 @@
1
from vllm.model_executor.layers.fused_moe.fused_moe import (
2
- fused_moe, get_config_file_name)
2
+ fused_experts, fused_moe, fused_topk, get_config_file_name)
3
4
__all__ = [
5
"fused_moe",
6
+ "fused_topk",
7
+ "fused_experts",
8
"get_config_file_name",
9
]
vllm/model_executor/layers/fused_moe/fused_moe.py
+90
-47
@@ -308,60 +308,16 @@ def get_moe_configs(E: int, N: int,
308
return None
309
310
311
-def fused_moe(
311
+def fused_topk(
312
hidden_states: torch.Tensor,
313
- w1: torch.Tensor,
314
- w2: torch.Tensor,
313
gating_output: torch.Tensor,
314
topk: int,
315
renormalize: bool,
318
- inplace: bool = False,
319
- override_config: Optional[Dict[str, Any]] = None,
320
- use_fp8: bool = False,
321
- w1_scale: Optional[torch.Tensor] = None,
322
- w2_scale: Optional[torch.Tensor] = None,
323
- a1_scale: Optional[torch.Tensor] = None,
324
- a2_scale: Optional[torch.Tensor] = None,
325
-) -> torch.Tensor:
326
- """
327
- This function computes a Mixture of Experts (MoE) layer using two sets of
328
- weights, w1 and w2, and top-k gating mechanism.
329
-
330
- Parameters:
331
- - hidden_states (torch.Tensor): The input tensor to the MoE layer.
332
- - w1 (torch.Tensor): The first set of expert weights.
333
- - w2 (torch.Tensor): The second set of expert weights.
334
- - gating_output (torch.Tensor): The output of the gating operation
335
- (before softmax).
336
- - topk (int): The number of top-k experts to select.
337
- - renormalize (bool): If True, renormalize the top-k weights to sum to 1.
338
- - inplace (bool): If True, perform the operation in-place.
339
- Defaults to False.
340
- - override_config (Optional[Dict[str, Any]]): Optional override
341
- for the kernel configuration.
342
- - use_fp8 (bool): If True, use fp8 arithmetic to compute the inner
343
- products for w1 and w2. Defaults to False.
344
- - w1_scale (Optional[torch.Tensor]): Optional scale to be used for
345
- w1.
346
- - w2_scale (Optional[torch.Tensor]): Optional scale to be used for
347
- w2.
348
-
349
- Returns:
350
- - torch.Tensor: The output tensor after applying the MoE layer.
351
- """
352
- # Check constraints.
316
+):
317
assert hidden_states.shape[0] == gating_output.shape[0], (
318
"Number of tokens mismatch")
355
- assert hidden_states.shape[1] == w1.shape[2], "Hidden size mismatch"
356
- assert gating_output.shape[1] == w1.shape[0], "Number of experts mismatch"
357
- assert hidden_states.is_contiguous(), "Hidden_states must be contiguous"
358
- assert w1.is_contiguous(), "Expert weights1 must be contiguous"
359
- assert w2.is_contiguous(), "Expert weights2 must be contiguous"
360
- assert hidden_states.dtype in [
361
- torch.float32, torch.float16, torch.bfloat16
362
- ]
319
+
320
M, _ = hidden_states.shape
364
- E, N, _ = w1.shape
321
322
if is_hip():
323
# The MoE kernels are not yet supported on ROCm.
@@ -393,6 +349,33 @@ def fused_moe(
349
del token_expert_indicies # Not used. Will be used in the future.
350
if renormalize:
351
topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True)
352
+ return topk_weights, topk_ids
353
+
354
+
355
+def fused_experts(hidden_states: torch.Tensor,
356
+ w1: torch.Tensor,
357
+ w2: torch.Tensor,
358
+ topk_weights: torch.Tensor,
359
+ topk_ids: torch.Tensor,
360
+ inplace: bool = False,
361
+ override_config: Optional[Dict[str, Any]] = None,
362
+ use_fp8: bool = False,
363
+ w1_scale: Optional[torch.Tensor] = None,
364
+ w2_scale: Optional[torch.Tensor] = None,
365
+ a1_scale: Optional[torch.Tensor] = None,
366
+ a2_scale: Optional[torch.Tensor] = None):
367
+ # Check constraints.
368
+ assert hidden_states.shape[1] == w1.shape[2], "Hidden size mismatch"
369
+ assert topk_weights.shape == topk_ids.shape, "topk shape mismatch"
370
+ assert hidden_states.is_contiguous(), "Hidden_states must be contiguous"
371
+ assert w1.is_contiguous(), "Expert weights1 must be contiguous"
372
+ assert w2.is_contiguous(), "Expert weights2 must be contiguous"
373
+ assert hidden_states.dtype in [
374
+ torch.float32, torch.float16, torch.bfloat16
375
+ ]
376
+
377
+ M, _ = hidden_states.shape
378
+ E, N, _ = w1.shape
379
380
if override_config:
381
config = override_config
@@ -477,3 +460,63 @@ def fused_moe(
460
out=hidden_states)
461
return torch.sum(intermediate_cache3.view(*intermediate_cache3.shape),
462
dim=1)
463
+
464
+
465
+def fused_moe(
466
+ hidden_states: torch.Tensor,
467
+ w1: torch.Tensor,
468
+ w2: torch.Tensor,
469
+ gating_output: torch.Tensor,
470
+ topk: int,
471
+ renormalize: bool,
472
+ inplace: bool = False,
473
+ override_config: Optional[Dict[str, Any]] = None,
474
+ use_fp8: bool = False,
475
+ w1_scale: Optional[torch.Tensor] = None,
476
+ w2_scale: Optional[torch.Tensor] = None,
477
+ a1_scale: Optional[torch.Tensor] = None,
478
+ a2_scale: Optional[torch.Tensor] = None,
479
+) -> torch.Tensor:
480
+ """
481
+ This function computes a Mixture of Experts (MoE) layer using two sets of
482
+ weights, w1 and w2, and top-k gating mechanism.
483
+
484
+ Parameters:
485
+ - hidden_states (torch.Tensor): The input tensor to the MoE layer.
486
+ - w1 (torch.Tensor): The first set of expert weights.
487
+ - w2 (torch.Tensor): The second set of expert weights.
488
+ - gating_output (torch.Tensor): The output of the gating operation
489
+ (before softmax).
490
+ - topk (int): The number of top-k experts to select.
491
+ - renormalize (bool): If True, renormalize the top-k weights to sum to 1.
492
+ - inplace (bool): If True, perform the operation in-place.
493
+ Defaults to False.
494
+ - override_config (Optional[Dict[str, Any]]): Optional override
495
+ for the kernel configuration.
496
+ - use_fp8 (bool): If True, use fp8 arithmetic to compute the inner
497
+ products for w1 and w2. Defaults to False.
498
+ - w1_scale (Optional[torch.Tensor]): Optional scale to be used for
499
+ w1.
500
+ - w2_scale (Optional[torch.Tensor]): Optional scale to be used for
501
+ w2.
502
+
503
+ Returns:
504
+ - torch.Tensor: The output tensor after applying the MoE layer.
505
+ """
506
+ # Check constraints.
507
+ assert gating_output.shape[1] == w1.shape[0], "Number of experts mismatch"
508
+
509
+ topk_weights, topk_ids = fused_topk(hidden_states, gating_output, topk,
510
+ renormalize)
511
+ return fused_experts(hidden_states,
512
+ w1,
513
+ w2,
514
+ topk_weights,
515
+ topk_ids,
516
+ inplace=inplace,
517
+ override_config=override_config,
518
+ use_fp8=use_fp8,
519
+ w1_scale=w1_scale,
520
+ w2_scale=w2_scale,
521
+ a1_scale=a1_scale,
522
+ a2_scale=a2_scale)
vllm/model_executor/layers/quantization/__init__.py
+3
@@ -4,6 +4,8 @@ from vllm.model_executor.layers.quantization.aqlm import AQLMConfig
4
from vllm.model_executor.layers.quantization.awq import AWQConfig
5
from vllm.model_executor.layers.quantization.base_config import (
6
QuantizationConfig)
7
+from vllm.model_executor.layers.quantization.deepspeedfp import (
8
+ DeepSpeedFPConfig)
9
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
10
from vllm.model_executor.layers.quantization.gptq import GPTQConfig
11
from vllm.model_executor.layers.quantization.gptq_marlin import (
@@ -19,6 +21,7 @@ QUANTIZATION_METHODS: Dict[str, Type[QuantizationConfig]] = {
21
"squeezellm": SqueezeLLMConfig,
22
"gptq_marlin": GPTQMarlinConfig,
23
"marlin": MarlinConfig,
24
+ "deepspeedfp": DeepSpeedFPConfig
25
}
26
27
vllm/model_executor/layers/quantization/deepspeedfp.py
new
+194
@@ -0,0 +1,194 @@
1
+from typing import Any, Dict, List, Optional
2
+
3
+import torch
4
+import torch.nn as nn
5
+import torch.nn.functional as F
6
+
7
+from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
8
+from vllm.model_executor.layers.quantization.base_config import (
9
+ QuantizationConfig)
10
+from vllm.model_executor.utils import set_weight_attrs
11
+
12
+
13
+class DeepSpeedFPConfig(QuantizationConfig):
14
+ """Config for DeepSpeed FP quantizer. It supports fp6 and fp8.
15
+
16
+ Args:
17
+ weight_bits: the target quantization bits, 6 or 8.
18
+ group_size: group size for quantizaiton, default to 128.
19
+ """
20
+
21
+ def __init__(
22
+ self,
23
+ weight_bits: int = 8,
24
+ group_size: int = 512,
25
+ ) -> None:
26
+ self.weight_bits = weight_bits
27
+ self.group_size = group_size
28
+ self.valid_types = [torch.bfloat16, torch.float16]
29
+
30
+ if self.weight_bits not in (6, 8):
31
+ raise ValueError(
32
+ "Currently, only 6-bit or 8-bit weight quantization are "
33
+ f"supported for DeepSpeed FP quantizaiton, but got "
34
+ f"{self.weight_bits} bits.")
35
+
36
+ def __repr__(self) -> str:
37
+ return (f"DeepSpeedFPConfig(weight_bits={self.weight_bits}), "
38
+ f"group_size={self.group_size}")
39
+
40
+ @classmethod
41
+ def get_name(cls) -> str:
42
+ return "DeepSpeedFP"
43
+
44
+ @classmethod
45
+ def from_config(cls, config: Dict[str, Any]) -> "DeepSpeedFPConfig":
46
+ weight_bits = cls.get_from_keys(config, ["bits"])
47
+ group_size = cls.get_from_keys(config, ["group_size"])
48
+ return cls(weight_bits=weight_bits, group_size=group_size)
49
+
50
+ def get_linear_method(self) -> "DeepSpeedFPLinearMethod":
51
+ return DeepSpeedFPLinearMethod(self)
52
+
53
+ def get_scaled_act_names(self) -> List[str]:
54
+ return []
55
+
56
+ @classmethod
57
+ def get_supported_act_dtypes(cls) -> List[torch.dtype]:
58
+ return [torch.half, torch.bfloat16]
59
+
60
+ @classmethod
61
+ # Need to figure it out
62
+ def get_min_capability(cls) -> int:
63
+ return 60
64
+
65
+ @staticmethod
66
+ def get_config_filenames() -> List[str]:
67
+ return [
68
+ "quant_config.json",
69
+ "quantize_config.json",
70
+ ]
71
+
72
+ def get_quant_method(
73
+ self,
74
+ layer: torch.nn.Module) -> Optional["DeepSpeedFPLinearMethod"]:
75
+ if isinstance(layer, LinearBase):
76
+ return DeepSpeedFPLinearMethod(self)
77
+ return None
78
+
79
+
80
+class DeepSpeedFPLinearMethod(LinearMethodBase):
81
+ """Linear method for DeepSpeedFP quantizer.
82
+
83
+ Args:
84
+ quant_config: the DeepSpeedFP quantization config.
85
+ """
86
+
87
+ def __init__(self, quant_config: DeepSpeedFPConfig):
88
+ self.quant_config = quant_config
89
+ self.weight = None
90
+
91
+ def create_weights(self,
92
+ layer: torch.nn.Module,
93
+ input_size_per_partition: int,
94
+ output_partition_sizes: List[int],
95
+ input_size: int,
96
+ output_size: int,
97
+ params_dtype: torch.dtype,
98
+ weight_loader=None,
99
+ **extra_weight_attrs):
100
+ del output_size
101
+ del input_size
102
+ output_size_per_partition = sum(output_partition_sizes)
103
+ weight = DeepSpeedFPParameter(
104
+ torch.Size((output_size_per_partition, input_size_per_partition)),
105
+ params_dtype=params_dtype,
106
+ quant_config=self.quant_config,
107
+ )
108
+ set_weight_attrs(weight, {
109
+ "input_dim": 1,
110
+ "output_dim": 0,
111
+ })
112
+ layer.register_parameter("weight", weight)
113
+
114
+ def quant_weight_loader(param, loaded_weight, *args, **kwargs):
115
+ # Calls the original weight loader (if any), quantizes the result,
116
+ # and then loads the quantized parameter.
117
+ if weight_loader is not None:
118
+ orig_param_data = param.data
119
+ param.data = param.ds_dequantize()
120
+ weight_loader(param, loaded_weight, *args, **kwargs)
121
+ param.data, loaded_weight = orig_param_data, param.data
122
+ param.ds_quantize_(loaded_weight.cuda())
123
+
124
+ extra_weight_attrs["weight_loader"] = quant_weight_loader
125
+ set_weight_attrs(weight, extra_weight_attrs)
126
+
127
+ def apply(self,
128
+ layer: torch.nn.Module,
129
+ x: torch.Tensor,
130
+ bias: Optional[torch.Tensor] = None) -> torch.Tensor:
131
+ weight = layer.weight
132
+ y = weight.ds_dequantize()
133
+ return F.linear(x, y, bias)
134
+
135
+
136
+class DeepSpeedFPParameter(nn.Parameter):
137
+ """
138
+ DeepSpeedFP quantized parameter class that implements fp8/fp6
139
+ quantization deepspeed. Weights are stored in quantized form on
140
+ GPUs, and can be dequantized on-the-fly when needed by the model.
141
+ """
142
+
143
+ def __new__(cls, orig_shape: torch.Size, params_dtype: torch.dtype,
144
+ quant_config: DeepSpeedFPConfig):
145
+ try:
146
+ import deepspeed
147
+ if deepspeed.__version__ < "0.14.2":
148
+ raise ImportError("deepspeed version is wrong. Please "
149
+ "install deepspeed>=0.14.2.")
150
+ from deepspeed.ops.fp_quantizer import FP_Quantize
151
+ except ImportError as err:
152
+ raise ImportError("Please install deepspeed>=0.14.2 via "
153
+ "`pip install deepspeed>=0.14.2` to use "
154
+ "deepspeedfp quantizer.") from err
155
+ data = torch.empty((
156
+ orig_shape.numel() // quant_config.group_size,
157
+ quant_config.group_size * quant_config.weight_bits // 8 + 4,
158
+ ),
159
+ dtype=torch.int8)
160
+ self = torch.Tensor._make_subclass(cls, data, data.requires_grad)
161
+ self.orig_shape = orig_shape
162
+ self.quant_config = quant_config
163
+ self.fp_quantizer = FP_Quantize(group_size=quant_config.group_size)
164
+ self.fp_quantizer.orig_shape = orig_shape
165
+ self.fp_quantizer.orig_dtype = params_dtype
166
+ return self
167
+
168
+ def ds_quantize_(self, tensor: torch.Tensor):
169
+ assert tensor.device.type == "cuda" and tensor.dtype != torch.int8
170
+ return self.data.copy_(
171
+ self.fp_quantizer.quantize(
172
+ tensor.data,
173
+ q_bits=self.quant_config.weight_bits,
174
+ ))
175
+
176
+ def ds_dequantize(self, fp_out=None) -> torch.Tensor:
177
+ """
178
+ Return a tensor containing the dequantized weights of this parameter.
179
+ """
180
+ assert self.data.device.type == "cuda" and self.data.dtype == torch.int8
181
+ return self.fp_quantizer.dequantize(
182
+ self.data, fp_out=fp_out, q_bits=self.quant_config.weight_bits)
183
+
184
+ def ds_selective_dequantize(self, indices, fp_out=None) -> torch.Tensor:
185
+ """
186
+ Return a tensor where only the weights at `indices` are dequantized
187
+ (to save HBM -> SRAM bandwidth).
188
+ """
189
+ assert self.data.device.type == "cuda" and self.data.dtype == torch.int8
190
+ return self.fp_quantizer.selective_dequantize(
191
+ self.data,
192
+ indices,
193
+ fp_out=fp_out,
194
+ q_bits=self.quant_config.weight_bits)
vllm/model_executor/models/__init__.py
+1
@@ -54,6 +54,7 @@ _MODELS = {
54
"StableLMEpochForCausalLM": ("stablelm", "StablelmForCausalLM"),
55
"StableLmForCausalLM": ("stablelm", "StablelmForCausalLM"),
56
"Starcoder2ForCausalLM": ("starcoder2", "Starcoder2ForCausalLM"),
57
+ "ArcticForCausalLM": ("arctic", "ArcticForCausalLM"),
58
"XverseForCausalLM": ("xverse", "XverseForCausalLM"),
59
}
60
vllm/model_executor/models/arctic.py
new
+521
@@ -0,0 +1,521 @@
1
+"""Inference-only Snowflake Arctic model."""
2
+from typing import Iterable, List, Optional, Tuple
3
+
4
+import torch
5
+from torch import nn
6
+
7
+from vllm.attention import Attention, AttentionMetadata
8
+from vllm.distributed import (get_tensor_model_parallel_rank,
9
+ get_tensor_model_parallel_world_size,
10
+ tensor_model_parallel_all_reduce)
11
+from vllm.logger import init_logger
12
+from vllm.model_executor.layers.activation import SiluAndMul
13
+from vllm.model_executor.layers.fused_moe import fused_experts, fused_topk
14
+from vllm.model_executor.layers.layernorm import RMSNorm
15
+from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
16
+ QKVParallelLinear,
17
+ ReplicatedLinear,
18
+ RowParallelLinear)
19
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
20
+from vllm.model_executor.layers.quantization.base_config import (
21
+ QuantizationConfig)
22
+from vllm.model_executor.layers.quantization.deepspeedfp import (
23
+ DeepSpeedFPConfig, DeepSpeedFPParameter)
24
+from vllm.model_executor.layers.rotary_embedding import get_rope
25
+from vllm.model_executor.layers.sampler import Sampler
26
+from vllm.model_executor.layers.vocab_parallel_embedding import (
27
+ ParallelLMHead, VocabParallelEmbedding)
28
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
29
+from vllm.model_executor.sampling_metadata import SamplingMetadata
30
+from vllm.model_executor.utils import set_weight_attrs
31
+from vllm.sequence import SamplerOutput
32
+from vllm.transformers_utils.configs.arctic import ArcticConfig
33
+
34
+logger = init_logger(__name__)
35
+
36
+
37
+class ArcticMLP(nn.Module):
38
+
39
+ def __init__(self,
40
+ config: ArcticConfig,
41
+ layer_id: int,
42
+ expert_id: int = -1,
43
+ is_residual_mlp: bool = False,
44
+ quant_config: Optional[QuantizationConfig] = None,
45
+ reduce_results: bool = True):
46
+ super(ArcticMLP, self).__init__()
47
+ self.hidden_size = config.hidden_size
48
+ self.expert_id = expert_id
49
+ self.layer_id = layer_id
50
+
51
+ self.ffn_dim = config.intermediate_size if not is_residual_mlp \
52
+ else self.hidden_size
53
+
54
+ self.w13 = MergedColumnParallelLinear(self.hidden_size,
55
+ [self.ffn_dim] * 2,
56
+ bias=False,
57
+ quant_config=quant_config)
58
+ self.w2 = RowParallelLinear(self.ffn_dim,
59
+ self.hidden_size,
60
+ bias=False,
61
+ reduce_results=reduce_results,
62
+ quant_config=quant_config)
63
+ if config.hidden_act != "silu":
64
+ raise ValueError(f"Unsupported activation: {config.hidden_act}. "
65
+ "Only silu is supported for now.")
66
+ self.act_fn = SiluAndMul()
67
+
68
+ def forward(self, hidden_states):
69
+ gate_up, _ = self.w13(hidden_states)
70
+ hidden_states = self.act_fn(gate_up)
71
+ hidden_states, _ = self.w2(hidden_states)
72
+ return hidden_states
73
+
74
+
75
+class ArcticMoE(nn.Module):
76
+ """
77
+ Model-parallel implementation of Arctic MoE Layer.
78
+ """
79
+
80
+ def __init__(self,
81
+ config: ArcticConfig,
82
+ layer_id: int,
83
+ tp_size: Optional[int] = None,
84
+ params_dtype: Optional[torch.dtype] = None,
85
+ quant_config: Optional[QuantizationConfig] = None,
86
+ reduce_results: bool = True):
87
+ super(ArcticMoE, self).__init__()
88
+
89
+ self.tp_size = tp_size or get_tensor_model_parallel_world_size()
90
+ self.hidden_size = config.hidden_size
91
+ self.num_experts = config.num_local_experts
92
+ self.layer_id = layer_id
93
+ self.top_k = config.num_experts_per_tok
94
+ self.intermediate_size = config.intermediate_size // self.tp_size
95
+
96
+ self.is_moe_layer = (layer_id + 1) % config.moe_layer_frequency == 0
97
+ self.is_quant = isinstance(quant_config, DeepSpeedFPConfig)
98
+ self.reduce_results = reduce_results
99
+ # Some other parameters
100
+ if params_dtype is None:
101
+ params_dtype = torch.get_default_dtype()
102
+ self.params_dtype = params_dtype
103
+
104
+ if not self.is_moe_layer:
105
+ self.mlp = ArcticMLP(config,
106
+ layer_id=layer_id,
107
+ quant_config=quant_config,
108
+ reduce_results=reduce_results)
109
+ else:
110
+ self.gate = ReplicatedLinear(self.hidden_size,
111
+ self.num_experts,
112
+ bias=False,
113
+ params_dtype=self.params_dtype,
114
+ quant_config=quant_config)
115
+ if self.is_quant:
116
+ self.ws = DeepSpeedFPParameter(
117
+ torch.Size((self.num_experts, 2 * self.intermediate_size,
118
+ self.hidden_size)),
119
+ params_dtype=params_dtype,
120
+ quant_config=quant_config,
121
+ )
122
+ self.w2s = DeepSpeedFPParameter(
123
+ torch.Size((self.num_experts, self.hidden_size,
124
+ self.intermediate_size)),
125
+ params_dtype=params_dtype,
126
+ quant_config=quant_config,
127
+ )
128
+ else:
129
+ self.ws = nn.Parameter(
130
+ torch.empty(self.num_experts,
131
+ 2 * self.intermediate_size,
132
+ self.hidden_size,
133
+ device="cuda",
134
+ dtype=self.params_dtype))
135
+ self.w2s = nn.Parameter(
136
+ torch.empty(self.num_experts,
137
+ self.hidden_size,
138
+ self.intermediate_size,
139
+ device="cuda",
140
+ dtype=self.params_dtype))
141
+ set_weight_attrs(self.ws, {
142
+ "weight_loader": self.weight_loader,
143
+ })
144
+ set_weight_attrs(self.w2s, {
145
+ "weight_loader": self.weight_loader,
146
+ })
147
+
148
+ def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor,
149
+ weight_name: str, expert_id: int):
150
+ tp_rank = get_tensor_model_parallel_rank()
151
+ param_data = param.ds_dequantize() if self.is_quant else param.data
152
+ shard_size = self.intermediate_size
153
+ shard = slice(tp_rank * shard_size, (tp_rank + 1) * shard_size)
154
+ if weight_name.endswith("w1.weight"):
155
+ param_data[expert_id, 0:shard_size, :] = loaded_weight[shard, :]
156
+ if weight_name.endswith("w3.weight"):
157
+ param_data[expert_id,
158
+ shard_size:2 * shard_size, :] = loaded_weight[shard, :]
159
+ if weight_name.endswith("w2.weight"):
160
+ param_data[expert_id, :, :] = loaded_weight[:, shard]
161
+ if self.is_quant:
162
+ param.ds_quantize_(param_data)
163
+
164
+ def local_moe_fused(self, hidden_states: torch.Tensor) -> torch.Tensor:
165
+ num_tokens, hidden_size = hidden_states.shape
166
+ hidden_states = hidden_states.view(-1, self.hidden_size)
167
+ # router_logits: (num_tokens, n_experts)
168
+ router_logits, _ = self.gate(hidden_states)
169
+ do_normalize = self.top_k > 1
170
+ topk_weights, topk_ids = fused_topk(hidden_states,
171
+ router_logits,
172
+ self.top_k,
173
+ renormalize=do_normalize)
174
+ # topk_ids: (num_tokens, k)
175
+ if self.is_quant:
176
+ if 2 * num_tokens <= self.num_experts:
177
+ # If much fewer tokens than experts, use selective dequantize.
178
+ ws_dequantized = self.ws.ds_selective_dequantize(
179
+ topk_ids.flatten())
180
+ w2s_dequantized = self.w2s.ds_selective_dequantize(
181
+ topk_ids.flatten())
182
+ # We gathered the experts to the tokens so update the mapping.
183
+ topk_ids = torch.arange(
184
+ 0,
185
+ topk_ids.numel(),
186
+ device=topk_ids.device,
187
+ ).reshape(topk_ids.shape)
188
+ else:
189
+ ws_dequantized = self.ws.ds_dequantize()
190
+ w2s_dequantized = self.w2s.ds_dequantize()
191
+
192
+ final_hidden_states = fused_experts(
193
+ hidden_states,
194
+ ws_dequantized if self.is_quant else self.ws,
195
+ w2s_dequantized if self.is_quant else self.w2s,
196
+ topk_weights,
197
+ topk_ids,
198
+ inplace=True)
199
+ if self.reduce_results and self.tp_size > 1:
200
+ final_hidden_states = tensor_model_parallel_all_reduce(
201
+ final_hidden_states)
202
+ return final_hidden_states.view(num_tokens, hidden_size)
203
+
204
+ def forward(self, hidden_states: torch.Tensor):
205
+ if self.is_moe_layer:
206
+ final_hidden_states = self.local_moe_fused(hidden_states)
207
+ else:
208
+ final_hidden_states = self.mlp(hidden_states)
209
+ return final_hidden_states
210
+
211
+
212
+class ArcticAttention(nn.Module):
213
+
214
+ def __init__(
215
+ self,
216
+ config: ArcticConfig,
217
+ layer_idx: Optional[int] = None,
218
+ quant_config: Optional[QuantizationConfig] = None,
219
+ ):
220
+ super().__init__()
221
+ self.config = config
222
+ self.layer_idx = layer_idx
223
+ self.hidden_size = config.hidden_size
224
+
225
+ tp_size = get_tensor_model_parallel_world_size()
226
+ self.total_num_heads = config.num_attention_heads
227
+ assert self.total_num_heads % tp_size == 0
228
+ self.num_heads = self.total_num_heads // tp_size
229
+ self.total_num_kv_heads = config.num_key_value_heads
230
+ if self.total_num_kv_heads >= tp_size:
231
+ assert self.total_num_kv_heads % tp_size == 0
232
+ else:
233
+ assert tp_size % self.total_num_kv_heads == 0
234
+ self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
235
+ self.head_dim = self.hidden_size // self.total_num_heads
236
+ self.q_size = self.num_heads * self.head_dim
237
+ self.kv_size = self.num_kv_heads * self.head_dim
238
+
239
+ self.max_position_embeddings = config.max_position_embeddings
240
+ self.rope_theta = config.rope_theta
241
+ self.scaling = self.head_dim**-0.5
242
+
243
+ self.qkv_proj = QKVParallelLinear(self.hidden_size,
244
+ self.head_dim,
245
+ self.total_num_heads,
246
+ self.total_num_kv_heads,
247
+ bias=False,
248
+ quant_config=quant_config)
249
+ self.o_proj = RowParallelLinear(
250
+ self.total_num_heads * self.head_dim,
251
+ self.hidden_size,
252
+ bias=False,
253
+ reduce_results=True,
254
+ quant_config=quant_config,
255
+ )
256
+
257
+ self.rotary_emb = get_rope(
258
+ self.head_dim,
259
+ rotary_dim=self.head_dim,
260
+ max_position=self.max_position_embeddings,
261
+ base=int(self.rope_theta),
262
+ is_neox_style=True,
263
+ )
264
+
265
+ self.attn = Attention(self.num_heads,
266
+ self.head_dim,
267
+ self.scaling,
268
+ num_kv_heads=self.num_kv_heads)
269
+
270
+ def forward(
271
+ self,
272
+ positions: torch.Tensor,
273
+ hidden_states: torch.Tensor,
274
+ kv_cache: torch.Tensor,
275
+ attn_metadata: AttentionMetadata,
276
+ ) -> torch.Tensor:
277
+ qkv, _ = self.qkv_proj(hidden_states)
278
+ q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
279
+ q, k = self.rotary_emb(positions, q, k)
280
+ attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
281
+ output, _ = self.o_proj(attn_output)
282
+ return output
283
+
284
+
285
+class ArcticDecoderLayer(nn.Module):
286
+
287
+ def __init__(
288
+ self,
289
+ config: ArcticConfig,
290
+ layer_idx: int,
291
+ quant_config: Optional[QuantizationConfig] = None,
292
+ ) -> None:
293
+ super().__init__()
294
+ self.layer_idx = layer_idx
295
+ self.hidden_size = config.hidden_size
296
+ is_moe_layer = (layer_idx + 1) % config.moe_layer_frequency == 0
297
+ self.use_residual = config.use_residual and is_moe_layer
298
+ self.self_attn = ArcticAttention(config,
299
+ layer_idx,
300
+ quant_config=quant_config)
301
+ self.block_sparse_moe = ArcticMoE(
302
+ config,
303
+ layer_id=layer_idx,
304
+ quant_config=quant_config,
305
+ reduce_results=(not self.use_residual))
306
+
307
+ self.input_layernorm = RMSNorm(config.hidden_size,
308
+ eps=config.rms_norm_eps)
309
+ self.post_attention_layernorm = RMSNorm(config.hidden_size,
310
+ eps=config.rms_norm_eps)
311
+
312
+ if self.use_residual:
313
+ self.residual_layernorm = RMSNorm(config.hidden_size,
314
+ eps=config.rms_norm_eps)
315
+ self.residual_mlp = ArcticMLP(config,
316
+ layer_id=layer_idx,
317
+ is_residual_mlp=True,
318
+ reduce_results=False)
319
+
320
+ def forward(
321
+ self,
322
+ positions: torch.Tensor,
323
+ hidden_states: torch.Tensor,
324
+ kv_cache: torch.Tensor,
325
+ attn_metadata: AttentionMetadata,
326
+ ) -> torch.Tensor:
327
+ residual_input = hidden_states
328
+ hidden_states = self.input_layernorm(hidden_states)
329
+ hidden_states = self.self_attn(
330
+ positions=positions,
331
+ hidden_states=hidden_states,
332
+ kv_cache=kv_cache,
333
+ attn_metadata=attn_metadata,
334
+ )
335
+ hidden_states = residual_input + hidden_states
336
+
337
+ residual_attn = hidden_states
338
+ if self.use_residual:
339
+ hidden_states = self.residual_layernorm(hidden_states)
340
+ hidden_states = self.residual_mlp(hidden_states)
341
+ residual_mlp = hidden_states
342
+ hidden_states = self.post_attention_layernorm(residual_input)
343
+ hidden_states = self.block_sparse_moe(hidden_states)
344
+ hidden_states = residual_mlp + hidden_states
345
+ hidden_states = tensor_model_parallel_all_reduce(hidden_states)
346
+ hidden_states = residual_attn + hidden_states
347
+ else:
348
+ hidden_states = self.post_attention_layernorm(hidden_states)
349
+ hidden_states = self.block_sparse_moe(hidden_states)
350
+ hidden_states = residual_attn + hidden_states
351
+ return hidden_states
352
+
353
+
354
+class ArcticModel(nn.Module):
355
+
356
+ def __init__(
357
+ self,
358
+ config: ArcticConfig,
359
+ quant_config: Optional[QuantizationConfig] = None,
360
+ ) -> None:
361
+ super().__init__()
362
+ self.padding_idx = config.pad_token_id
363
+ self.vocab_size = config.vocab_size
364
+ self.embed_tokens = VocabParallelEmbedding(
365
+ self.vocab_size,
366
+ config.hidden_size,
367
+ org_num_embeddings=self.vocab_size)
368
+ self.layers = nn.ModuleList([
369
+ ArcticDecoderLayer(config, layer_idx, quant_config=quant_config)
370
+ for layer_idx in range(config.num_hidden_layers)
371
+ ])
372
+ self._attn_implementation = config._attn_implementation
373
+ self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
374
+
375
+ def forward(
376
+ self,
377
+ input_ids: torch.Tensor,
378
+ positions: torch.Tensor,
379
+ kv_caches: List[torch.Tensor],
380
+ attn_metadata: AttentionMetadata,
381
+ ) -> torch.Tensor:
382
+ hidden_states = self.embed_tokens(input_ids)
383
+ for i in range(len(self.layers)):
384
+ layer = self.layers[i]
385
+ hidden_states = layer(positions, hidden_states, kv_caches[i],
386
+ attn_metadata)
387
+ hidden_states = self.norm(hidden_states)
388
+ return hidden_states
389
+
390
+
391
+class ArcticForCausalLM(nn.Module):
392
+
393
+ def __init__(self,
394
+ config: ArcticConfig,
395
+ quant_config: Optional[QuantizationConfig] = None,
396
+ **kwargs) -> None:
397
+ super().__init__()
398
+ self.config = config
399
+ self.model = ArcticModel(config, quant_config)
400
+ self.vocab_size = config.vocab_size
401
+ self.lm_head = ParallelLMHead(
402
+ self.vocab_size,
403
+ config.hidden_size,
404
+ )
405
+ self.num_experts = config.num_local_experts
406
+ self.num_experts_per_tok = config.num_experts_per_tok
407
+ self.unpadded_vocab_size = config.vocab_size
408
+ self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
409
+ config.vocab_size)
410
+ self.sampler = Sampler()
411
+
412
+ def forward(
413
+ self,
414
+ input_ids: torch.Tensor,
415
+ positions: torch.Tensor,
416
+ kv_caches: List[torch.Tensor],
417
+ attn_metadata: AttentionMetadata,
418
+ ) -> torch.Tensor:
419
+ hidden_states = self.model(input_ids, positions, kv_caches,
420
+ attn_metadata)
421
+ return hidden_states
422
+
423
+ def compute_logits(self, hidden_states: torch.Tensor,
424
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
425
+ logits = self.logits_processor(self.lm_head.weight, hidden_states,
426
+ sampling_metadata)
427
+ return logits
428
+
429
+ def sample(
430
+ self,
431
+ logits: Optional[torch.Tensor],
432
+ sampling_metadata: SamplingMetadata,
433
+ ) -> Optional[SamplerOutput]:
434
+ next_tokens = self.sampler(logits, sampling_metadata)
435
+ return next_tokens
436
+
437
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
438
+ stacked_params_mapping = [
439
+ # (param_name, shard_name, shard_id)
440
+ ("qkv_proj", "q_proj", "q"),
441
+ ("qkv_proj", "k_proj", "k"),
442
+ ("qkv_proj", "v_proj", "v"),
443
+ ]
444
+
445
+ mlp_params_mapping = []
446
+ expert_params_mapping = []
447
+ num_layers = self.config.num_hidden_layers
448
+
449
+ for layer in range(num_layers):
450
+ mlp_params_mapping.append(
451
+ (f"layers.{layer}.residual_mlp.w13.weight",
452
+ f"layers.{layer}.residual_mlp.w1.weight", 0))
453
+ mlp_params_mapping.append(
454
+ (f"layers.{layer}.residual_mlp.w13.weight",
455
+ f"layers.{layer}.residual_mlp.w3.weight", 1))
456
+ if layer % 2 == 0:
457
+ # MLP layers
458
+ mlp_params_mapping.append(
459
+ (f"layers.{layer}.block_sparse_moe.mlp.w13.weight",
460
+ f"layers.{layer}.block_sparse_moe.mlp.w1.weight", 0))
461
+ mlp_params_mapping.append(
462
+ (f"layers.{layer}.block_sparse_moe.mlp.w13.weight",
463
+ f"layers.{layer}.block_sparse_moe.mlp.w3.weight", 1))
464
+ else:
465
+ # MoE layers
466
+ for expert_id in range(self.config.num_local_experts):
467
+ expert_params_mapping.append(
468
+ ("ws", f"experts.{expert_id}.w1.weight", expert_id))
469
+ expert_params_mapping.append(
470
+ ("w2s", f"experts.{expert_id}.w2.weight", expert_id))
471
+ expert_params_mapping.append(
472
+ ("ws", f"experts.{expert_id}.w3.weight", expert_id))
473
+
474
+ params_dict = dict(self.named_parameters())
475
+
476
+ logger.info(
477
+ "It will take ~10 minutes loading from the 16-bit weights. "
478
+ "Alternatively, use the prequantized 8-bit weights of arctic "
479
+ "and set load-format to `sharded_state` will accelerate loading.")
480
+ for name, loaded_weight in weights:
481
+ for (param_name, weight_name, shard_id) in stacked_params_mapping:
482
+ if weight_name not in name:
483
+ continue
484
+ name = name.replace(weight_name, param_name)
485
+ # Skip loading extra bias for GPTQ models.
486
+ if name.endswith(".bias") and name not in params_dict:
487
+ continue
488
+ param = params_dict[name]
489
+ weight_loader = param.weight_loader
490
+ weight_loader(param, loaded_weight, shard_id)
491
+ break
492
+ else:
493
+ for param_name, weight_name, shard_id in mlp_params_mapping:
494
+ if weight_name not in name:
495
+ continue
496
+ name = name.replace(weight_name, param_name)
497
+ param = params_dict[name]
498
+ weight_loader = param.weight_loader
499
+ weight_loader(param, loaded_weight, shard_id)
500
+ break
501
+ else:
502
+ for param_name, weight_name, shard_id \
503
+ in expert_params_mapping:
504
+ if weight_name not in name:
505
+ continue
506
+ name = name.replace(weight_name, param_name)
507
+ param = params_dict[name]
508
+ weight_loader = param.weight_loader
509
+ weight_loader(param,
510
+ loaded_weight,
511
+ weight_name,
512
+ expert_id=shard_id)
513
+ break
514
+ else:
515
+ if name.endswith(".bias") and name not in params_dict:
516
+ continue
517
+ param = params_dict[name]
518
+
519
+ weight_loader = getattr(param, "weight_loader",
520
+ default_weight_loader)
521
+ weight_loader(param, loaded_weight)
vllm/transformers_utils/configs/arctic.py
new
+204
@@ -0,0 +1,204 @@
1
+# yapf: disable
2
+# ruff: noqa: E501
3
+# coding=utf-8
4
+# Copied from
5
+# https://huggingface.co/Snowflake/snowflake-arctic-instruct/blob/main/configuration_arctic.py
6
+""" Arctic model configuration"""
7
+
8
+from dataclasses import asdict, dataclass
9
+from typing import Any, Dict
10
+
11
+from transformers.configuration_utils import PretrainedConfig
12
+from transformers.utils import logging
13
+
14
+logger = logging.get_logger(__name__)
15
+
16
+ARCTIC_PRETRAINED_CONFIG_ARCHIVE_MAP = {
17
+ "arctic": "https://huggingface.co/Snowflake/snowflake-arctic-instruct/tree/main/config.json",
18
+}
19
+
20
+
21
+@dataclass
22
+class ArcticLoraConfig:
23
+ lora_r: int = 64
24
+ lora_alpha: float = 16
25
+ shard_base_weights: bool = False
26
+
27
+
28
+@dataclass
29
+class ArcticQuantizationConfig:
30
+ q_bits: int = 8
31
+ rounding: str = "nearest"
32
+ mantissa_bits: int = 3
33
+ group_size: int = 128
34
+
35
+
36
+class ArcticConfig(PretrainedConfig):
37
+ r"""
38
+ This is the configuration class to store the configuration of a [`ArcticModel`]. It is used to instantiate an
39
+ Arctic model according to the specified arguments, defining the model architecture. Instantiating a configuration
40
+ with the defaults will yield a similar configuration to that of the #TODO(rsamdani): add what model has the default config..
41
+
42
+
43
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
44
+ documentation from [`PretrainedConfig`] for more information.
45
+
46
+
47
+ Args:
48
+ vocab_size (`int`, *optional*, defaults to 32000):
49
+ Vocabulary size of the Arctic model. Defines the number of different tokens that can be represented by the
50
+ `inputs_ids` passed when calling [`ArcticModel`]
51
+ hidden_size (`int`, *optional*, defaults to 4096):
52
+ Dimension of the hidden representations.
53
+ intermediate_size (`int`, *optional*, defaults to 14336):
54
+ Dimension of the MLP representations.
55
+ num_hidden_layers (`int`, *optional*, defaults to 32):
56
+ Number of hidden layers in the Transformer encoder.
57
+ num_attention_heads (`int`, *optional*, defaults to 32):
58
+ Number of attention heads for each attention layer in the Transformer encoder.
59
+ num_key_value_heads (`int`, *optional*, defaults to 8):
60
+ This is the number of key_value heads that should be used to implement Grouped Query Attention. If
61
+ `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
62
+ `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When
63
+ converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
64
+ by meanpooling all the original heads within that group. For more details checkout [this
65
+ paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to `8`.
66
+ hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
67
+ The non-linear activation function (function or string) in the decoder.
68
+ max_position_embeddings (`int`, *optional*, defaults to `4096*32`):
69
+ The maximum sequence length that this model might ever be used with. Arctic's sliding window attention
70
+ allows sequence of up to 4096*32 tokens.
71
+ initializer_range (`float`, *optional*, defaults to 0.02):
72
+ The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
73
+ rms_norm_eps (`float`, *optional*, defaults to 1e-05):
74
+ The epsilon used by the rms normalization layers.
75
+ use_cache (`bool`, *optional*, defaults to `True`):
76
+ Whether or not the model should return the last key/values attentions (not used by all models). Only
77
+ relevant if `config.is_decoder=True`.
78
+ pad_token_id (`int`, *optional*):
79
+ The id of the padding token.
80
+ bos_token_id (`int`, *optional*, defaults to 1):
81
+ The id of the "beginning-of-sequence" token.
82
+ eos_token_id (`int`, *optional*, defaults to 2):
83
+ The id of the "end-of-sequence" token.
84
+ tie_word_embeddings (`bool`, *optional*, defaults to `False`):
85
+ Whether the model's input and output word embeddings should be tied.
86
+ rope_theta (`float`, *optional*, defaults to 1000000.0):
87
+ The base period of the RoPE embeddings.
88
+ sliding_window (`int`, *optional*):
89
+ Sliding window attention window size. If not specified, will default to `4096`.
90
+ attention_dropout (`float`, *optional*, defaults to 0.0):
91
+ The dropout ratio for the attention probabilities.
92
+ num_experts_per_tok (`int`, *optional*, defaults to 2):
93
+ The number of experts to root per-token, can be also interpreted as the `top-p` routing
94
+ parameter
95
+ num_local_experts (`int`, *optional*, defaults to 8):
96
+ Number of experts per Sparse MLP layer.
97
+ router_aux_loss_coef (`float`, *optional*, defaults to 0.001):
98
+ The aux loss factor for the total loss.
99
+
100
+ ```python
101
+ >>> from transformers import ArcticModel, ArcticConfig
102
+
103
+ >>> # Initializing a Arctic 7B style configuration TODO(rsamdani): verify which model does the default configuration correspond to.
104
+ >>> configuration = ArcticConfig()
105
+
106
+ >>> # Initializing a model from the Arctic 7B style configuration
107
+ >>> model = ArcticModel(configuration)
108
+
109
+ >>> # Accessing the model configuration
110
+ >>> configuration = model.config
111
+ ```"""
112
+
113
+ model_type = "arctic"
114
+ keys_to_ignore_at_inference = ["past_key_values"]
115
+
116
+ def __init__(
117
+ self,
118
+ vocab_size=32000,
119
+ hidden_size=4096,
120
+ intermediate_size=14336,
121
+ num_hidden_layers=32,
122
+ num_attention_heads=32,
123
+ num_key_value_heads=None,
124
+ hidden_act="silu",
125
+ max_position_embeddings=4096,
126
+ initializer_range=0.02,
127
+ rms_norm_eps=1e-5,
128
+ use_cache=True,
129
+ pad_token_id=None,
130
+ bos_token_id=1,
131
+ eos_token_id=2,
132
+ tie_word_embeddings=False,
133
+ rope_theta=1e6,
134
+ sliding_window=None,
135
+ attention_dropout=0.0,
136
+ num_experts_per_tok=1,
137
+ num_local_experts=8,
138
+ router_aux_loss_coef=0.001,
139
+ moe_layer_frequency=2,
140
+ parallel_attn_mlp_res=False,
141
+ moe_train_capacity_factor=1,
142
+ moe_eval_capacity_factor=1,
143
+ enable_expert_tensor_parallelism=False,
144
+ moe_min_capacity=0,
145
+ moe_token_dropping=True,
146
+ quantization=None,
147
+ **kwargs,
148
+ ):
149
+ self.vocab_size = vocab_size
150
+ self.max_position_embeddings = max_position_embeddings
151
+ self.hidden_size = hidden_size
152
+ self.intermediate_size = intermediate_size
153
+ self.num_hidden_layers = num_hidden_layers
154
+ self.num_attention_heads = num_attention_heads
155
+ self.sliding_window = sliding_window
156
+
157
+ # for backward compatibility
158
+ if num_key_value_heads is None:
159
+ num_key_value_heads = num_attention_heads
160
+
161
+ self.num_key_value_heads = num_key_value_heads
162
+ self.hidden_act = hidden_act
163
+ self.initializer_range = initializer_range
164
+ self.rms_norm_eps = rms_norm_eps
165
+ self.use_cache = use_cache
166
+ self.rope_theta = rope_theta
167
+ self.attention_dropout = attention_dropout
168
+
169
+ self.num_experts_per_tok = num_experts_per_tok
170
+ self.num_local_experts = num_local_experts
171
+ self.router_aux_loss_coef = router_aux_loss_coef
172
+ self.moe_layer_frequency = moe_layer_frequency
173
+ self.moe_train_capacity_factor = moe_train_capacity_factor
174
+ self.moe_eval_capacity_factor = moe_eval_capacity_factor
175
+ self.enable_expert_tensor_parallelism = enable_expert_tensor_parallelism
176
+ self.moe_min_capacity = moe_min_capacity
177
+ self.moe_token_dropping = moe_token_dropping
178
+ self.parallel_attn_mlp_res = parallel_attn_mlp_res
179
+ if isinstance(quantization, dict):
180
+ self.quantization = ArcticQuantizationConfig(**quantization)
181
+ else:
182
+ self.quantization = quantization
183
+
184
+ super().__init__(
185
+ pad_token_id=pad_token_id,
186
+ bos_token_id=bos_token_id,
187
+ eos_token_id=eos_token_id,
188
+ tie_word_embeddings=tie_word_embeddings,
189
+ **kwargs,
190
+ )
191
+
192
+ @classmethod
193
+ def from_dict(cls, config_dict: Dict[str, Any], **kwargs) -> "ArcticConfig":
194
+ result = super().from_dict(config_dict, **kwargs)
195
+ config = result[0] if isinstance(result, tuple) else result
196
+ if isinstance(config.quantization, dict):
197
+ config.quantization = ArcticQuantizationConfig(**config.quantization)
198
+ return result
199
+
200
+ def to_dict(self) -> Dict[str, Any]:
201
+ ret = super().to_dict()
202
+ if isinstance(ret["quantization"], ArcticQuantizationConfig):
203
+ ret["quantization"] = asdict(ret["quantization"])
204
+ return ret