@tduncan / Nvidia-Optimization-Stack-VLLM
Code Commits Issues Pulls Cloud Agent CI/CD
main
main
py 277 lines 13.5 KB
Raw
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 # SPDX-FileCopyrightText: Copyright 2025 NAVER Cloud HyperCLOVA team
4 #
5 # Copyright 2025 NAVER Cloud HyperCLOVA team. All rights reserved.
6 #
7 # Licensed under the Apache License, Version 2.0 (the "License");
8 # you may not use this file except in compliance with the License.
9 # You may obtain a copy of the License at
10 #
11 # http://www.apache.org/licenses/LICENSE-2.0
12 #
13 # Unless required by applicable law or agreed to in writing, software
14 # distributed under the License is distributed on an "AS IS" BASIS,
15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 # See the License for the specific language governing permissions and
17 # limitations under the License.
18 """HyperCLOVA X model configuration."""
19
20 from transformers.configuration_utils import PreTrainedConfig
21
22
23 class HyperCLOVAXConfig(PreTrainedConfig):
24 r"""This is the configuration class to store the configuration of a
25 [`HyperCLOVAXModel`]. It is used to instantiate a HyperCLOVAX model
26 according to the specified arguments, defining the model architecture.
27 Configuration objects inherit from [`PreTrainedConfig`] and can be used
28 to control the model outputs. Read the documentation from
29 [`PreTrainedConfig`] for more information.
30
31 Args:
32 vocab_size (`int`, *optional*, defaults to 32000):
33 Vocabulary size of the HyperCLOVAX model. Defines the number of
34 different tokens that can be represented by the `input_ids`
35 passed when calling [`HyperCLOVAXModel`]
36 hidden_size (`int`, *optional*, defaults to 4096):
37 Dimension of the hidden representations.
38 intermediate_size (`int`, *optional*, defaults to 11008):
39 Dimension of the MLP representations.
40 num_hidden_layers (`int`, *optional*, defaults to 32):
41 Number of hidden layers in the Transformer decoder.
42 num_attention_heads (`int`, *optional*, defaults to 32):
43 Number of attention heads for each attention layer in the
44 Transformer decoder.
45 num_key_value_heads (`int`, *optional*):
46 This is the number of key_value heads that should be used to
47 implement Grouped Query Attention. If
48 `num_key_value_heads=num_attention_heads`, the model will use
49 Multi Head Attention (MHA), if `num_key_value_heads=1` the model
50 will use Multi Query Attention (MQA) otherwise GQA is used. When
51 converting a multi-head checkpoint to a GQA checkpoint, each
52 group key and value head should be constructed by meanpooling all
53 the original heads within that group. For more details checkout
54 [this paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not
55 specified, will default to `num_attention_heads`.
56 hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
57 The non-linear activation function (function or string) in the
58 decoder.
59 max_position_embeddings (`int`, *optional*, defaults to 2048):
60 The maximum sequence length that this model might ever be used
61 with.
62 initializer_range (`float`, *optional*, defaults to 0.02):
63 The standard deviation of the truncated_normal_initializer for
64 initializing all weight matrices.
65 rms_norm_eps (`float`, *optional*, defaults to 1e-06):
66 The epsilon used by the rms normalization layers.
67 use_cache (`bool`, *optional*, defaults to `True`):
68 Whether or not the model should return the last key/values
69 attentions (not used by all models). Only relevant if
70 `config.is_decoder=True`.
71 pad_token_id (`int`, *optional*):
72 Padding token id.
73 bos_token_id (`int`, *optional*, defaults to 1):
74 Beginning of stream token id.
75 eos_token_id (`int`, *optional*, defaults to 2):
76 End of stream token id.
77 pretraining_tp (`int`, *optional*, defaults to 1):
78 Experimental feature. Tensor parallelism rank used during
79 pretraining. Please refer to [this document](https://huggingface.
80 co/docs/transformers/main/perf_train_gpu_many#tensor-parallelism)
81 to understand more about it. This value is necessary to ensure
82 exact reproducibility of the pretraining results. Please refer to
83 [this issue](https://github.com/pytorch/pytorch/issues/76232).
84 tie_word_embeddings (`bool`, *optional*, defaults to `False`):
85 Whether to tie weight embeddings
86 rope_theta (`float`, *optional*, defaults to 10000.0):
87 The base period of the RoPE embeddings.
88 rope_scaling (`Dict`, *optional*):
89 Dictionary containing the scaling configuration for the RoPE
90 embeddings. NOTE: if you apply new rope type and you expect the
91 model to work on longer `max_position_embeddings`, we recommend
92 you to update this value accordingly.
93 Expected contents:
94 `rope_type` (`str`):
95 The sub-variant of RoPE to use. Can be one of ['default',
96 'linear', 'dynamic', 'yarn', 'longrope', 'llama3'], with
97 'default' being the original RoPE implementation.
98 `factor` (`float`, *optional*):
99 Used with all rope types except 'default'. The scaling
100 factor to apply to the RoPE embeddings. In most scaling
101 types, a `factor` of x will enable the model to handle
102 sequences of length x * original maximum pre-trained
103 length.
104 `original_max_position_embeddings` (`int`, *optional*):
105 Used with 'dynamic', 'longrope' and 'llama3'. The
106 original max position embeddings used during pretraining.
107 `attention_factor` (`float`, *optional*):
108 Used with 'yarn' and 'longrope'. The scaling factor to be
109 applied on the attention computation. If unspecified, it
110 defaults to value recommended by the implementation, using
111 the `factor` field to infer the suggested value.
112 `beta_fast` (`float`, *optional*):
113 Only used with 'yarn'. Parameter to set the boundary for
114 extrapolation (only) in the linear ramp function. If
115 unspecified, it defaults to 32.
116 `beta_slow` (`float`, *optional*):
117 Only used with 'yarn'. Parameter to set the boundary for
118 interpolation (only) in the linear ramp function. If
119 unspecified, it defaults to 1.
120 `short_factor` (`List[float]`, *optional*):
121 Only used with 'longrope'. The scaling factor to be
122 applied to short contexts (<
123 `original_max_position_embeddings`). Must be a list of
124 numbers with the same length as the hidden size divided
125 by the number of attention heads divided by 2
126 `long_factor` (`List[float]`, *optional*):
127 Only used with 'longrope'. The scaling factor to be
128 applied to long contexts (<
129 `original_max_position_embeddings`). Must be a list of
130 numbers with the same length as the hidden size divided
131 by the number of attention heads divided by 2
132 `low_freq_factor` (`float`, *optional*):
133 Only used with 'llama3'. Scaling factor applied to low
134 frequency components of the RoPE
135 `high_freq_factor` (`float`, *optional*):
136 Only used with 'llama3'. Scaling factor applied to high
137 frequency components of the RoPE
138 attention_bias (`bool`, *optional*, defaults to `False`):
139 Whether to use a bias in the query, key, value and output
140 projection layers during self-attention.
141 attention_dropout (`float`, *optional*, defaults to 0.0):
142 The dropout ratio for the attention probabilities.
143 mlp_bias (`bool`, *optional*, defaults to `False`):
144 Whether to use a bias in up_proj, down_proj and gate_proj layers
145 in the MLP layers.
146 head_dim (`int`, *optional*):
147 The attention head dimension. If None, it will default to
148 hidden_size // num_heads
149 embedding_multiplier (`float`, *optional*, defaults to `None`):
150 Multiplier applied to the embedding weights. If `None`, it is
151 equivalent to `1.0`.
152 logits_scaling (`float`, *optional*, defaults to `None`):
153 Scaling factor for logits. If `None`, it is equivalent to `1.0`.
154 attention_multiplier (`float`, *optional*, defaults to `None`):
155 Multiplier applied to the attention weights. If `None`, it is
156 equivalent to `self.head_dim ** -0.5`.
157 residual_multiplier (`float`, *optional*, defaults to `None`):
158 Scaling factor for residual connections. If `None`, it is
159 equivalent to `1.0`.
160 use_post_norm (`bool`, *optional*, defaults to `True`):
161 Determines whether to apply Peri-Layer Normalization. Set to
162 False to disable this feature.
163 rope_parameters (`dict`, *optional*):
164 Dictionary containing the RoPE parameters used by vLLM's
165 `get_rope`. When provided, takes precedence over `rope_theta`
166 and `rope_scaling`. If `None`, it is derived from `rope_theta`
167 and `rope_scaling` automatically.
168
169 """
170
171 model_type = "hyperclovax"
172 keys_to_ignore_at_inference = ["past_key_values"]
173
174 def __init__(
175 self,
176 vocab_size=32000,
177 hidden_size=4096,
178 intermediate_size=11008,
179 num_hidden_layers=32,
180 num_attention_heads=32,
181 num_key_value_heads=None,
182 hidden_act="silu",
183 max_position_embeddings=2048,
184 initializer_range=0.02,
185 rms_norm_eps=1e-6,
186 use_cache=True,
187 pad_token_id=None,
188 bos_token_id=1,
189 eos_token_id=2,
190 pretraining_tp=1,
191 tie_word_embeddings=False,
192 rope_theta=10000.0,
193 rope_scaling=None,
194 attention_bias=False,
195 attention_dropout=0.0,
196 mlp_bias=False,
197 head_dim=None,
198 embedding_multiplier=None, # mup
199 logits_scaling=None, # mup
200 attention_multiplier=None, # mup
201 residual_multiplier=None, # mup
202 use_post_norm=True, # post-norm(peri-LN)
203 rope_parameters=None,
204 auto_map=None,
205 **kwargs,
206 ):
207 self.vocab_size = vocab_size
208 self.max_position_embeddings = max_position_embeddings
209 self.hidden_size = hidden_size
210 self.intermediate_size = intermediate_size
211 self.num_hidden_layers = num_hidden_layers
212 self.num_attention_heads = num_attention_heads
213
214 # for backward compatibility
215 if num_key_value_heads is None:
216 num_key_value_heads = num_attention_heads
217
218 self.num_key_value_heads = num_key_value_heads
219 self.hidden_act = hidden_act
220 self.initializer_range = initializer_range
221 self.rms_norm_eps = rms_norm_eps
222 self.pretraining_tp = pretraining_tp
223 self.use_cache = use_cache
224 self.rope_theta = rope_theta
225 self.rope_scaling = rope_scaling
226 self.attention_bias = attention_bias
227 self.attention_dropout = attention_dropout
228 self.mlp_bias = mlp_bias
229 self.head_dim = (
230 head_dim
231 if head_dim is not None
232 else self.hidden_size // self.num_attention_heads
233 )
234 # Derive rope_parameters for vLLM's get_rope() from rope_theta /
235 # rope_scaling, unless the caller already provided rope_parameters.
236 if rope_parameters is None:
237 if rope_scaling is not None:
238 # Shallow-copy to avoid mutating the caller's dict.
239 rope_parameters = dict(rope_scaling)
240 # BC: 'type' field -> 'rope_type', remove stale key.
241 if "type" in rope_parameters:
242 rope_parameters.setdefault("rope_type", rope_parameters.pop("type"))
243 else:
244 rope_parameters = {"rope_type": "default"}
245 if "rope_theta" not in rope_parameters:
246 rope_parameters["rope_theta"] = rope_theta
247 self.rope_parameters = rope_parameters
248
249 # BC: keep self.rope_scaling consistent for HF serialization.
250 if self.rope_scaling is not None and "type" in self.rope_scaling:
251 self.rope_scaling["rope_type"] = self.rope_scaling["type"]
252
253 # mup
254 self.embedding_multiplier = (
255 embedding_multiplier if embedding_multiplier is not None else 1.0
256 )
257 self.logits_scaling = logits_scaling if logits_scaling is not None else 1.0
258 self.attention_multiplier = (
259 attention_multiplier
260 if attention_multiplier is not None
261 else self.head_dim**-0.5
262 )
263 self.residual_multiplier = (
264 residual_multiplier if residual_multiplier is not None else 1.0
265 )
266
267 # post-norm (Peri-LN)
268 self.use_post_norm = use_post_norm
269
270 super().__init__(
271 pad_token_id=pad_token_id,
272 bos_token_id=bos_token_id,
273 eos_token_id=eos_token_id,
274 tie_word_embeddings=tie_word_embeddings,
275 auto_map=auto_map,
276 **kwargs,
277 )