@tduncan / Nvidia-Optimization-Stack-VLLM
Code Commits Issues Pulls Cloud Agent CI/CD
main
main
py 455 lines 20.8 KB
Raw
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 # ruff: noqa
5 # mypy: ignore-errors
6 # coding=utf-8
7 # Copyright 2026 The Qwen team, Alibaba Group and the HuggingFace Inc. team. All rights reserved.
8 #
9 # Licensed under the Apache License, Version 2.0 (the "License");
10 # you may not use this file except in compliance with the License.
11 # You may obtain a copy of the License at
12 #
13 # http://www.apache.org/licenses/LICENSE-2.0
14 #
15 # Unless required by applicable law or agreed to in writing, software
16 # distributed under the License is distributed on an "AS IS" BASIS,
17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18 # See the License for the specific language governing permissions and
19 # limitations under the License.
20 from transformers.configuration_utils import PreTrainedConfig
21 from transformers.modeling_rope_utils import rope_config_validation
22 from transformers.utils import logging
23
24 logger = logging.get_logger(__name__)
25
26
27 class Qwen3ASRAudioEncoderConfig(PreTrainedConfig):
28 r"""
29 This is the configuration class to store the configuration of a [`Qwen3ASRAudioEncoder`]. It is used to instantiate a
30 Qwen3-ASR audio encoder according to the specified arguments, defining the model architecture. Instantiating a
31 configuration with the defaults will yield a similar configuration to that of the audio encoder of the Qwen2-Audio
32 architecture.
33
34 e.g. [Qwen/Qwen3-ASR-1.7B](https://huggingface.co/Qwen/Qwen3-ASR-1.7B)
35
36 Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the
37 documentation from [`PreTrainedConfig`] for more information.
38
39 Args:
40 num_mel_bins (`int`, *optional*, defaults to 128):
41 Number of mel features used per input features. Should correspond to the value used in the
42 `Qwen3ASRProcessor` class.
43 encoder_layers (`int`, *optional*, defaults to 32):
44 Number of encoder layers.
45 encoder_attention_heads (`int`, *optional*, defaults to 20):
46 Number of attention heads for each attention layer in the Transformer encoder.
47 encoder_ffn_dim (`int`, *optional*, defaults to 5120):
48 Dimensionality of the "intermediate" (often named feed-forward) layer in encoder.
49 d_model (`int`, *optional*, defaults to 1280):
50 Dimensionality of the layers.
51 dropout (`float`, *optional*, defaults to 0.0):
52 The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
53 attention_dropout (`float`, *optional*, defaults to 0.0):
54 The dropout ratio for the attention probabilities.
55 activation_function (`str`, *optional*, defaults to `"gelu"`):
56 The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
57 `"relu"`, `"silu"` and `"gelu_new"` are supported.
58 activation_dropout (`float`, *optional*, defaults to 0.0):
59 The dropout ratio for activations inside the fully connected layer.
60 scale_embedding (`bool`, *optional*, defaults to `False`):
61 Scale embeddings by diving by sqrt(d_model).
62 initializer_range (`float`, *optional*, defaults to 0.02):
63 The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
64 max_source_positions (`int`, *optional*, defaults to 1500):
65 The maximum sequence length of log-mel filter-bank features that this model might ever be used with.
66 n_window (`int`, *optional*, defaults to 100):
67 The chunk for conv and flash attn in AudioEncoder.
68 output_dim (`int`, *optional*, defaults to 3584):
69 The output dimension of AudioEncoder.
70
71 Example:
72
73 ```python
74 >>> from transformers import Qwen3ASRAudioEncoderConfig, Qwen3ASRAudioEncoder
75
76 >>> # Initializing a Qwen3ASRAudioEncoderConfig
77 >>> configuration = Qwen3ASRAudioEncoderConfig()
78
79 >>> # Initializing a Qwen3ASRAudioEncoder (with random weights)
80 >>> model = Qwen3ASRAudioEncoder(configuration)
81
82 >>> # Accessing the model configuration
83 >>> configuration = model.config
84 ```"""
85
86 model_type = "qwen3_asr_audio_encoder"
87
88 def __init__(
89 self,
90 num_mel_bins=128,
91 encoder_layers=32,
92 encoder_attention_heads=20,
93 encoder_ffn_dim=5120,
94 d_model=1280,
95 dropout=0,
96 attention_dropout=0,
97 activation_function="gelu",
98 activation_dropout=0,
99 scale_embedding=False,
100 initializer_range=0.02,
101 max_source_positions=1500,
102 n_window=100,
103 output_dim=3584,
104 n_window_infer=400,
105 conv_chunksize=500,
106 downsample_hidden_size=480,
107 **kwargs,
108 ):
109 super().__init__(**kwargs)
110
111 self.num_mel_bins = num_mel_bins
112 self.d_model = d_model
113 self.encoder_layers = encoder_layers
114 self.encoder_attention_heads = encoder_attention_heads
115 self.encoder_ffn_dim = encoder_ffn_dim
116 self.dropout = dropout
117 self.attention_dropout = attention_dropout
118 self.activation_function = activation_function
119 self.activation_dropout = activation_dropout
120 self.num_hidden_layers = encoder_layers
121 self.initializer_range = initializer_range
122 self.scale_embedding = (
123 scale_embedding # scale factor will be sqrt(d_model) if True
124 )
125 self.max_source_positions = max_source_positions
126 self.n_window = n_window
127 self.output_dim = output_dim
128 self.n_window_infer = n_window_infer
129 self.conv_chunksize = conv_chunksize
130 self.downsample_hidden_size = downsample_hidden_size
131
132
133 class Qwen3ASRTextConfig(PreTrainedConfig):
134 r"""
135 This is the configuration class to store the configuration of a [`Qwen3ASRTextModel`]. It is used to instantiate a
136 Qwen3-ASR model according to the specified arguments, defining the model architecture. Instantiating a configuration
137 with the defaults will yield a similar configuration to that of
138 Qwen3-ASR-1.7B [Qwen/Qwen3-ASR-1.7B](https://huggingface.co/Qwen/Qwen3-ASR-1.7B)
139
140 Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the
141 documentation from [`PreTrainedConfig`] for more information.
142
143 Args:
144 vocab_size (`int`, *optional*, defaults to 151936):
145 Vocabulary size of the Qwen3ASR model. Defines the number of different tokens that can be represented by the
146 `inputs_ids` passed when calling [`Qwen3ASRModel`]
147 hidden_size (`int`, *optional*, defaults to 4096):
148 Dimension of the hidden representations.
149 intermediate_size (`int`, *optional*, defaults to 22016):
150 Dimension of the MLP representations.
151 num_hidden_layers (`int`, *optional*, defaults to 32):
152 Number of hidden layers in the Transformer encoder.
153 num_attention_heads (`int`, *optional*, defaults to 32):
154 Number of attention heads for each attention layer in the Transformer encoder.
155 num_key_value_heads (`int`, *optional*, defaults to 32):
156 This is the number of key_value heads that should be used to implement Grouped Query Attention. If
157 `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
158 `num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When
159 converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
160 by meanpooling all the original heads within that group. For more details, check out [this
161 paper](https://huggingface.co/papers/2305.13245). If it is not specified, will default to `32`.
162 head_dim (`int`, *optional*, defaults to 128):
163 The dimension of the head. If not specified, will default to `hidden_size // num_attention_heads`.
164 hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
165 The non-linear activation function (function or string) in the decoder.
166 max_position_embeddings (`int`, *optional*, defaults to 128000):
167 The maximum sequence length that this model might ever be used with.
168 initializer_range (`float`, *optional*, defaults to 0.02):
169 The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
170 rms_norm_eps (`float`, *optional*, defaults to 1e-06):
171 The epsilon used by the rms normalization layers.
172 use_cache (`bool`, *optional*, defaults to `True`):
173 Whether or not the model should return the last key/values attentions (not used by all models). Only
174 relevant if `config.is_decoder=True`.
175 tie_word_embeddings (`bool`, *optional*, defaults to `False`):
176 Whether the model's input and output word embeddings should be tied.
177 rope_theta (`float`, *optional*, defaults to 5000000.0):
178 The base period of the RoPE embeddings.
179 rope_scaling (`Dict`, *optional*):
180 Dictionary containing the scaling configuration for the RoPE embeddings. NOTE: if you apply new rope type
181 and you expect the model to work on longer `max_position_embeddings`, we recommend you to update this value
182 accordingly.
183 Expected contents:
184 `rope_type` (`str`):
185 The sub-variant of RoPE to use. Can be one of ['default', 'linear', 'dynamic', 'yarn', 'longrope',
186 'llama3'], with 'default' being the original RoPE implementation.
187 `factor` (`float`, *optional*):
188 Used with all rope types except 'default'. The scaling factor to apply to the RoPE embeddings. In
189 most scaling types, a `factor` of x will enable the model to handle sequences of length x *
190 original maximum pre-trained length.
191 `original_max_position_embeddings` (`int`, *optional*):
192 Used with 'dynamic', 'longrope' and 'llama3'. The original max position embeddings used during
193 pretraining.
194 `attention_factor` (`float`, *optional*):
195 Used with 'yarn' and 'longrope'. The scaling factor to be applied on the attention
196 computation. If unspecified, it defaults to value recommended by the implementation, using the
197 `factor` field to infer the suggested value.
198 `beta_fast` (`float`, *optional*):
199 Only used with 'yarn'. Parameter to set the boundary for extrapolation (only) in the linear
200 ramp function. If unspecified, it defaults to 32.
201 `beta_slow` (`float`, *optional*):
202 Only used with 'yarn'. Parameter to set the boundary for interpolation (only) in the linear
203 ramp function. If unspecified, it defaults to 1.
204 `short_factor` (`list[float]`, *optional*):
205 Only used with 'longrope'. The scaling factor to be applied to short contexts (<
206 `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
207 size divided by the number of attention heads divided by 2
208 `long_factor` (`list[float]`, *optional*):
209 Only used with 'longrope'. The scaling factor to be applied to long contexts (<
210 `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
211 size divided by the number of attention heads divided by 2
212 `low_freq_factor` (`float`, *optional*):
213 Only used with 'llama3'. Scaling factor applied to low frequency components of the RoPE
214 `high_freq_factor` (`float`, *optional*):
215 Only used with 'llama3'. Scaling factor applied to high frequency components of the RoPE
216 attention_bias (`bool`, defaults to `False`, *optional*, defaults to `False`):
217 Whether to use a bias in the query, key, value and output projection layers during self-attention.
218 attention_dropout (`float`, *optional*, defaults to 0.0):
219 The dropout ratio for the attention probabilities.
220
221 ```python
222 >>> from transformers import Qwen3ASRTextModel, Qwen3ASRTextConfig
223
224 >>> # Initializing a Qwen3ASR style configuration
225 >>> configuration = Qwen3ASRTextConfig()
226
227 >>> # Initializing a model from the Qwen3-VL-7B style configuration
228 >>> model = Qwen3ASRTextModel(configuration)
229
230 >>> # Accessing the model configuration
231 >>> configuration = model.config
232 ```"""
233
234 model_type = "qwen3_asr_text"
235 base_config_key = "text_config"
236
237 def __init__(
238 self,
239 vocab_size=151936,
240 hidden_size=4096,
241 intermediate_size=22016,
242 num_hidden_layers=32,
243 num_attention_heads=32,
244 num_key_value_heads=32,
245 head_dim=128,
246 hidden_act="silu",
247 max_position_embeddings=128000,
248 initializer_range=0.02,
249 rms_norm_eps=1e-6,
250 use_cache=True,
251 tie_word_embeddings=False,
252 rope_theta=5000000.0,
253 rope_scaling=None,
254 attention_bias=False,
255 attention_dropout=0.0,
256 **kwargs,
257 ):
258 self.vocab_size = vocab_size
259 self.max_position_embeddings = max_position_embeddings
260 self.hidden_size = hidden_size
261 self.intermediate_size = intermediate_size
262 self.num_hidden_layers = num_hidden_layers
263 self.num_attention_heads = num_attention_heads
264
265 # for backward compatibility
266 if num_key_value_heads is None:
267 num_key_value_heads = num_attention_heads
268
269 self.num_key_value_heads = num_key_value_heads
270 self.head_dim = head_dim
271 self.hidden_act = hidden_act
272 self.initializer_range = initializer_range
273 self.rms_norm_eps = rms_norm_eps
274 self.use_cache = use_cache
275 self.rope_theta = rope_theta
276 self.rope_scaling = rope_scaling
277 self.attention_bias = attention_bias
278 self.attention_dropout = attention_dropout
279 # Validate the correctness of rotary position embeddings parameters
280 # BC: if there is a 'type' field, move it to 'rope_type'.
281 if self.rope_scaling is not None and "type" in self.rope_scaling:
282 self.rope_scaling["rope_type"] = self.rope_scaling["type"]
283 rope_config_validation(self)
284
285 super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)
286
287
288 class Qwen3ASRThinkerConfig(PreTrainedConfig):
289 r"""
290 This is the configuration class to store the configuration of a [`Qwen3ASRThinker`]. It is used to instantiate a
291 Qwen3-ASR-Thinker model according to the specified arguments, defining the model architecture. Instantiating a
292 configuration with the defaults will yield a similar configuration to that of the thinker component of the Qwen3-Omni
293 architecture.
294
295 e.g. [Qwen/Qwen3-ASR-1.7B](https://huggingface.co/Qwen/Qwen3-ASR-1.7B)
296
297 Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the
298 documentation from [`PreTrainedConfig`] for more information.
299
300 Args:
301 audio_config (`dict`, *optional*):
302 The config dictionary of the audio backbone.
303 text_config (`dict`, *optional*):
304 The config dictionary of the text backbone.
305 audio_token_id (`int`, *optional*, defaults to 151646):
306 The audio token id to encode the audio prompt.
307 audio_start_token_id (`int`, *optional*, defaults to 151647):
308 The audio start token id to encode the audio prompt.
309 user_token_id (`int`, *optional*, defaults to 872):
310 The user token id to encode the user token.
311 initializer_range (`float`, *optional*, defaults to 0.02):
312 The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
313
314 Example:
315
316 ```python
317 >>> from transformers import Qwen3ASRThinkerModel, Qwen3ASRThinkerConfig
318
319 >>> # Initializing a default Qwen3ASRThinkerConfig
320 >>> configuration = Qwen3ASRThinkerConfig()
321
322 >>> # Initializing a model (with random weights) from the default configuration
323 >>> model = Qwen3ASRThinkerModel(configuration)
324
325 >>> # Accessing the model configuration
326 >>> configuration = model.config
327 ```"""
328
329 model_type = "qwen3_asr_thinker"
330
331 attribute_map = {}
332 sub_configs = {
333 "audio_config": Qwen3ASRAudioEncoderConfig,
334 "text_config": Qwen3ASRTextConfig,
335 }
336
337 def __init__(
338 self,
339 audio_config=None,
340 text_config=None,
341 audio_token_id=151646,
342 audio_start_token_id=151647,
343 user_token_id=872,
344 initializer_range=0.02,
345 classify_num=None,
346 **kwargs,
347 ):
348 super().__init__(**kwargs)
349 self.user_token_id = user_token_id
350 self.audio_start_token_id = audio_start_token_id
351 self.initializer_range = initializer_range
352 self.classify_num = classify_num
353
354 if isinstance(audio_config, dict):
355 audio_config = Qwen3ASRAudioEncoderConfig(**audio_config)
356 elif audio_config is None:
357 audio_config = Qwen3ASRAudioEncoderConfig()
358 self.audio_config = audio_config
359
360 if isinstance(text_config, dict):
361 text_config = Qwen3ASRTextConfig(**text_config)
362 elif text_config is None:
363 text_config = Qwen3ASRTextConfig()
364 self.text_config = text_config
365 self.audio_token_id = audio_token_id
366
367
368 class Qwen3ASRConfig(PreTrainedConfig):
369 """
370 This is the configuration class to store the configuration of a [`Qwen3ASRForConditionalGeneration`]. It is used to instantiate a Qwen3ASR
371 model according to the specified sub-models configurations, defining the model architecture.
372
373 Instantiating a configuration with the defaults will yield a similar configuration to that of the
374 [Qwen/Qwen3-ASR-1.7B](https://huggingface.co/Qwen/Qwen3-ASR-1.7B) architecture.
375
376 Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the
377 documentation from [`PreTrainedConfig`] for more information.
378
379 Args:
380 thinker_config (`dict`, *optional*): Configuration of the underlying thinker sub-model.
381 support_languages (`List[str]`, *optional*): The languages supported by the model.
382
383 Example:
384
385 ```python
386 >>> from transformers import (
387 ... Qwen3ASRThinkerConfig,
388 ... Qwen3ASRForConditionalGeneration,
389 ... Qwen3ASRConfig,
390 ... )
391
392 >>> # Initializing a Qwen3ASR style configuration
393 >>> configuration = Qwen3ASRConfig()
394
395 >>> # Initializing a model from the configuration
396 >>> model = Qwen3ASRForConditionalGeneration(configuration)
397
398 >>> # Accessing the model configuration
399 >>> configuration = model.config
400 ```"""
401
402 model_type = "qwen3_asr"
403 sub_configs = {
404 "thinker_config": Qwen3ASRThinkerConfig,
405 }
406
407 def __init__(
408 self,
409 thinker_config=None,
410 audio_config=None,
411 text_config=None,
412 audio_token_id=None,
413 support_languages=None,
414 timestamp_token_id=None,
415 timestamp_segment_time=None,
416 **kwargs,
417 ):
418 if thinker_config is None:
419 thinker_config = {
420 key: value
421 for key, value in (
422 ("audio_config", audio_config),
423 ("text_config", text_config),
424 ("audio_token_id", audio_token_id),
425 )
426 if value is not None
427 }
428 if not thinker_config:
429 logger.info(
430 "thinker_config is None. Initializing thinker model with "
431 "default values"
432 )
433
434 self.thinker_config = Qwen3ASRThinkerConfig(**thinker_config)
435 self.support_languages = support_languages
436 self.timestamp_token_id = timestamp_token_id
437 self.timestamp_segment_time = timestamp_segment_time
438 super().__init__(**kwargs)
439
440 def get_text_config(self, decoder=False) -> "PreTrainedConfig":
441 """
442 Returns the config that is meant to be used with text IO. On most models, it is the original config instance
443 itself. On specific composite models, it is under a set of valid names.
444
445 Args:
446 decoder (`Optional[bool]`, *optional*, defaults to `False`):
447 If set to `True`, then only search for decoder config names.
448 """
449 # Overridden for deeply nested config like Qwen2.5-Omni. We don't have any omni model
450 # except for Qwen yet. This has to be generalized if more deeply nested configs are
451 # added. NOTE: currently method used only by vLLM
452 return self.thinker_config.get_text_config()
453
454
455 __all__ = ["Qwen3ASRConfig", "Qwen3ASRThinkerConfig", "Qwen3ASRAudioEncoderConfig"]