| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | # Copyright 2025 Antgroup and The HuggingFace Inc. team. All rights reserved. |
| 4 | # Adapted from |
| 5 | # https://huggingface.co/inclusionAI/Ling-3.0-flash/blob/51fd444268df5267074ef0b289de0e1c4cb9b381/configuration_bailing_moe_v3.py |
| 6 | """Bailing V3 vision-language model configuration.""" |
| 7 | |
| 8 | from __future__ import annotations |
| 9 | |
| 10 | from typing import Any |
| 11 | |
| 12 | from transformers.configuration_utils import PreTrainedConfig |
| 13 | |
| 14 | |
| 15 | def _normalize_mrope_parameters( |
| 16 | config: PreTrainedConfig, |
| 17 | mrope_section: list[int], |
| 18 | ) -> None: |
| 19 | """Add the top-level Bailing M-RoPE split to a text config.""" |
| 20 | rope_parameters = dict(getattr(config, "rope_parameters", None) or {}) |
| 21 | text_mrope_section = rope_parameters.get("mrope_section") |
| 22 | if text_mrope_section is not None and list(text_mrope_section) != mrope_section: |
| 23 | raise ValueError( |
| 24 | "Bailing V3 VL has conflicting M-RoPE sections: " |
| 25 | f"text={text_mrope_section}, top-level={mrope_section}" |
| 26 | ) |
| 27 | |
| 28 | if "rope_type" not in rope_parameters and "type" not in rope_parameters: |
| 29 | rope_parameters["rope_type"] = "default" |
| 30 | rope_parameters.setdefault("rope_theta", getattr(config, "rope_theta", 10_000)) |
| 31 | rope_parameters["mrope_section"] = list(mrope_section) |
| 32 | config.rope_parameters = rope_parameters |
| 33 | |
| 34 | |
| 35 | def _build_layer_types( |
| 36 | num_hidden_layers: int, |
| 37 | layer_group_size: int, |
| 38 | ) -> list[str]: |
| 39 | if layer_group_size <= 0: |
| 40 | raise ValueError( |
| 41 | f"Bailing V3 VL layer_group_size must be positive, got {layer_group_size}" |
| 42 | ) |
| 43 | |
| 44 | grouped_layers = num_hidden_layers // layer_group_size * layer_group_size |
| 45 | return [ |
| 46 | "linear_attention" |
| 47 | if ((layer_idx + 1) % layer_group_size != 0 and layer_idx < grouped_layers) |
| 48 | else "full_attention" |
| 49 | for layer_idx in range(num_hidden_layers) |
| 50 | ] |
| 51 | |
| 52 | |
| 53 | class BailingMoeV3TextConfig(PreTrainedConfig): |
| 54 | model_type = "bailing_hybrid" |
| 55 | base_config_key = "text_config" |
| 56 | keys_to_ignore_at_inference = ["past_key_values"] |
| 57 | ignore_keys_at_rope_validation = {"mrope_section"} |
| 58 | |
| 59 | def __init__( |
| 60 | self, |
| 61 | *, |
| 62 | vocab_size: int = 157184, |
| 63 | hidden_size: int = 2048, |
| 64 | intermediate_size: int = 5120, |
| 65 | num_hidden_layers: int = 20, |
| 66 | num_attention_heads: int = 16, |
| 67 | num_key_value_heads: int = 4, |
| 68 | hidden_act: str = "silu", |
| 69 | use_qkv_bias: bool = False, |
| 70 | use_bias: bool = False, |
| 71 | rms_norm_eps: float = 1e-6, |
| 72 | tie_word_embeddings: bool = False, |
| 73 | embedding_dropout: float = 0.0, |
| 74 | attention_dropout: float = 0.0, |
| 75 | output_dropout: float = 0.0, |
| 76 | initializer_range: float = 0.02, |
| 77 | max_position_embeddings: int = 32768, |
| 78 | rope_theta: float = 600000.0, |
| 79 | rope_parameters: dict[str, Any] | None = None, |
| 80 | rope_scaling: dict[str, Any] | None = None, |
| 81 | partial_rotary_factor: float | None = None, |
| 82 | mrope_section: list[int] | tuple[int, ...] | None = None, |
| 83 | use_cache: bool = True, |
| 84 | max_window_layers: int = 20, |
| 85 | pad_token_id: int = 156892, |
| 86 | eos_token_id: int = 156892, |
| 87 | num_experts: int = 256, |
| 88 | num_shared_experts: int = 1, |
| 89 | num_experts_per_tok: int = 8, |
| 90 | n_group: int = 8, |
| 91 | topk_group: int = 4, |
| 92 | moe_intermediate_size: int = 512, |
| 93 | moe_shared_expert_intermediate_size: int = 512, |
| 94 | first_k_dense_replace: int = 1, |
| 95 | head_dim: int | None = 128, |
| 96 | output_router_logits: bool = False, |
| 97 | use_qk_norm: bool = True, |
| 98 | num_nextn_predict_layers: int = 0, |
| 99 | mtp_loss_scaling_factor: float = 0, |
| 100 | moe_router_enable_expert_bias: bool = True, |
| 101 | routed_scaling_factor: float = 1.0, |
| 102 | layer_group_size: int = 5, |
| 103 | layer_types: list[str] | None = None, |
| 104 | layers_block_type: list[str] | None = None, |
| 105 | kv_lora_rank: int = 512, |
| 106 | q_lora_rank: int | None = None, |
| 107 | qk_rope_head_dim: int = 64, |
| 108 | v_head_dim: int = 128, |
| 109 | qk_nope_head_dim: int = 128, |
| 110 | rope_interleave: bool = True, |
| 111 | score_function: str = "sigmoid", |
| 112 | scoring_func: str = "sigmoid", |
| 113 | seq_aux: bool = True, |
| 114 | topk_method: str = "noaux_tc", |
| 115 | router_dtype: str = "fp32", |
| 116 | gated_attention_proj_granularity_type: str | None = None, |
| 117 | no_kda_lora: bool = False, |
| 118 | kda_safe_gate: bool = False, |
| 119 | kda_lower_bound: float | None = None, |
| 120 | short_conv_kernel_size: int = 4, |
| 121 | architectures: list[str] | None = None, |
| 122 | **kwargs: Any, |
| 123 | ) -> None: |
| 124 | kwargs.pop("model_type", None) |
| 125 | if architectures is None: |
| 126 | architectures = ["BailingMoeV3ForCausalLM"] |
| 127 | |
| 128 | if rope_parameters is not None: |
| 129 | self.rope_parameters = dict(rope_parameters) |
| 130 | elif rope_scaling is not None: |
| 131 | self.rope_parameters = dict(rope_scaling) |
| 132 | else: |
| 133 | self.rope_parameters = {} |
| 134 | if ( |
| 135 | "rope_type" not in self.rope_parameters |
| 136 | and "type" not in self.rope_parameters |
| 137 | ): |
| 138 | self.rope_parameters["rope_type"] = "default" |
| 139 | self.rope_parameters.setdefault("rope_theta", rope_theta) |
| 140 | if mrope_section is not None: |
| 141 | _normalize_mrope_parameters(self, list(mrope_section)) |
| 142 | |
| 143 | self.vocab_size = vocab_size |
| 144 | self.hidden_size = hidden_size |
| 145 | self.intermediate_size = intermediate_size |
| 146 | self.num_hidden_layers = num_hidden_layers |
| 147 | self.num_attention_heads = num_attention_heads |
| 148 | self.num_key_value_heads = num_key_value_heads |
| 149 | self.hidden_act = hidden_act |
| 150 | self.use_qkv_bias = use_qkv_bias |
| 151 | self.use_bias = use_bias |
| 152 | self.rms_norm_eps = rms_norm_eps |
| 153 | self.embedding_dropout = embedding_dropout |
| 154 | self.attention_dropout = attention_dropout |
| 155 | self.output_dropout = output_dropout |
| 156 | self.initializer_range = initializer_range |
| 157 | self.max_position_embeddings = max_position_embeddings |
| 158 | self.rope_theta = rope_theta |
| 159 | self.partial_rotary_factor = partial_rotary_factor |
| 160 | self.use_cache = use_cache |
| 161 | self.max_window_layers = max_window_layers |
| 162 | self.num_experts = num_experts |
| 163 | self.num_shared_experts = num_shared_experts |
| 164 | self.num_experts_per_tok = num_experts_per_tok |
| 165 | self.n_group = n_group |
| 166 | self.topk_group = topk_group |
| 167 | self.moe_intermediate_size = moe_intermediate_size |
| 168 | self.moe_shared_expert_intermediate_size = moe_shared_expert_intermediate_size |
| 169 | self.first_k_dense_replace = first_k_dense_replace |
| 170 | self.head_dim = head_dim or hidden_size // num_attention_heads |
| 171 | self.output_router_logits = output_router_logits |
| 172 | self.use_qk_norm = use_qk_norm |
| 173 | self.num_nextn_predict_layers = num_nextn_predict_layers |
| 174 | self.mtp_loss_scaling_factor = mtp_loss_scaling_factor |
| 175 | self.moe_router_enable_expert_bias = moe_router_enable_expert_bias |
| 176 | self.routed_scaling_factor = routed_scaling_factor |
| 177 | self.layer_group_size = layer_group_size |
| 178 | self.kv_lora_rank = kv_lora_rank |
| 179 | self.q_lora_rank = q_lora_rank |
| 180 | self.qk_rope_head_dim = qk_rope_head_dim |
| 181 | self.v_head_dim = v_head_dim |
| 182 | self.qk_nope_head_dim = qk_nope_head_dim |
| 183 | self.qk_head_dim = qk_nope_head_dim + qk_rope_head_dim |
| 184 | self.rope_interleave = rope_interleave |
| 185 | self.score_function = score_function |
| 186 | self.scoring_func = scoring_func |
| 187 | self.seq_aux = seq_aux |
| 188 | self.topk_method = topk_method |
| 189 | self.router_dtype = router_dtype |
| 190 | self.gated_attention_proj_granularity_type = ( |
| 191 | gated_attention_proj_granularity_type |
| 192 | ) |
| 193 | self.no_kda_lora = no_kda_lora |
| 194 | self.kda_safe_gate = kda_safe_gate |
| 195 | self.kda_lower_bound = kda_lower_bound |
| 196 | self.short_conv_kernel_size = short_conv_kernel_size |
| 197 | |
| 198 | expected_layer_types = _build_layer_types( |
| 199 | num_hidden_layers, |
| 200 | layer_group_size, |
| 201 | ) |
| 202 | if layers_block_type is not None: |
| 203 | legacy_type_mapping = { |
| 204 | "attention": "full_attention", |
| 205 | "mamba": "linear_attention", |
| 206 | "conv": "linear_attention", |
| 207 | } |
| 208 | normalized_block_types = [ |
| 209 | legacy_type_mapping.get(layer_type, layer_type) |
| 210 | for layer_type in layers_block_type |
| 211 | ] |
| 212 | if layer_types is not None and layer_types != normalized_block_types: |
| 213 | raise ValueError( |
| 214 | "Bailing V3 VL has conflicting layer_types and layers_block_type" |
| 215 | ) |
| 216 | layer_types = normalized_block_types |
| 217 | if layer_types is not None and layer_types != expected_layer_types: |
| 218 | raise ValueError( |
| 219 | "Bailing V3 VL layer_types must match the layer_group_size schedule" |
| 220 | ) |
| 221 | self.layer_types = expected_layer_types |
| 222 | |
| 223 | super().__init__( |
| 224 | architectures=architectures, |
| 225 | pad_token_id=pad_token_id, |
| 226 | eos_token_id=eos_token_id, |
| 227 | tie_word_embeddings=tie_word_embeddings, |
| 228 | **kwargs, |
| 229 | ) |
| 230 | self.validate_layer_type() |
| 231 | |
| 232 | |
| 233 | class BailingMoeV3VisionConfig(PreTrainedConfig): |
| 234 | model_type = "qwen3_moe_vit" |
| 235 | base_config_key = "vision_config" |
| 236 | |
| 237 | def __init__( |
| 238 | self, |
| 239 | *, |
| 240 | depth: int = 27, |
| 241 | hidden_size: int = 1152, |
| 242 | hidden_act: str = "gelu_pytorch_tanh", |
| 243 | intermediate_size: int = 4304, |
| 244 | num_heads: int = 16, |
| 245 | in_channels: int = 3, |
| 246 | patch_size: int = 16, |
| 247 | spatial_merge_size: int = 2, |
| 248 | temporal_patch_size: int = 2, |
| 249 | out_hidden_size: int = 3584, |
| 250 | num_position_embeddings: int = 2304, |
| 251 | deepstack_visual_indexes: list[int] | None = None, |
| 252 | initializer_range: float = 0.02, |
| 253 | **kwargs: Any, |
| 254 | ) -> None: |
| 255 | kwargs.pop("model_type", None) |
| 256 | super().__init__(**kwargs) |
| 257 | self.depth = depth |
| 258 | self.hidden_size = hidden_size |
| 259 | self.hidden_act = hidden_act |
| 260 | self.intermediate_size = intermediate_size |
| 261 | self.num_heads = num_heads |
| 262 | self.in_channels = in_channels |
| 263 | self.patch_size = patch_size |
| 264 | self.spatial_merge_size = spatial_merge_size |
| 265 | self.temporal_patch_size = temporal_patch_size |
| 266 | self.out_hidden_size = out_hidden_size |
| 267 | self.num_position_embeddings = num_position_embeddings |
| 268 | self.deepstack_visual_indexes = ( |
| 269 | [8, 16, 24] |
| 270 | if deepstack_visual_indexes is None |
| 271 | else deepstack_visual_indexes |
| 272 | ) |
| 273 | self.initializer_range = initializer_range |
| 274 | |
| 275 | |
| 276 | class BailingMoeV3VLConfig(PreTrainedConfig): |
| 277 | model_type = "bailing_moe_v3_vl" |
| 278 | keys_to_ignore_at_inference = ["past_key_values"] |
| 279 | sub_configs = { |
| 280 | "text_config": BailingMoeV3TextConfig, |
| 281 | "vision_config": BailingMoeV3VisionConfig, |
| 282 | } |
| 283 | |
| 284 | def __init__( |
| 285 | self, |
| 286 | *, |
| 287 | text_config: BailingMoeV3TextConfig | dict[str, Any] | None = None, |
| 288 | vision_config: BailingMoeV3VisionConfig | dict[str, Any] | None = None, |
| 289 | image_token_id: int = 151655, |
| 290 | video_token_id: int = 151656, |
| 291 | vision_start_token_id: int = 151652, |
| 292 | vision_end_token_id: int = 151653, |
| 293 | tie_word_embeddings: bool = False, |
| 294 | mrope_section: list[int] | tuple[int, ...] | None = None, |
| 295 | architectures: list[str] | None = None, |
| 296 | **kwargs: Any, |
| 297 | ) -> None: |
| 298 | kwargs.pop("model_type", None) |
| 299 | if mrope_section is None: |
| 300 | mrope_section = [12, 10, 10] |
| 301 | self.mrope_section = list(mrope_section) |
| 302 | |
| 303 | if text_config is None: |
| 304 | text_config = BailingMoeV3TextConfig() |
| 305 | elif isinstance(text_config, dict): |
| 306 | text_config = dict(text_config) |
| 307 | nested_mrope_section = text_config.pop("mrope_section", None) |
| 308 | if ( |
| 309 | nested_mrope_section is not None |
| 310 | and list(nested_mrope_section) != self.mrope_section |
| 311 | ): |
| 312 | raise ValueError( |
| 313 | "Bailing V3 VL has conflicting M-RoPE sections: " |
| 314 | f"text={nested_mrope_section}, " |
| 315 | f"top-level={self.mrope_section}" |
| 316 | ) |
| 317 | text_config = BailingMoeV3TextConfig(**text_config) |
| 318 | self.text_config = text_config |
| 319 | _normalize_mrope_parameters(self.text_config, self.mrope_section) |
| 320 | |
| 321 | if vision_config is None: |
| 322 | vision_config = BailingMoeV3VisionConfig() |
| 323 | elif isinstance(vision_config, dict): |
| 324 | vision_config = BailingMoeV3VisionConfig(**vision_config) |
| 325 | self.vision_config = vision_config |
| 326 | |
| 327 | self.image_token_id = image_token_id |
| 328 | self.video_token_id = video_token_id |
| 329 | self.vision_start_token_id = vision_start_token_id |
| 330 | self.vision_end_token_id = vision_end_token_id |
| 331 | if architectures is None: |
| 332 | architectures = ["BailingMoeV3VLForConditionalGeneration"] |
| 333 | |
| 334 | super().__init__( |
| 335 | architectures=architectures, |
| 336 | tie_word_embeddings=tie_word_embeddings, |
| 337 | **kwargs, |
| 338 | ) |
| 339 | |
| 340 | |
| 341 | __all__ = [ |
| 342 | "BailingMoeV3TextConfig", |
| 343 | "BailingMoeV3VisionConfig", |
| 344 | "BailingMoeV3VLConfig", |
| 345 | ] |