| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Kimi-K3 multimodal configuration.""" |
| 4 | |
| 5 | from transformers.configuration_utils import PreTrainedConfig |
| 6 | |
| 7 | from vllm.logger import init_logger |
| 8 | from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig |
| 9 | |
| 10 | logger = init_logger(__name__) |
| 11 | |
| 12 | |
| 13 | class KimiK3VisionConfig(PreTrainedConfig): |
| 14 | model_type = "kimi_k3_vision" |
| 15 | |
| 16 | def __init__( |
| 17 | self, |
| 18 | patch_size: int = 14, |
| 19 | init_pos_emb_height: int = 64, |
| 20 | init_pos_emb_width: int = 64, |
| 21 | init_pos_emb_time: int = 4, |
| 22 | pos_emb_type: str = "divided_fixed", |
| 23 | vt_num_attention_heads: int = 12, |
| 24 | vt_num_hidden_layers: int = 27, |
| 25 | vt_hidden_size: int = 1024, |
| 26 | vt_intermediate_size: int = 4096, |
| 27 | merge_kernel_size: tuple[int, int] = (2, 2), |
| 28 | video_attn_type: str = "spatial_temporal", |
| 29 | merge_type: str = "sd2_tpool", |
| 30 | _attn_implementation: str = "flash_attention_2", |
| 31 | mm_projector_type: str = "patchmergerv2", |
| 32 | mm_hidden_size: int | None = None, |
| 33 | projector_hidden_act: str = "gelu", |
| 34 | projector_ln_eps: float = 1e-5, |
| 35 | qkv_hidden_size: int = 1536, |
| 36 | norm_type: str = "rmsnorm", |
| 37 | attn_bias: bool = False, |
| 38 | patch_embed_proj_bias: bool = False, |
| 39 | mlp_type: str = "mlp2", |
| 40 | linear_bias: bool = False, |
| 41 | activation_func: str = "gelu_pytorch_tanh", |
| 42 | pos_emb_interpolation_mode: str = "bilinear", |
| 43 | text_hidden_size: int = 2304, |
| 44 | **kwargs, |
| 45 | ): |
| 46 | super().__init__(**kwargs) |
| 47 | |
| 48 | self.patch_size = patch_size |
| 49 | self.init_pos_emb_height = init_pos_emb_height |
| 50 | self.init_pos_emb_width = init_pos_emb_width |
| 51 | self.init_pos_emb_time = init_pos_emb_time |
| 52 | self.pos_emb_type = pos_emb_type |
| 53 | self.vt_num_attention_heads = vt_num_attention_heads |
| 54 | self.vt_num_hidden_layers = vt_num_hidden_layers |
| 55 | self.vt_hidden_size = vt_hidden_size |
| 56 | self.vt_intermediate_size = vt_intermediate_size |
| 57 | self.merge_kernel_size = tuple(merge_kernel_size) |
| 58 | self.video_attn_type = video_attn_type |
| 59 | self.merge_type = merge_type |
| 60 | self._attn_implementation = _attn_implementation |
| 61 | |
| 62 | self.mm_projector_type = mm_projector_type |
| 63 | self.mm_hidden_size = ( |
| 64 | mm_hidden_size if mm_hidden_size is not None else vt_hidden_size |
| 65 | ) |
| 66 | self.projector_hidden_act = projector_hidden_act |
| 67 | self.projector_ln_eps = projector_ln_eps |
| 68 | self.text_hidden_size = text_hidden_size |
| 69 | |
| 70 | self.qkv_hidden_size = qkv_hidden_size |
| 71 | self.norm_type = norm_type |
| 72 | self.attn_bias = attn_bias |
| 73 | self.patch_embed_proj_bias = patch_embed_proj_bias |
| 74 | self.mlp_type = mlp_type |
| 75 | self.linear_bias = linear_bias |
| 76 | self.activation_func = activation_func |
| 77 | self.pos_emb_interpolation_mode = pos_emb_interpolation_mode |
| 78 | |
| 79 | # Aliases consumed by the Kimi-K2.5 vision implementation. |
| 80 | self.num_attention_heads = vt_num_attention_heads |
| 81 | self.num_hidden_layers = vt_num_hidden_layers |
| 82 | self.hidden_size = vt_hidden_size |
| 83 | self.intermediate_size = vt_intermediate_size |
| 84 | |
| 85 | |
| 86 | class KimiK3Config(PreTrainedConfig): |
| 87 | model_type = "kimi_k3" |
| 88 | |
| 89 | def __init__( |
| 90 | self, |
| 91 | text_config: dict | KimiLinearConfig | None = None, |
| 92 | vision_config: dict | KimiK3VisionConfig | None = None, |
| 93 | ignore_index: int = -100, |
| 94 | media_placeholder_token_id: int = 163605, |
| 95 | pad_token_id: int = 0, |
| 96 | image_placeholder: str = "<|kimi_image_placeholder|>", |
| 97 | **kwargs, |
| 98 | ): |
| 99 | if text_config is None: |
| 100 | self.text_config = KimiLinearConfig() |
| 101 | elif isinstance(text_config, dict): |
| 102 | self.text_config = KimiLinearConfig(**text_config) |
| 103 | else: |
| 104 | self.text_config = text_config |
| 105 | |
| 106 | if vision_config is None: |
| 107 | self.vision_config = KimiK3VisionConfig() |
| 108 | elif isinstance(vision_config, dict): |
| 109 | self.vision_config = KimiK3VisionConfig(**vision_config) |
| 110 | else: |
| 111 | self.vision_config = vision_config |
| 112 | |
| 113 | # K3's vision projector output must match the text model's hidden size. |
| 114 | # Override any value provided in vision_config for safety. |
| 115 | if self.vision_config.text_hidden_size != self.text_config.hidden_size: |
| 116 | logger.info( |
| 117 | "Overriding vision_config.text_hidden_size from %s to %s " |
| 118 | "to match text_config.hidden_size", |
| 119 | self.vision_config.text_hidden_size, |
| 120 | self.text_config.hidden_size, |
| 121 | ) |
| 122 | self.vision_config.text_hidden_size = self.text_config.hidden_size |
| 123 | |
| 124 | self.ignore_index = ignore_index |
| 125 | self.media_placeholder_token_id = media_placeholder_token_id |
| 126 | self.image_placeholder = image_placeholder |
| 127 | |
| 128 | if getattr(self.text_config, "quantization_config", None) is not None: |
| 129 | self.quantization_config = self.text_config.quantization_config |
| 130 | |
| 131 | super().__init__(pad_token_id=pad_token_id, **kwargs) |
| 132 | |
| 133 | @property |
| 134 | def hidden_size(self) -> int: |
| 135 | return self.text_config.hidden_size |
| 136 | |
| 137 | @property |
| 138 | def vocab_size(self) -> int: |
| 139 | return self.text_config.vocab_size |