| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Kimi-K2.5 Model Configuration. |
| 4 | |
| 5 | This configuration supports video-chunk as an internal modality type. |
| 6 | A video-chunk is the smallest independently processable unit of video. |
| 7 | """ |
| 8 | |
| 9 | from transformers import DeepseekV3Config |
| 10 | from transformers.configuration_utils import PreTrainedConfig |
| 11 | |
| 12 | |
| 13 | class KimiK25VisionConfig(PreTrainedConfig): |
| 14 | model_type = "kimi_k25_vision" |
| 15 | |
| 16 | def __init__( |
| 17 | self, |
| 18 | # Vision Tower |
| 19 | patch_size: int = 14, |
| 20 | init_pos_emb_height: int = 64, |
| 21 | init_pos_emb_width: int = 64, |
| 22 | init_pos_emb_time: int = 4, |
| 23 | pos_emb_type: str = "divided_fixed", |
| 24 | num_attention_heads: int = 16, |
| 25 | num_hidden_layers: int = 27, |
| 26 | hidden_size: int = 1152, |
| 27 | intermediate_size: int = 4304, |
| 28 | merge_kernel_size: tuple[int, int] = (2, 2), |
| 29 | video_attn_type: str = "spatial_temporal", |
| 30 | merge_type: str = "sd2_tpool", |
| 31 | # MM Projector |
| 32 | mm_projector_type: str = "patchmerger", |
| 33 | mm_hidden_size: int | None = None, |
| 34 | projector_hidden_act: str = "gelu", |
| 35 | projector_ln_eps: float = 1e-5, |
| 36 | **kwargs, |
| 37 | ): |
| 38 | super().__init__(**kwargs) |
| 39 | # Vision Tower |
| 40 | self.patch_size = patch_size |
| 41 | self.init_pos_emb_height = init_pos_emb_height |
| 42 | self.init_pos_emb_width = init_pos_emb_width |
| 43 | self.init_pos_emb_time = init_pos_emb_time |
| 44 | self.pos_emb_type = pos_emb_type |
| 45 | self.num_attention_heads = num_attention_heads |
| 46 | self.num_hidden_layers = num_hidden_layers |
| 47 | self.hidden_size = hidden_size |
| 48 | self.intermediate_size = intermediate_size |
| 49 | self.merge_kernel_size = merge_kernel_size |
| 50 | self.video_attn_type = video_attn_type |
| 51 | self.merge_type = merge_type |
| 52 | # MM Projector |
| 53 | self.mm_projector_type = mm_projector_type |
| 54 | if mm_hidden_size is not None: |
| 55 | self.mm_hidden_size = mm_hidden_size |
| 56 | else: |
| 57 | self.mm_hidden_size = hidden_size |
| 58 | self.projector_hidden_act = projector_hidden_act |
| 59 | self.projector_ln_eps = projector_ln_eps |
| 60 | |
| 61 | |
| 62 | class KimiK25Config(PreTrainedConfig): |
| 63 | """Kimi-K2.5 model configuration. |
| 64 | |
| 65 | Kimi-K2.5 extends Kimi-K2 with vision support using video-chunks. |
| 66 | A video-chunk consists of multiple consecutive frames |
| 67 | that are processed together with temporal pooling. |
| 68 | |
| 69 | Args: |
| 70 | vision_config: Configuration for the vision tower and projector. |
| 71 | text_config: Configuration for the text model (DeepseekV3). |
| 72 | ignore_index: The ignore index for the loss function. |
| 73 | media_placeholder_token_id: The token ID for media placeholders. |
| 74 | pad_token_id: The token ID for padding. |
| 75 | |
| 76 | """ |
| 77 | |
| 78 | model_type = "kimi_k25" |
| 79 | |
| 80 | def __init__( |
| 81 | self, |
| 82 | vision_config: dict | KimiK25VisionConfig | None = None, |
| 83 | text_config: dict | DeepseekV3Config | None = None, |
| 84 | ignore_index: int = -100, |
| 85 | media_placeholder_token_id: int = 163605, |
| 86 | pad_token_id: int = 0, |
| 87 | use_unified_vision_chunk: bool = False, |
| 88 | video_placeholder: str = "<|kimi_k25_video_placeholder|>", |
| 89 | **kwargs, |
| 90 | ): |
| 91 | # Vision config |
| 92 | if vision_config is None: |
| 93 | self.vision_config = KimiK25VisionConfig() |
| 94 | elif isinstance(vision_config, dict): |
| 95 | self.vision_config = KimiK25VisionConfig(**vision_config) |
| 96 | else: |
| 97 | self.vision_config = vision_config |
| 98 | |
| 99 | # Text config |
| 100 | if text_config is None: |
| 101 | self.text_config = DeepseekV3Config() |
| 102 | elif isinstance(text_config, dict): |
| 103 | self.text_config = DeepseekV3Config(**text_config) |
| 104 | else: |
| 105 | self.text_config = text_config |
| 106 | |
| 107 | # Set mm_hidden_size to text hidden size if not explicitly set |
| 108 | if self.vision_config.mm_hidden_size == self.vision_config.hidden_size: |
| 109 | self.vision_config.mm_hidden_size = self.text_config.hidden_size |
| 110 | |
| 111 | # Other config |
| 112 | self.ignore_index = ignore_index |
| 113 | self.media_placeholder_token_id = media_placeholder_token_id |
| 114 | self.use_unified_vision_chunk = use_unified_vision_chunk |
| 115 | self.video_placeholder = video_placeholder |
| 116 | |
| 117 | # Propagate quantization config from text model |
| 118 | if getattr(self.text_config, "quantization_config", None) is not None: |
| 119 | self.quantization_config = self.text_config.quantization_config |
| 120 | |
| 121 | super().__init__(pad_token_id=pad_token_id, **kwargs) |
| 122 | |
| 123 | @property |
| 124 | def hidden_size(self) -> int: |
| 125 | """Get hidden size from text config for compatibility.""" |
| 126 | return self.text_config.hidden_size |
| 127 | |
| 128 | @property |
| 129 | def vocab_size(self) -> int: |
| 130 | """Get vocab size from text config for compatibility.""" |
| 131 | return self.text_config.vocab_size |