| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | # Adapted from https://huggingface.co/moonshotai/Kimi-VL-A3B-Instruct/blob/main/configuration_kimi_vl.py |
| 4 | from transformers.configuration_utils import PreTrainedConfig |
| 5 | |
| 6 | |
| 7 | class MoonViTConfig(PreTrainedConfig): |
| 8 | model_type = "moonvit" |
| 9 | |
| 10 | def __init__( |
| 11 | self, |
| 12 | patch_size: int = 14, |
| 13 | init_pos_emb_height: int = 64, |
| 14 | init_pos_emb_width: int = 64, |
| 15 | num_attention_heads: int = 16, |
| 16 | num_hidden_layers: int = 27, |
| 17 | hidden_size: int = 1152, |
| 18 | intermediate_size: int = 4304, |
| 19 | merge_kernel_size: tuple[int, int] = (2, 2), |
| 20 | **kwargs, |
| 21 | ): |
| 22 | super().__init__(**kwargs) |
| 23 | self.patch_size = patch_size |
| 24 | # Positional embedding config |
| 25 | self.init_pos_emb_height = init_pos_emb_height |
| 26 | self.init_pos_emb_width = init_pos_emb_width |
| 27 | # Transformer config |
| 28 | self.num_hidden_layers = num_hidden_layers |
| 29 | self.num_attention_heads = num_attention_heads |
| 30 | self.hidden_size = hidden_size |
| 31 | self.intermediate_size = intermediate_size |
| 32 | # Patch merger config |
| 33 | self.merge_kernel_size = merge_kernel_size |