| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | |
| 5 | from transformers import PreTrainedConfig |
| 6 | |
| 7 | |
| 8 | class Mimo_VLVisionConfig(PreTrainedConfig): |
| 9 | model_type = "mimovl" |
| 10 | base_config_key = "vision_config" |
| 11 | |
| 12 | def __init__( |
| 13 | self, |
| 14 | depth=28, |
| 15 | hidden_size=1280, |
| 16 | hidden_act="silu", |
| 17 | intermediate_size=4608, |
| 18 | num_heads=32, |
| 19 | in_channels=3, |
| 20 | patch_size=16, |
| 21 | spatial_merge_size=2, |
| 22 | temporal_patch_size=2, |
| 23 | tokens_per_second=2, |
| 24 | window_size=128, |
| 25 | out_hidden_size=2048, |
| 26 | fullatt_block_indexes=None, |
| 27 | initializer_range=0.02, |
| 28 | kv_channels=64, # HACK |
| 29 | qk_channels=64, |
| 30 | num_query_groups=4, |
| 31 | num_key_value_heads=8, |
| 32 | vit_window_attn_types=None, |
| 33 | visual_token_window_size=64, |
| 34 | **kwargs, |
| 35 | ): |
| 36 | super().__init__(**kwargs) |
| 37 | |
| 38 | self.depth = depth |
| 39 | self.hidden_size = hidden_size |
| 40 | self.hidden_act = hidden_act |
| 41 | self.intermediate_size = intermediate_size |
| 42 | self.num_heads = num_heads |
| 43 | # Support GQA: if num_key_value_heads is not provided, |
| 44 | # default to num_heads (MHA) |
| 45 | if num_key_value_heads is None: |
| 46 | num_key_value_heads = num_heads |
| 47 | self.num_key_value_heads = num_key_value_heads |
| 48 | self.in_channels = in_channels |
| 49 | self.patch_size = patch_size |
| 50 | self.spatial_merge_size = spatial_merge_size |
| 51 | self.temporal_patch_size = temporal_patch_size |
| 52 | self.tokens_per_second = tokens_per_second |
| 53 | self.window_size = window_size |
| 54 | self.fullatt_block_indexes = ( |
| 55 | fullatt_block_indexes |
| 56 | if fullatt_block_indexes is not None |
| 57 | else [7, 15, 23, 31] |
| 58 | ) |
| 59 | self.out_hidden_size = out_hidden_size |
| 60 | self.initializer_range = initializer_range |
| 61 | self.kv_channels = kv_channels |
| 62 | self.qk_channels = qk_channels |
| 63 | self.num_query_groups = num_query_groups |
| 64 | self.vit_window_attn_types = vit_window_attn_types or [-1] * depth |
| 65 | self.visual_token_window_size = visual_token_window_size |