| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | # ruff: noqa: E501 |
| 5 | # adapted from https://huggingface.co/AIDC-AI/Ovis2-1B/blob/main/configuration_aimv2.py |
| 6 | # and https://huggingface.co/AIDC-AI/Ovis2-1B/blob/main/configuration_ovis.py |
| 7 | # Ovis Config with AimV2 config registration removed for Transformers compatibility |
| 8 | from typing import Any |
| 9 | |
| 10 | from transformers import AutoConfig, PreTrainedConfig |
| 11 | |
| 12 | |
| 13 | class AIMv2Config(PreTrainedConfig): |
| 14 | """This is the configuration class to store the configuration of an [`AIMv2Model`]. |
| 15 | Instantiating a configuration with the defaults will yield a similar configuration |
| 16 | to that of the [apple/aimv2-large-patch14-224](https://huggingface.co/apple/aimv2-large-patch14-224). |
| 17 | |
| 18 | Args: |
| 19 | hidden_size: Dimension of the hidden representations. |
| 20 | intermediate_size: Dimension of the SwiGLU representations. |
| 21 | num_hidden_layers: Number of hidden layers in the Transformer. |
| 22 | num_attention_heads: Number of attention heads for each attention layer |
| 23 | in the Transformer. |
| 24 | num_channels: Number of input channels. |
| 25 | image_size: Image size. |
| 26 | patch_size: Patch size. |
| 27 | rms_norm_eps: Epsilon value used for the RMS normalization layer. |
| 28 | attention_dropout: Dropout ratio for attention probabilities. |
| 29 | projection_dropout: Dropout ratio for the projection layer after the attention. |
| 30 | qkv_bias: Whether to add a bias to the queries, keys and values. |
| 31 | use_bias: Whether to add a bias in the feed-forward and projection layers. |
| 32 | kwargs: Keyword arguments for the [`PreTrainedConfig`]. |
| 33 | |
| 34 | """ |
| 35 | |
| 36 | model_type: str = "aimv2" |
| 37 | |
| 38 | def __init__( |
| 39 | self, |
| 40 | hidden_size: int = 1024, |
| 41 | intermediate_size: int = 2816, |
| 42 | num_hidden_layers: int = 24, |
| 43 | num_attention_heads: int = 8, |
| 44 | num_channels: int = 3, |
| 45 | image_size: int = 224, |
| 46 | patch_size: int = 14, |
| 47 | rms_norm_eps: float = 1e-5, |
| 48 | attention_dropout: float = 0.0, |
| 49 | projection_dropout: float = 0.0, |
| 50 | qkv_bias: bool = False, |
| 51 | use_bias: bool = False, |
| 52 | **kwargs: Any, |
| 53 | ): |
| 54 | super().__init__(**kwargs) |
| 55 | self.hidden_size = hidden_size |
| 56 | self.intermediate_size = intermediate_size |
| 57 | self.num_hidden_layers = num_hidden_layers |
| 58 | self.num_attention_heads = num_attention_heads |
| 59 | self.num_channels = num_channels |
| 60 | self.patch_size = patch_size |
| 61 | self.image_size = image_size |
| 62 | self.attention_dropout = attention_dropout |
| 63 | self.rms_norm_eps = rms_norm_eps |
| 64 | |
| 65 | self.projection_dropout = projection_dropout |
| 66 | self.qkv_bias = qkv_bias |
| 67 | self.use_bias = use_bias |
| 68 | |
| 69 | |
| 70 | # ---------------------------------------------------------------------- |
| 71 | # Visual Tokenizer Configuration |
| 72 | # ---------------------------------------------------------------------- |
| 73 | class BaseVisualTokenizerConfig(PreTrainedConfig): |
| 74 | def __init__( |
| 75 | self, |
| 76 | vocab_size=16384, |
| 77 | tokenize_function="softmax", |
| 78 | tau=1.0, |
| 79 | depths=None, |
| 80 | drop_cls_token=False, |
| 81 | backbone_config: PreTrainedConfig | dict | None = None, |
| 82 | hidden_stride: int = 1, |
| 83 | **kwargs, |
| 84 | ): |
| 85 | super().__init__(**kwargs) |
| 86 | self.vocab_size = vocab_size |
| 87 | self.tokenize_function = tokenize_function |
| 88 | self.tau = tau |
| 89 | if isinstance(depths, str): |
| 90 | depths = [int(x) for x in depths.split("|")] |
| 91 | self.depths = depths |
| 92 | self.backbone_kwargs = dict[str, Any]() |
| 93 | self.drop_cls_token = drop_cls_token |
| 94 | if backbone_config is not None: |
| 95 | assert isinstance(backbone_config, (PreTrainedConfig, dict)), ( |
| 96 | f"expect `backbone_config` to be instance of PreTrainedConfig or dict, but got {type(backbone_config)} type" |
| 97 | ) |
| 98 | if not isinstance(backbone_config, PreTrainedConfig): |
| 99 | model_type = backbone_config["model_type"] |
| 100 | if model_type != "aimv2": |
| 101 | backbone_config.pop("model_type") |
| 102 | backbone_config = AutoConfig.for_model( |
| 103 | model_type, **backbone_config |
| 104 | ) |
| 105 | else: |
| 106 | backbone_config = AIMv2Config(**backbone_config) |
| 107 | self.backbone_config = backbone_config |
| 108 | self.hidden_stride = hidden_stride |
| 109 | |
| 110 | |
| 111 | class Aimv2VisualTokenizerConfig(BaseVisualTokenizerConfig): |
| 112 | model_type = "aimv2_visual_tokenizer" |
| 113 | |
| 114 | def __init__(self, **kwargs): |
| 115 | super().__init__(**kwargs) |
| 116 | if self.drop_cls_token: |
| 117 | self.drop_cls_token = False |
| 118 | if self.depths: |
| 119 | assert len(self.depths) == 1 |
| 120 | self.backbone_kwargs["num_hidden_layers"] = self.depths[0] |
| 121 | |
| 122 | |
| 123 | class SiglipVisualTokenizerConfig(BaseVisualTokenizerConfig): |
| 124 | model_type = "siglip_visual_tokenizer" |
| 125 | |
| 126 | def __init__(self, **kwargs): |
| 127 | super().__init__(**kwargs) |
| 128 | if self.drop_cls_token: |
| 129 | self.drop_cls_token = False |
| 130 | if self.depths: |
| 131 | assert len(self.depths) == 1 |
| 132 | self.backbone_kwargs["num_hidden_layers"] = self.depths[0] |
| 133 | |
| 134 | |
| 135 | AutoConfig.register("siglip_visual_tokenizer", SiglipVisualTokenizerConfig) |
| 136 | AutoConfig.register("aimv2_visual_tokenizer", Aimv2VisualTokenizerConfig) |
| 137 | |
| 138 | |
| 139 | # ---------------------------------------------------------------------- |
| 140 | # Ovis Configuration |
| 141 | # ---------------------------------------------------------------------- |
| 142 | class OvisConfig(PreTrainedConfig): |
| 143 | model_type = "ovis" |
| 144 | |
| 145 | def __init__( |
| 146 | self, |
| 147 | llm_config: PreTrainedConfig | dict | None = None, |
| 148 | visual_tokenizer_config: PreTrainedConfig | dict | None = None, |
| 149 | multimodal_max_length=8192, |
| 150 | hidden_size=None, |
| 151 | conversation_formatter_class=None, |
| 152 | llm_attn_implementation=None, |
| 153 | disable_tie_weight=False, |
| 154 | **kwargs, |
| 155 | ): |
| 156 | super().__init__(**kwargs) |
| 157 | if llm_config is not None: |
| 158 | assert isinstance(llm_config, (PreTrainedConfig, dict)), ( |
| 159 | f"expect `llm_config` to be instance of PreTrainedConfig or dict, but got {type(llm_config)} type" |
| 160 | ) |
| 161 | if not isinstance(llm_config, PreTrainedConfig): |
| 162 | model_type = llm_config["model_type"] |
| 163 | llm_config.pop("model_type") |
| 164 | llm_config = AutoConfig.for_model(model_type, **llm_config) |
| 165 | |
| 166 | # map llm_config to text_config |
| 167 | self.text_config = llm_config |
| 168 | if visual_tokenizer_config is not None: |
| 169 | assert isinstance(visual_tokenizer_config, (PreTrainedConfig, dict)), ( |
| 170 | f"expect `visual_tokenizer_config` to be instance of PreTrainedConfig or dict, but got {type(visual_tokenizer_config)} type" |
| 171 | ) |
| 172 | if not isinstance(visual_tokenizer_config, PreTrainedConfig): |
| 173 | model_type = visual_tokenizer_config["model_type"] |
| 174 | visual_tokenizer_config.pop("model_type") |
| 175 | visual_tokenizer_config = AutoConfig.for_model( |
| 176 | model_type, **visual_tokenizer_config |
| 177 | ) |
| 178 | |
| 179 | self.visual_tokenizer_config = visual_tokenizer_config |
| 180 | self.multimodal_max_length = multimodal_max_length |
| 181 | self.hidden_size = hidden_size |
| 182 | self.conversation_formatter_class = conversation_formatter_class |
| 183 | self.llm_attn_implementation = llm_attn_implementation |
| 184 | self.disable_tie_weight = disable_tie_weight |