| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | from typing import Any |
| 4 | |
| 5 | from transformers.configuration_utils import PreTrainedConfig |
| 6 | from transformers.models.qwen2 import Qwen2Config |
| 7 | |
| 8 | |
| 9 | class DotsVisionConfig(PreTrainedConfig): |
| 10 | model_type: str = "dots_vit" |
| 11 | |
| 12 | def __init__( |
| 13 | self, |
| 14 | embed_dim: int = 1536, # vision encoder embed size |
| 15 | hidden_size: int = 1536, # after merger hidden size |
| 16 | intermediate_size: int = 4224, |
| 17 | num_hidden_layers: int = 42, |
| 18 | num_attention_heads: int = 12, |
| 19 | num_channels: int = 3, |
| 20 | patch_size: int = 14, |
| 21 | spatial_merge_size: int = 2, |
| 22 | temporal_patch_size: int = 1, |
| 23 | rms_norm_eps: float = 1e-5, |
| 24 | use_bias: bool = False, |
| 25 | attn_implementation="flash_attention_2", |
| 26 | initializer_range=0.02, |
| 27 | init_merger_std=0.02, |
| 28 | is_causal=False, # ve causal forward |
| 29 | post_norm=True, |
| 30 | gradient_checkpointing=False, |
| 31 | **kwargs: Any, |
| 32 | ): |
| 33 | super().__init__(**kwargs) |
| 34 | self.embed_dim = embed_dim |
| 35 | self.hidden_size = hidden_size |
| 36 | self.intermediate_size = intermediate_size |
| 37 | self.num_hidden_layers = num_hidden_layers |
| 38 | self.num_attention_heads = num_attention_heads |
| 39 | self.num_channels = num_channels |
| 40 | self.patch_size = patch_size |
| 41 | self.spatial_merge_size = spatial_merge_size |
| 42 | self.temporal_patch_size = temporal_patch_size |
| 43 | self.rms_norm_eps = rms_norm_eps |
| 44 | self.use_bias = use_bias |
| 45 | self.attn_implementation = attn_implementation |
| 46 | self.initializer_range = initializer_range |
| 47 | self.init_merger_std = init_merger_std |
| 48 | self.is_causal = is_causal |
| 49 | self.post_norm = post_norm |
| 50 | self.gradient_checkpointing = gradient_checkpointing |
| 51 | |
| 52 | |
| 53 | class DotsOCRConfig(Qwen2Config): |
| 54 | model_type = "dots_ocr" |
| 55 | |
| 56 | def __init__( |
| 57 | self, |
| 58 | image_token_id=151665, |
| 59 | video_token_id=151656, |
| 60 | vision_config: dict | None = None, |
| 61 | *args, |
| 62 | **kwargs, |
| 63 | ): |
| 64 | super().__init__(*args, **kwargs) |
| 65 | self.image_token_id = image_token_id |
| 66 | self.video_token_id = video_token_id |
| 67 | self.vision_config = DotsVisionConfig(**(vision_config or {})) |
| 68 | |
| 69 | def save_pretrained(self, save_directory, **kwargs): |
| 70 | self._auto_class = None |
| 71 | super().save_pretrained(save_directory, **kwargs) |