| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Radio vision model configuration.""" |
| 4 | |
| 5 | from typing import Any |
| 6 | |
| 7 | from transformers.configuration_utils import PreTrainedConfig |
| 8 | from transformers.utils import logging |
| 9 | |
| 10 | logger = logging.get_logger(__name__) |
| 11 | |
| 12 | VIT_TIMM_DIM_BY_NAME: dict[str, tuple[int, int, int, int]] = { |
| 13 | "vit_small_patch16_224": (384, 12, 6, 1536), |
| 14 | "vit_base_patch16_224": (768, 12, 12, 3072), |
| 15 | "vit_large_patch16_224": (1024, 24, 16, 4096), |
| 16 | "vit_huge_patch16_224": (1280, 32, 16, 5120), |
| 17 | } |
| 18 | |
| 19 | OPENAI_CLIP_MEAN = (0.48145466, 0.4578275, 0.40821073) |
| 20 | OPENAI_CLIP_STD = (0.26862954, 0.26130258, 0.27577711) |
| 21 | |
| 22 | |
| 23 | class RadioConfig(PreTrainedConfig): |
| 24 | r"""This is the configuration class to store the configuration of a Radio |
| 25 | vision model. It is used to instantiate a Radio model according to the |
| 26 | specified arguments, defining the model architecture. |
| 27 | |
| 28 | Args: |
| 29 | model_name: Name of the vision transformer model |
| 30 | (e.g., "vit_base_patch16_224"). Used to determine architecture |
| 31 | dimensions from `VIT_TIMM_DIM_BY_NAME`. |
| 32 | image_size: The size (resolution) of each image. |
| 33 | patch_size: The size (resolution) of each patch. |
| 34 | qkv_bias: Whether to add a bias to the queries, keys and values. |
| 35 | qk_normalization: Whether to apply normalization to queries and keys. |
| 36 | norm_type: The normalization type to use. |
| 37 | layer_norm_eps: The epsilon used by the layer normalization layers. |
| 38 | initializer_factor: A factor for initializing all weight matrices. |
| 39 | hidden_act: The non-linear activation function in the encoder. |
| 40 | cpe_max_size: Maximum image size for position embeddings. |
| 41 | norm_mean: Mean values for image normalization (RGB channels). |
| 42 | Defaults to (0.48145466, 0.4578275, 0.40821073)). |
| 43 | norm_std: Standard deviation values for image normalization |
| 44 | (RGB channels). Defaults to (0.26862954, 0.26130258, 0.27577711)). |
| 45 | register_multiple: Number of register tokens to use. |
| 46 | teachers: A list of teacher model configurations. Each teacher configuration is |
| 47 | a dict with keys like "name" and some may have "use_summary". |
| 48 | cls_token_per_teacher: Whether to use a separate CLS token for each teacher. |
| 49 | video_temporal_patch_size: Number of consecutive video frames grouped into |
| 50 | a single tubelet for temporal compression. Default 1 (no compression). |
| 51 | When > 1, a dedicated video_embedder (3*T*P*P -> hidden) is created |
| 52 | alongside the image embedder (3*P*P -> hidden). |
| 53 | separate_video_embedder: When True and video_temporal_patch_size > 1, use a |
| 54 | dedicated video patch embedder (3*T*P*P -> hidden) separate from the |
| 55 | image embedder (3*P*P -> hidden). When False, a single embedder with |
| 56 | input size 3*T*P*P is used for both (images are duplicated T times). |
| 57 | |
| 58 | """ |
| 59 | |
| 60 | model_type = "radio" |
| 61 | |
| 62 | def __init__( |
| 63 | self, |
| 64 | model_name: str, |
| 65 | image_size: int = 224, |
| 66 | patch_size: int = 16, |
| 67 | qkv_bias: bool = True, |
| 68 | qk_normalization: bool = False, |
| 69 | norm_type: str = "layer_norm", |
| 70 | layer_norm_eps: float = 1e-6, |
| 71 | initializer_factor: float = 1.0, |
| 72 | hidden_act: str = "gelu", |
| 73 | cpe_max_size: int = 2048, |
| 74 | norm_mean: tuple[float, float, float] | list = OPENAI_CLIP_MEAN, |
| 75 | norm_std: tuple[float, float, float] | list = OPENAI_CLIP_STD, |
| 76 | register_multiple: int | None = None, |
| 77 | teachers: list[dict[str, Any]] | None = None, |
| 78 | cls_token_per_teacher: bool = False, |
| 79 | video_temporal_patch_size: int = 1, |
| 80 | separate_video_embedder: bool = True, |
| 81 | **kwargs, |
| 82 | ): |
| 83 | self.model_name = model_name |
| 84 | ( |
| 85 | self.hidden_size, |
| 86 | self.num_hidden_layers, |
| 87 | self.num_attention_heads, |
| 88 | self.intermediate_size, |
| 89 | ) = VIT_TIMM_DIM_BY_NAME[model_name] |
| 90 | self.image_size = image_size |
| 91 | self.patch_size = patch_size |
| 92 | self.qkv_bias = qkv_bias |
| 93 | self.qk_normalization = qk_normalization |
| 94 | self.norm_type = norm_type |
| 95 | self.layer_norm_eps = layer_norm_eps |
| 96 | self.initializer_factor = initializer_factor |
| 97 | self.hidden_act = hidden_act |
| 98 | self.cpe_max_size = cpe_max_size |
| 99 | self.norm_mean = ( |
| 100 | list(norm_mean) if isinstance(norm_mean, (tuple, list)) else norm_mean |
| 101 | ) |
| 102 | self.norm_std = ( |
| 103 | list(norm_std) if isinstance(norm_std, (tuple, list)) else norm_std |
| 104 | ) |
| 105 | self.register_multiple = register_multiple |
| 106 | self.teachers = teachers if teachers is not None else [] |
| 107 | self.cls_token_per_teacher = cls_token_per_teacher |
| 108 | self.video_temporal_patch_size = video_temporal_patch_size |
| 109 | self.separate_video_embedder = separate_video_embedder |
| 110 | super().__init__(**kwargs) |