| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | from typing import Any |
| 5 | |
| 6 | import transformers |
| 7 | |
| 8 | |
| 9 | class Granite4VisionConfig(transformers.PreTrainedConfig): |
| 10 | """Configuration for Granite 4 Vision model. |
| 11 | |
| 12 | This config is needed because the granite4_vision model type is not yet |
| 13 | in the transformers version pinned by vLLM. Once transformers adds native |
| 14 | support, this file can be removed and the _CONFIG_REGISTRY entry dropped. |
| 15 | """ |
| 16 | |
| 17 | model_type = "granite4_vision" |
| 18 | is_composition = False |
| 19 | |
| 20 | def __init__( |
| 21 | self, |
| 22 | vision_config: dict[str, Any] | None = None, |
| 23 | text_config: dict[str, Any] | None = None, |
| 24 | image_token_index: int = 100352, |
| 25 | image_seq_length: int = 576, |
| 26 | image_grid_pinpoints: list[list[int]] | None = None, |
| 27 | vision_feature_select_strategy: str = "full", |
| 28 | vision_feature_layer: int | list[int] = -2, |
| 29 | projector_hidden_act: str = "gelu", |
| 30 | projector_dropout: float = 0.1, |
| 31 | downsample_rate: str | None = None, |
| 32 | use_image_newline_parameter: bool = True, |
| 33 | deepstack_layer_map: list[list[int]] | None = None, |
| 34 | use_spatial_sampling: bool = False, |
| 35 | spatial_stride: int = 2, |
| 36 | spatial_vision_layer: int = -1, |
| 37 | spatial_target_layers: list[int] | None = None, |
| 38 | # Hub aliases — base model config uses different field names |
| 39 | vision_layer_to_llm_layer: list[list[int]] | None = None, |
| 40 | use_checkerboard_sampling: bool | None = None, |
| 41 | checkerboard_stride: int | None = None, |
| 42 | checkerboard_vision_layer: int | None = None, |
| 43 | checkerboard_llm_layers: list[int] | None = None, |
| 44 | **kwargs: Any, |
| 45 | ): |
| 46 | self.image_token_index = image_token_index |
| 47 | self.image_seq_length = image_seq_length |
| 48 | self.image_grid_pinpoints = image_grid_pinpoints or [] |
| 49 | self.vision_feature_select_strategy = vision_feature_select_strategy |
| 50 | self.vision_feature_layer = vision_feature_layer |
| 51 | self.projector_hidden_act = projector_hidden_act |
| 52 | self.projector_dropout = projector_dropout |
| 53 | self.downsample_rate = downsample_rate |
| 54 | self.use_image_newline_parameter = use_image_newline_parameter |
| 55 | self.deepstack_layer_map = deepstack_layer_map or vision_layer_to_llm_layer |
| 56 | self.use_spatial_sampling = ( |
| 57 | use_spatial_sampling |
| 58 | if use_checkerboard_sampling is None |
| 59 | else use_checkerboard_sampling |
| 60 | ) |
| 61 | self.spatial_stride = ( |
| 62 | spatial_stride if checkerboard_stride is None else checkerboard_stride |
| 63 | ) |
| 64 | self.spatial_vision_layer = ( |
| 65 | spatial_vision_layer |
| 66 | if checkerboard_vision_layer is None |
| 67 | else checkerboard_vision_layer |
| 68 | ) |
| 69 | self.spatial_target_layers = ( |
| 70 | spatial_target_layers or checkerboard_llm_layers or [0, 10, 20, 30] |
| 71 | ) |
| 72 | |
| 73 | if vision_config is None: |
| 74 | vision_config = {} |
| 75 | if text_config is None: |
| 76 | text_config = {} |
| 77 | |
| 78 | vision_model_type = vision_config.get("model_type", "siglip_vision_model") |
| 79 | if vision_model_type in transformers.CONFIG_MAPPING: |
| 80 | self.vision_config = transformers.CONFIG_MAPPING[vision_model_type]( |
| 81 | **vision_config |
| 82 | ) |
| 83 | else: |
| 84 | self.vision_config = transformers.PreTrainedConfig(**vision_config) |
| 85 | |
| 86 | text_model_type = text_config.get("model_type", "granite") |
| 87 | if text_model_type in transformers.CONFIG_MAPPING: |
| 88 | self.text_config = transformers.CONFIG_MAPPING[text_model_type]( |
| 89 | **text_config |
| 90 | ) |
| 91 | else: |
| 92 | self.text_config = transformers.PreTrainedConfig(**text_config) |
| 93 | |
| 94 | super().__init__(**kwargs) |