| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """ColPali configuration that extends PaliGemmaConfig with embedding projection |
| 4 | fields. This allows ColPali models to be loaded without trust_remote_code |
| 5 | by mapping their custom model_type (colpali) to a standard config class |
| 6 | that vLLM understands. |
| 7 | |
| 8 | Supported model_types: |
| 9 | - colpali (vidore/colpali-v1.3-hf) |
| 10 | """ |
| 11 | |
| 12 | from transformers import PaliGemmaConfig |
| 13 | |
| 14 | |
| 15 | class ColPaliConfig(PaliGemmaConfig): |
| 16 | """Configuration class for ColPali models. |
| 17 | |
| 18 | Extends PaliGemmaConfig with additional fields used by ColPali variants |
| 19 | for the embedding projection layer. |
| 20 | """ |
| 21 | |
| 22 | model_type = "colpali" |
| 23 | |
| 24 | def __init__( |
| 25 | self, |
| 26 | embedding_dim: int | None = None, |
| 27 | embed_dim: int | None = None, |
| 28 | dim: int | None = None, |
| 29 | colbert_dim: int | None = None, |
| 30 | pooling: str | None = None, |
| 31 | vlm_config: dict | None = None, |
| 32 | **kwargs, |
| 33 | ): |
| 34 | # Store embedding projection config fields |
| 35 | self.embedding_dim = embedding_dim |
| 36 | self.embed_dim = embed_dim |
| 37 | self.dim = dim |
| 38 | self.colbert_dim = colbert_dim |
| 39 | self.pooling = pooling |
| 40 | |
| 41 | # The HF checkpoint nests PaliGemma config inside "vlm_config". |
| 42 | # Flatten it so PaliGemmaConfig receives vision_config, text_config, |
| 43 | # image_token_index, etc. directly. |
| 44 | # Use setdefault to avoid overwriting keys already set (e.g. |
| 45 | # model_type="colpali" would be clobbered by "paligemma" from |
| 46 | # vlm_config). |
| 47 | if vlm_config is not None: |
| 48 | vlm_dict = ( |
| 49 | vlm_config if isinstance(vlm_config, dict) else vlm_config.to_dict() |
| 50 | ) |
| 51 | _conflicting = {"model_type", "_name_or_path"} |
| 52 | for key, value in vlm_dict.items(): |
| 53 | if key not in _conflicting: |
| 54 | kwargs.setdefault(key, value) |
| 55 | |
| 56 | super().__init__(**kwargs) |