| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """OpenVLA configuration support. |
| 4 | |
| 5 | OpenVLA checkpoints use a custom ``model_type`` and nest the language model |
| 6 | configuration under ``text_config``. This shim lets vLLM load the checkpoint |
| 7 | configuration without executing Hugging Face remote code. |
| 8 | """ |
| 9 | |
| 10 | from typing import Any |
| 11 | |
| 12 | from transformers import LlamaConfig, PreTrainedConfig |
| 13 | |
| 14 | |
| 15 | class OpenVLAConfig(PreTrainedConfig): |
| 16 | """Configuration class for OpenVLA models.""" |
| 17 | |
| 18 | model_type = "openvla" |
| 19 | |
| 20 | def __init__( |
| 21 | self, |
| 22 | timm_model_ids: list[str] | None = None, |
| 23 | timm_override_act_layers: list[str | None] | None = None, |
| 24 | image_sizes: list[int] | None = None, |
| 25 | use_fused_vision_backbone: bool = True, |
| 26 | image_token_index: int = 32000, |
| 27 | n_action_bins: int = 256, |
| 28 | text_config: dict[str, Any] | LlamaConfig | None = None, |
| 29 | **kwargs: Any, |
| 30 | ) -> None: |
| 31 | kwargs.setdefault("architectures", ["OpenVLAForActionPrediction"]) |
| 32 | super().__init__(**kwargs) |
| 33 | |
| 34 | self.timm_model_ids = timm_model_ids or [ |
| 35 | "vit_large_patch14_reg4_dinov2.lvd142m", |
| 36 | "vit_so400m_patch14_siglip_224", |
| 37 | ] |
| 38 | self.timm_override_act_layers = timm_override_act_layers or [None, None] |
| 39 | self.image_sizes = image_sizes or [224, 224] |
| 40 | self.use_fused_vision_backbone = use_fused_vision_backbone |
| 41 | self.image_token_index = image_token_index |
| 42 | self.n_action_bins = n_action_bins |
| 43 | |
| 44 | if text_config is None: |
| 45 | text_config = LlamaConfig(architectures=["LlamaForCausalLM"]) |
| 46 | elif isinstance(text_config, dict): |
| 47 | text_config = text_config.copy() |
| 48 | text_config.setdefault("architectures", ["LlamaForCausalLM"]) |
| 49 | text_config = LlamaConfig(**text_config) |
| 50 | self.text_config = text_config |