| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import math |
| 5 | import os |
| 6 | from collections.abc import Callable, Iterator, Sequence |
| 7 | from contextlib import contextmanager |
| 8 | from dataclasses import asdict |
| 9 | from functools import cache, partial, wraps |
| 10 | from pathlib import Path |
| 11 | from typing import Any, Literal, TypeAlias |
| 12 | |
| 13 | import huggingface_hub |
| 14 | import torch |
| 15 | import transformers.configuration_utils as hf_configuration_utils |
| 16 | from huggingface_hub import constants |
| 17 | from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE |
| 18 | from transformers import GenerationConfig, PreTrainedConfig |
| 19 | from transformers.configuration_utils import ALLOWED_LAYER_TYPES |
| 20 | from transformers.models.auto.image_processing_auto import get_image_processor_config |
| 21 | from transformers.models.auto.modeling_auto import ( |
| 22 | MODEL_MAPPING_NAMES, |
| 23 | ) |
| 24 | from transformers.models.auto.tokenization_auto import get_tokenizer_config |
| 25 | from transformers.utils import CONFIG_NAME as HF_CONFIG_NAME |
| 26 | |
| 27 | from vllm import envs |
| 28 | from vllm.logger import init_logger |
| 29 | from vllm.transformers_utils.repo_utils import is_mistral_model_repo |
| 30 | from vllm.transformers_utils.utils import ( |
| 31 | parse_safetensors_file_metadata, |
| 32 | without_trust_remote_code, |
| 33 | ) |
| 34 | from vllm.utils.torch_utils import common_broadcastable_dtype |
| 35 | |
| 36 | from .config_parser_base import ConfigParserBase |
| 37 | from .repo_utils import ( |
| 38 | file_or_path_exists, |
| 39 | get_hf_file_to_dict, |
| 40 | hf_api, |
| 41 | list_repo_files, |
| 42 | try_get_local_file, |
| 43 | with_retry, |
| 44 | ) |
| 45 | |
| 46 | if envs.VLLM_USE_MODELSCOPE: |
| 47 | from modelscope import AutoConfig |
| 48 | else: |
| 49 | from transformers import AutoConfig |
| 50 | |
| 51 | MISTRAL_CONFIG_NAME = "params.json" |
| 52 | |
| 53 | logger = init_logger(__name__) |
| 54 | |
| 55 | _ST_POOLING_MODULE_TYPES = { |
| 56 | "sentence_transformers.models.Pooling", |
| 57 | "sentence_transformers.sentence_transformer.modules.pooling.Pooling", |
| 58 | } |
| 59 | _ST_NORMALIZE_MODULE_TYPES = { |
| 60 | "sentence_transformers.models.Normalize", |
| 61 | "sentence_transformers.base.modules.normalize.Normalize", |
| 62 | "sentence_transformers.sentence_transformer.modules.normalize.Normalize", |
| 63 | } |
| 64 | _DENSE_MODULE_TYPES = { |
| 65 | "sentence_transformers.models.Dense", |
| 66 | "sentence_transformers.base.modules.dense.Dense", |
| 67 | "pylate.models.Dense.Dense", |
| 68 | } |
| 69 | |
| 70 | |
| 71 | class LazyConfigDict(dict): |
| 72 | def __getitem__(self, key): |
| 73 | if isinstance(value := super().__getitem__(key), type): |
| 74 | return value |
| 75 | |
| 76 | import vllm.transformers_utils.configs as configs |
| 77 | |
| 78 | return getattr(configs, value) |
| 79 | |
| 80 | |
| 81 | _CONFIG_REGISTRY: dict[str, type[PreTrainedConfig]] = LazyConfigDict( |
| 82 | afmoe="AfmoeConfig", |
| 83 | axk1="AXK1Config", |
| 84 | bagel="BagelConfig", |
| 85 | bailing_moe_v3_vl="BailingMoeV3VLConfig", |
| 86 | chatglm="ChatGLMConfig", |
| 87 | modernvbert="ColModernVBertConfig", |
| 88 | colpali="ColPaliConfig", |
| 89 | colqwen3="ColQwen3Config", |
| 90 | ops_colqwen3="OpsColQwen3Config", |
| 91 | qwen3_vl_nemotron_embed="Qwen3VLNemotronEmbedConfig", |
| 92 | cosmos3_omni="Cosmos3Config", |
| 93 | cosmos3_edge="Cosmos3EdgeConfig", |
| 94 | diffusion_gemma="DiffusionGemmaConfig", |
| 95 | deepseek_vl_v2="DeepseekVLV2Config", |
| 96 | deepseek_v32="DeepseekV3Config", |
| 97 | deepseek_v4="DeepseekV4Config", |
| 98 | dots3_note="Dots3NoteConfig", |
| 99 | k3_dspark="K3DSparkConfig", |
| 100 | funaudiochat="FunAudioChatConfig", |
| 101 | granite4_vision="Granite4VisionConfig", |
| 102 | hyperclovax="HyperCLOVAXConfig", |
| 103 | hy_v3="HYV3Config", |
| 104 | hy_v4="HYV4Config", |
| 105 | isaac="IsaacConfig", |
| 106 | kimi_k2="DeepseekV3Config", # Kimi K2 uses same architecture as DeepSeek V3 |
| 107 | kimi_linear="KimiLinearConfig", |
| 108 | kimi_vl="KimiVLConfig", |
| 109 | kimi_k25="KimiK25Config", |
| 110 | muse_glimmer="MuseGlimmerConfig", |
| 111 | muse_glimmer_text="MuseGlimmerTextConfig", |
| 112 | muse_glimmer_vision="MuseGlimmerVisionConfig", |
| 113 | muse_glimmer_assistant="MuseGlimmerAssistantConfig", |
| 114 | kimi_k3="KimiK3Config", |
| 115 | RefinedWeb="RWConfig", # For tiiuae/falcon-40b(-instruct) |
| 116 | RefinedWebModel="RWConfig", # For tiiuae/falcon-7b(-instruct) |
| 117 | mlp_speculator="MLPSpeculatorConfig", |
| 118 | medusa="MedusaConfig", |
| 119 | midashenglm="MiDashengLMConfig", |
| 120 | minimax_m3_vl="MiniMaxM3Config", |
| 121 | minimax_m3_mtp="MiniMaxM3MTPConfig", |
| 122 | moondream3="Moondream3Config", |
| 123 | moss_transcribe_diarize="MossTranscribeDiarizeConfig", |
| 124 | eagle="EAGLEConfig", |
| 125 | speculators="SpeculatorsConfig", |
| 126 | nemotron="NemotronConfig", |
| 127 | olmo_hybrid="OlmoHybridConfig", |
| 128 | openvla="OpenVLAConfig", |
| 129 | ovis="OvisConfig", |
| 130 | ultravox="UltravoxConfig", |
| 131 | step3_vl="Step3VLConfig", |
| 132 | step3_text="Step3TextConfig", |
| 133 | step3p5="Step3p5Config", |
| 134 | qianfan_ocr="QianfanOCRConfig", |
| 135 | qwen3_asr="Qwen3ASRConfig", |
| 136 | qwen3_next="Qwen3NextConfig", |
| 137 | qwen3_5="Qwen3_5Config", |
| 138 | qwen3_5_text="Qwen3_5TextConfig", |
| 139 | qwen3_5_moe="Qwen3_5MoeConfig", |
| 140 | qwen3_5_moe_text="Qwen3_5MoeTextConfig", |
| 141 | lfm2_moe="Lfm2MoeConfig", |
| 142 | **{"unlimited-ocr": "UnlimitedOCRConfig"}, |
| 143 | **{"deepseek_v41": "DeepseekV41Config"}, |
| 144 | inkling_mm_model="InklingMMConfig", |
| 145 | inkling_model="InklingModelConfig", |
| 146 | ) |
| 147 | |
| 148 | _SPECULATIVE_DECODING_CONFIGS: set[str] = {"eagle", "speculators", "medusa"} |
| 149 | |
| 150 | _PATCH_HF_VALIDATE_ROPE: set[str] = {"sarvam_mla"} |
| 151 | |
| 152 | # Model types whose checkpoints carry shared RoPE parameters alongside the |
| 153 | # per-layer-type dicts. Since transformers 5.17, `validate_rope` treats every |
| 154 | # top-level value of such a dict as a layer's parameters and raises on the |
| 155 | # shared ones. `laguna` gets them injected by `convert_rope_params_to_dict`; |
| 156 | # `gemma4_text` ships them in the checkpoint itself. |
| 157 | _PATCH_HF_NESTED_ROPE_VALIDATION: set[str] = {"laguna", "gemma4_text"} |
| 158 | |
| 159 | # Model types whose checkpoints declare `layer_types` entries that upstream |
| 160 | # transformers has not added to `ALLOWED_LAYER_TYPES` yet, so its strict config |
| 161 | # validation rejects them (e.g. GLM-5.2 `glm_moe_dsa` use |
| 162 | # `deepseek_sparse_attention`). Extend the allowed set for these model types. |
| 163 | _PATCH_HF_ALLOWED_LAYER_TYPES: dict[str, tuple[str, ...]] = { |
| 164 | "glm_moe_dsa": ("deepseek_sparse_attention",), |
| 165 | } |
| 166 | |
| 167 | _CONFIG_ATTRS_MAPPING: dict[str, str] = { |
| 168 | "llm_config": "text_config", |
| 169 | } |
| 170 | |
| 171 | _AUTO_CONFIG_KWARGS_OVERRIDES: dict[str, dict[str, Any]] = { |
| 172 | "internvl_chat": {"has_no_defaults_at_init": True}, |
| 173 | "Llama_Nemotron_Nano_VL": {"attn_implementation": "eager"}, |
| 174 | "NVLM_D": {"has_no_defaults_at_init": True}, |
| 175 | } |
| 176 | |
| 177 | |
| 178 | def _register_config_class( |
| 179 | model_type: str, config_class: type[PreTrainedConfig] |
| 180 | ) -> None: |
| 181 | config_class.model_type = model_type |
| 182 | AutoConfig.register(model_type, config_class, exist_ok=True) |
| 183 | |
| 184 | |
| 185 | def _maybe_register_hf_config(config: PreTrainedConfig | None) -> None: |
| 186 | if config is None: |
| 187 | return |
| 188 | |
| 189 | model_type = getattr(config, "model_type", None) |
| 190 | if isinstance(model_type, str) and model_type in _CONFIG_REGISTRY: |
| 191 | _register_config_class(model_type, _CONFIG_REGISTRY[model_type]) |
| 192 | |
| 193 | |
| 194 | def is_rope_parameters_nested(rope_parameters: dict[str, Any]) -> bool: |
| 195 | """Check if rope_parameters is nested by layer types.""" |
| 196 | # Cannot be nested if rope_parameters is empty |
| 197 | if not rope_parameters: |
| 198 | return False |
| 199 | return set(rope_parameters.keys()).issubset(ALLOWED_LAYER_TYPES) |
| 200 | |
| 201 | |
| 202 | @contextmanager |
| 203 | def _mistral_patch_hf_hub_constants() -> Iterator[None]: |
| 204 | hf_safetensors_single_file = constants.SAFETENSORS_SINGLE_FILE |
| 205 | hf_safetensors_index_file = constants.SAFETENSORS_INDEX_FILE |
| 206 | constants.SAFETENSORS_SINGLE_FILE = "consolidated.safetensors" |
| 207 | constants.SAFETENSORS_INDEX_FILE = "consolidated.safetensors.index.json" |
| 208 | try: |
| 209 | yield |
| 210 | finally: |
| 211 | constants.SAFETENSORS_SINGLE_FILE = hf_safetensors_single_file |
| 212 | constants.SAFETENSORS_INDEX_FILE = hf_safetensors_index_file |
| 213 | |
| 214 | |
| 215 | def _install_hf_config_validator( |
| 216 | name: str, validator: Callable, supersedes: Callable |
| 217 | ) -> None: |
| 218 | """Replace a ``PreTrainedConfig`` validator on every ``@strict`` snapshot. |
| 219 | |
| 220 | ``@strict`` snapshots each ``validate_*`` method into ``__class_validators__`` |
| 221 | at class creation and automatic post-``__init__`` validation dispatches off |
| 222 | that frozen list, so assigning the class attribute alone only affects |
| 223 | explicit ``config.validate_*()`` calls. Every ``@strict``-decorated config |
| 224 | class owns a snapshot, so rewrite them all, matching ``supersedes`` by |
| 225 | identity to leave a genuine per-model override in place. |
| 226 | """ |
| 227 | setattr(PreTrainedConfig, name, validator) |
| 228 | |
| 229 | seen: set[int] = set() |
| 230 | stack = [PreTrainedConfig] |
| 231 | while stack: |
| 232 | cls = stack.pop() |
| 233 | if id(cls) in seen: |
| 234 | continue |
| 235 | seen.add(id(cls)) |
| 236 | validators = cls.__dict__.get("__class_validators__") |
| 237 | if isinstance(validators, list) and any(v is supersedes for v in validators): |
| 238 | cls.__class_validators__ = [ |
| 239 | validator if v is supersedes else v for v in validators |
| 240 | ] |
| 241 | stack.extend(cls.__subclasses__()) |
| 242 | |
| 243 | |
| 244 | def _patch_hf_transformers_validate_rope(): |
| 245 | """Transformers v5 moved the ignore_keys option from the method signature of |
| 246 | validate_rope and replaced it with the ignore_keys_at_rope_validation parameter |
| 247 | in the PreTrainedConfig class. This is a patch to make older versions of |
| 248 | validate_rope() with the ignore_keys parameter work with newer versions of |
| 249 | hf transformers (from v5 onwards) |
| 250 | """ |
| 251 | if hasattr(PreTrainedConfig.validate_rope, "__vllm_patched__"): |
| 252 | return |
| 253 | |
| 254 | _original_validate_rope = PreTrainedConfig.validate_rope |
| 255 | |
| 256 | @wraps(_original_validate_rope) |
| 257 | def patched_validate_rope(self, *args, **kwargs): |
| 258 | ignore_keys_param = kwargs.pop("ignore_keys", None) |
| 259 | original_ignore_keys = self.ignore_keys_at_rope_validation |
| 260 | if ignore_keys_param is not None: |
| 261 | logger.warning( |
| 262 | "validate_rope() was called with the legacy 'ignore_keys' " |
| 263 | "argument; use 'ignore_keys_at_rope_validation' instead" |
| 264 | ) |
| 265 | self.ignore_keys_at_rope_validation = original_ignore_keys or ignore_keys_param |
| 266 | result = _original_validate_rope(self, *args, **kwargs) |
| 267 | return result |
| 268 | |
| 269 | patched_validate_rope.__vllm_patched__ = True # type: ignore[attr-defined] |
| 270 | _install_hf_config_validator( |
| 271 | "validate_rope", patched_validate_rope, _original_validate_rope |
| 272 | ) |
| 273 | |
| 274 | |
| 275 | def _patch_hf_transformers_nested_rope_validation() -> None: |
| 276 | """Drop shared entries sitting alongside a nested ``rope_parameters`` dict. |
| 277 | |
| 278 | Transformers validates a dict with any layer-type key by iterating all of |
| 279 | its values, so a shared entry next to the per-layer dicts raises an |
| 280 | ``AttributeError``. The per-layer dicts already carry their own defaults by |
| 281 | the time validation runs, so the shared entries can be dropped. |
| 282 | """ |
| 283 | if hasattr(PreTrainedConfig.validate_rope, "__vllm_nested_rope_patched__"): |
| 284 | return |
| 285 | |
| 286 | _original_validate_rope = PreTrainedConfig.validate_rope |
| 287 | |
| 288 | @wraps(_original_validate_rope) |
| 289 | def patched_validate_rope(self, *args, **kwargs): |
| 290 | rope_parameters = getattr(self, "rope_parameters", None) |
| 291 | if isinstance(rope_parameters, dict): |
| 292 | layer_types = set(rope_parameters) & set( |
| 293 | hf_configuration_utils.ALLOWED_LAYER_TYPES |
| 294 | ) |
| 295 | if layer_types and (shared_keys := set(rope_parameters) - layer_types): |
| 296 | for key in shared_keys: |
| 297 | del rope_parameters[key] |
| 298 | logger.warning( |
| 299 | "Dropping %s from rope_parameters; entries shared across " |
| 300 | "layer types are not supported", |
| 301 | sorted(shared_keys), |
| 302 | ) |
| 303 | return _original_validate_rope(self, *args, **kwargs) |
| 304 | |
| 305 | patched_validate_rope.__vllm_nested_rope_patched__ = True # type: ignore[attr-defined] |
| 306 | _install_hf_config_validator( |
| 307 | "validate_rope", patched_validate_rope, _original_validate_rope |
| 308 | ) |
| 309 | |
| 310 | |
| 311 | def _patch_hf_transformers_allowed_layer_types( |
| 312 | extra_layer_types: tuple[str, ...], |
| 313 | ) -> None: |
| 314 | """Extend transformers' ``ALLOWED_LAYER_TYPES`` so its strict config |
| 315 | validation accepts layer types (e.g. ``deepseek_sparse_attention``) that a |
| 316 | checkpoint declares but upstream transformers has not registered yet. |
| 317 | """ |
| 318 | import transformers.configuration_utils as hf_configuration_utils |
| 319 | |
| 320 | missing = tuple( |
| 321 | layer_type |
| 322 | for layer_type in extra_layer_types |
| 323 | if layer_type not in hf_configuration_utils.ALLOWED_LAYER_TYPES |
| 324 | ) |
| 325 | if missing: |
| 326 | hf_configuration_utils.ALLOWED_LAYER_TYPES += missing |
| 327 | |
| 328 | |
| 329 | class HFConfigParser(ConfigParserBase): |
| 330 | def parse( |
| 331 | self, |
| 332 | model: str | Path, |
| 333 | trust_remote_code: bool, |
| 334 | revision: str | None = None, |
| 335 | code_revision: str | None = None, |
| 336 | **kwargs, |
| 337 | ) -> tuple[dict, PreTrainedConfig]: |
| 338 | kwargs["local_files_only"] = huggingface_hub.constants.HF_HUB_OFFLINE |
| 339 | trust_remote_code |= kwargs.get("trust_remote_code", False) |
| 340 | kwargs = without_trust_remote_code(kwargs) |
| 341 | config_dict, _ = PreTrainedConfig.get_config_dict( |
| 342 | model, |
| 343 | revision=revision, |
| 344 | code_revision=code_revision, |
| 345 | **kwargs, |
| 346 | ) |
| 347 | # Use custom model class if it's in our registry |
| 348 | model_type = config_dict.get("model_type") |
| 349 | if model_type is None: |
| 350 | model_type = ( |
| 351 | "speculators" |
| 352 | if config_dict.get("speculators_config") is not None |
| 353 | else model_type |
| 354 | ) |
| 355 | # Allow hf_overrides to override model_type before checking _CONFIG_REGISTRY |
| 356 | if (hf_overrides := kwargs.pop("hf_overrides", None)) is not None: |
| 357 | if isinstance(hf_overrides, dict) and "model_type" in hf_overrides: |
| 358 | model_type = hf_overrides["model_type"] |
| 359 | elif callable(hf_overrides): |
| 360 | # If hf_overrides doesn't modify model_type, it will be passed straight |
| 361 | # through and remain unchanged by this elif block |
| 362 | dummy_model_type = f"dummy_{model_type}" |
| 363 | dummy_kwargs = dict(architectures=[""], model_type=dummy_model_type) |
| 364 | dummy_config = PreTrainedConfig(**dummy_kwargs) |
| 365 | dummy_model_type = hf_overrides(dummy_config).model_type |
| 366 | model_type = dummy_model_type.removeprefix("dummy_") |
| 367 | |
| 368 | if model_type in _PATCH_HF_VALIDATE_ROPE: |
| 369 | _patch_hf_transformers_validate_rope() |
| 370 | |
| 371 | if model_type in _PATCH_HF_NESTED_ROPE_VALIDATION: |
| 372 | _patch_hf_transformers_nested_rope_validation() |
| 373 | |
| 374 | if extra_layer_types := _PATCH_HF_ALLOWED_LAYER_TYPES.get(model_type): |
| 375 | _patch_hf_transformers_allowed_layer_types(extra_layer_types) |
| 376 | |
| 377 | rope_parameters = config_dict.get("rope_parameters") or {} |
| 378 | if model_type == "gemma4_text" and "full_attention" in rope_parameters: |
| 379 | from transformers import Gemma4TextConfig |
| 380 | |
| 381 | # Published DSpark configs mix redundant scalar entries with per-layer |
| 382 | # RoPE dicts. Remove them before Transformers' constructor validates. |
| 383 | config_dict["rope_parameters"] = { |
| 384 | k: v |
| 385 | for k, v in rope_parameters.items() |
| 386 | if k not in ("rope_type", "rope_theta") |
| 387 | } |
| 388 | kwargs.setdefault("name_or_path", str(model)) |
| 389 | config = Gemma4TextConfig.from_dict(config_dict, **kwargs) |
| 390 | elif model_type in _SPECULATIVE_DECODING_CONFIGS: |
| 391 | config_class = _CONFIG_REGISTRY[model_type] |
| 392 | config = config_class.from_pretrained( |
| 393 | model, |
| 394 | revision=revision, |
| 395 | code_revision=code_revision, |
| 396 | trust_remote_code=trust_remote_code, |
| 397 | **kwargs, |
| 398 | ) |
| 399 | else: |
| 400 | if model_type in _CONFIG_REGISTRY: |
| 401 | # Register the config class to AutoConfig to ensure it's used |
| 402 | # in future calls to `from_pretrained` (e.g. from |
| 403 | # AutoTokenizer or AutoProcessor). |
| 404 | config_class = _CONFIG_REGISTRY[model_type] |
| 405 | _register_config_class(model_type, config_class) |
| 406 | # If the on-disk model_type differs from the overridden |
| 407 | # one, register under both so AutoConfig.from_pretrained |
| 408 | # returns the correct class regardless of what the |
| 409 | # checkpoint says |
| 410 | if ( |
| 411 | config_model_type := config_dict.get("model_type") |
| 412 | ) and config_model_type != model_type: |
| 413 | _register_config_class(config_model_type, config_class) |
| 414 | config_class.model_type = model_type |
| 415 | # Now that it is registered, it is not considered remote code anymore |
| 416 | trust_remote_code = False |
| 417 | try: |
| 418 | kwargs = _maybe_update_auto_config_kwargs(kwargs, model_type=model_type) |
| 419 | config = AutoConfig.from_pretrained( |
| 420 | model, |
| 421 | trust_remote_code=trust_remote_code, |
| 422 | revision=revision, |
| 423 | code_revision=code_revision, |
| 424 | **kwargs, |
| 425 | ) |
| 426 | except ValueError as e: |
| 427 | if ( |
| 428 | not trust_remote_code |
| 429 | and "requires you to execute the configuration file" in str(e) |
| 430 | ): |
| 431 | err_msg = ( |
| 432 | "Failed to load the model config. If the model " |
| 433 | "is a custom model not yet available in the " |
| 434 | "HuggingFace transformers library, consider setting " |
| 435 | "`trust_remote_code=True` in LLM or using the " |
| 436 | "`--trust-remote-code` flag in the CLI." |
| 437 | ) |
| 438 | raise RuntimeError(err_msg) from e |
| 439 | else: |
| 440 | raise e |
| 441 | config = _maybe_remap_hf_config_attrs(config) |
| 442 | return config_dict, config |
| 443 | |
| 444 | |
| 445 | class MistralConfigParser(ConfigParserBase): |
| 446 | def parse( |
| 447 | self, |
| 448 | model: str | Path, |
| 449 | trust_remote_code: bool, |
| 450 | revision: str | None = None, |
| 451 | code_revision: str | None = None, |
| 452 | **kwargs, |
| 453 | ) -> tuple[dict, PreTrainedConfig]: |
| 454 | # This function loads a params.json config which |
| 455 | # should be used when loading models in mistral format |
| 456 | config_dict = _download_mistral_config_file(model, revision) |
| 457 | if ( |
| 458 | max_position_embeddings := config_dict.get("max_position_embeddings") |
| 459 | ) is None: |
| 460 | max_position_embeddings = _maybe_retrieve_max_pos_from_hf( |
| 461 | model, revision, **kwargs |
| 462 | ) |
| 463 | config_dict["max_position_embeddings"] = max_position_embeddings |
| 464 | |
| 465 | from vllm.transformers_utils.configs.mistral import adapt_config_dict |
| 466 | |
| 467 | # Get missing fields from HF config if available |
| 468 | try: |
| 469 | hf_config_dict, _ = PreTrainedConfig.get_config_dict( |
| 470 | model, |
| 471 | revision=revision, |
| 472 | code_revision=code_revision, |
| 473 | **without_trust_remote_code(kwargs), |
| 474 | ) |
| 475 | except OSError: # Not found |
| 476 | hf_config_dict = {} |
| 477 | |
| 478 | if config_dict.get("dtype") is None: |
| 479 | with _mistral_patch_hf_hub_constants(): |
| 480 | model_str = model if isinstance(model, str) else model.as_posix() |
| 481 | param_mt = get_safetensors_params_metadata(model_str, revision=revision) |
| 482 | if param_mt: |
| 483 | param_dtypes: set[torch.dtype] = { |
| 484 | _SAFETENSORS_TO_TORCH_DTYPE[dtype] |
| 485 | for info in param_mt.values() |
| 486 | if (dtype := info.get("dtype", None)) |
| 487 | and dtype in _SAFETENSORS_TO_TORCH_DTYPE |
| 488 | } |
| 489 | |
| 490 | if param_dtypes: |
| 491 | config_dict["dtype"] = common_broadcastable_dtype(param_dtypes) |
| 492 | logger.info_once( |
| 493 | "Inferred from consolidated*.safetensors files " |
| 494 | f"{config_dict['dtype']} dtype." |
| 495 | ) |
| 496 | |
| 497 | config = adapt_config_dict(config_dict, defaults=hf_config_dict) |
| 498 | |
| 499 | return config_dict, config |
| 500 | |
| 501 | |
| 502 | _CONFIG_FORMAT_TO_CONFIG_PARSER: dict[str, type[ConfigParserBase]] = { |
| 503 | "hf": HFConfigParser, |
| 504 | "mistral": MistralConfigParser, |
| 505 | } |
| 506 | |
| 507 | ConfigFormat = Literal[ |
| 508 | "auto", |
| 509 | "hf", |
| 510 | "mistral", |
| 511 | ] |
| 512 | |
| 513 | |
| 514 | def get_config_parser(config_format: str) -> ConfigParserBase: |
| 515 | """Get the config parser for a given config format.""" |
| 516 | if config_format not in _CONFIG_FORMAT_TO_CONFIG_PARSER: |
| 517 | raise ValueError(f"Unknown config format `{config_format}`.") |
| 518 | return _CONFIG_FORMAT_TO_CONFIG_PARSER[config_format]() |
| 519 | |
| 520 | |
| 521 | def register_config_parser(config_format: str): |
| 522 | """Register a customized vllm config parser. |
| 523 | When a config format is not supported by vllm, you can register a customized |
| 524 | config parser to support it. |
| 525 | |
| 526 | Args: |
| 527 | config_format (str): The config parser format name. |
| 528 | |
| 529 | Examples: |
| 530 | >>> from vllm.transformers_utils.config import (get_config_parser, |
| 531 | register_config_parser) |
| 532 | >>> from vllm.transformers_utils.config_parser_base import ConfigParserBase |
| 533 | >>> |
| 534 | >>> @register_config_parser("custom_config_parser") |
| 535 | ... class CustomConfigParser(ConfigParserBase): |
| 536 | ... def parse( |
| 537 | ... self, |
| 538 | ... model: Union[str, Path], |
| 539 | ... trust_remote_code: bool, |
| 540 | ... revision: str | None = None, |
| 541 | ... code_revision: str | None = None, |
| 542 | ... **kwargs, |
| 543 | ... ) -> tuple[dict, PreTrainedConfig]: |
| 544 | ... raise NotImplementedError |
| 545 | >>> |
| 546 | >>> type(get_config_parser("custom_config_parser")) |
| 547 | <class 'CustomConfigParser'> |
| 548 | |
| 549 | """ # noqa: E501 |
| 550 | |
| 551 | def _wrapper(config_parser_cls): |
| 552 | if config_format in _CONFIG_FORMAT_TO_CONFIG_PARSER: |
| 553 | logger.warning( |
| 554 | "Config format `%s` is already registered, and will be " |
| 555 | "overwritten by the new parser class `%s`.", |
| 556 | config_format, |
| 557 | config_parser_cls, |
| 558 | ) |
| 559 | if not issubclass(config_parser_cls, ConfigParserBase): |
| 560 | raise ValueError( |
| 561 | "The config parser must be a subclass of `ConfigParserBase`." |
| 562 | ) |
| 563 | _CONFIG_FORMAT_TO_CONFIG_PARSER[config_format] = config_parser_cls |
| 564 | logger.info( |
| 565 | "Registered config parser `%s` with config format `%s`", |
| 566 | config_parser_cls, |
| 567 | config_format, |
| 568 | ) |
| 569 | return config_parser_cls |
| 570 | |
| 571 | return _wrapper |
| 572 | |
| 573 | |
| 574 | def set_default_rope_theta(config: PreTrainedConfig, default_theta: float) -> None: |
| 575 | """Some models may have no rope_theta in their config but still use RoPE. |
| 576 | This function sets a default rope_theta if it's missing.""" |
| 577 | if getattr(config, "rope_parameters", None) is None: |
| 578 | config.rope_parameters = {"rope_type": "default"} |
| 579 | if "rope_theta" not in config.rope_parameters: |
| 580 | config.rope_parameters["rope_theta"] = default_theta |
| 581 | |
| 582 | |
| 583 | def patch_legacy_rope_type(rope_parameters: dict[str, Any] | None) -> None: |
| 584 | """Patch legacy RoPE type fields for backwards compatibility with |
| 585 | older custom models which would otherwise fail to load.""" |
| 586 | # No RoPE parameters to patch |
| 587 | if rope_parameters is None: |
| 588 | return |
| 589 | |
| 590 | def _patch_legacy_rope_type(rope_parameters: dict[str, Any]) -> None: |
| 591 | # Case 1: Both legacy and modern fields present - check for conflicts |
| 592 | if "rope_type" in rope_parameters and "type" in rope_parameters: |
| 593 | rope_type = rope_parameters["rope_type"] |
| 594 | rope_type_legacy = rope_parameters["type"] |
| 595 | if (rope_type_legacy == "su" and rope_type == "longrope") or ( |
| 596 | rope_type_legacy == "mrope" and rope_type == "default" |
| 597 | ): |
| 598 | pass # No action needed |
| 599 | elif rope_type != rope_type_legacy: |
| 600 | raise ValueError( |
| 601 | f"Found conflicts between 'rope_type={rope_type}' (modern " |
| 602 | f"field) and 'type={rope_type_legacy}' (legacy field). " |
| 603 | "You should only specify one of them." |
| 604 | ) |
| 605 | # Case 2: Only legacy field present - patch to modern format with warning |
| 606 | if "rope_type" not in rope_parameters and "type" in rope_parameters: |
| 607 | rope_parameters["rope_type"] = rope_parameters["type"] |
| 608 | logger.info("Replacing legacy 'type' key with 'rope_type'") |
| 609 | # Case 3: No rope_type field present - nothing to patch |
| 610 | if "rope_type" not in rope_parameters: |
| 611 | return |
| 612 | # Patch legacy rope_type values with warning |
| 613 | if rope_parameters["rope_type"] == "su": |
| 614 | rope_parameters["rope_type"] = "longrope" |
| 615 | logger.warning("Replacing legacy rope_type 'su' with 'longrope'") |
| 616 | elif rope_parameters["rope_type"] == "mrope": |
| 617 | if "mrope_section" not in rope_parameters: |
| 618 | raise ValueError( |
| 619 | "Legacy rope_type 'mrope' requires " |
| 620 | "'mrope_section' in rope_parameters" |
| 621 | ) |
| 622 | rope_parameters["rope_type"] = "default" |
| 623 | logger.warning("Replacing legacy rope_type 'mrope' with 'default'") |
| 624 | elif rope_parameters["rope_type"] == "telechat3-yarn": |
| 625 | # TeleChat3 is YaRN with 0.07 in place of YaRN's 0.1 attention |
| 626 | # scaling coefficient. Precompute it so the config is plain YaRN. |
| 627 | factor = rope_parameters["factor"] |
| 628 | rope_parameters["rope_type"] = "yarn" |
| 629 | rope_parameters["attention_factor"] = 0.07 * math.log(factor) + 1.0 |
| 630 | rope_parameters.pop("type", None) |
| 631 | logger.warning("Replacing rope_type 'telechat3-yarn' with 'yarn'") |
| 632 | |
| 633 | # Handle nested rope_parameters in interleaved sliding attention models |
| 634 | if is_rope_parameters_nested(rope_parameters): |
| 635 | for rope_parameters_layer_type in rope_parameters.values(): |
| 636 | if rope_parameters_layer_type is not None: |
| 637 | _patch_legacy_rope_type(rope_parameters_layer_type) |
| 638 | else: |
| 639 | _patch_legacy_rope_type(rope_parameters) |
| 640 | |
| 641 | |
| 642 | def patch_rope_parameters(config: PreTrainedConfig) -> None: |
| 643 | """Provide backwards compatibility for RoPE.""" |
| 644 | from vllm.config.utils import getattr_iter |
| 645 | |
| 646 | # Older custom models may use non-standard field names which need patching. |
| 647 | names = ["rope_theta", "rotary_emb_base"] |
| 648 | rope_theta = getattr_iter(config, names, None, warn=True) |
| 649 | names = ["partial_rotary_factor", "rotary_pct", "rotary_emb_fraction"] |
| 650 | partial_rotary_factor = getattr_iter(config, names, None, warn=True) |
| 651 | |
| 652 | if rope_theta is not None or getattr(config, "rope_parameters", None): |
| 653 | # Patch these fields in case they used non-standard names |
| 654 | if rope_theta is not None: |
| 655 | config.rope_theta = rope_theta |
| 656 | if partial_rotary_factor is not None: |
| 657 | config.partial_rotary_factor = partial_rotary_factor |
| 658 | # Standardize and validate RoPE parameters |
| 659 | patch_legacy_rope_type(getattr(config, "rope_parameters", None)) |
| 660 | config.standardize_rope_params() |
| 661 | config.validate_rope() |
| 662 | |
| 663 | |
| 664 | def _iter_rope_parameters(config: PreTrainedConfig) -> Iterator[dict[str, Any]]: |
| 665 | """Yield a config's rope parameters, one dict per layer type if nested.""" |
| 666 | rope_parameters = getattr(config, "rope_parameters", None) |
| 667 | if not isinstance(rope_parameters, dict): |
| 668 | return |
| 669 | |
| 670 | if is_rope_parameters_nested(rope_parameters): |
| 671 | yield from (p for p in rope_parameters.values() if isinstance(p, dict)) |
| 672 | else: |
| 673 | yield rope_parameters |
| 674 | |
| 675 | |
| 676 | def _mrope_section(config: PreTrainedConfig) -> Sequence[int] | None: |
| 677 | """Return the M-RoPE section this config declares, if any. |
| 678 | |
| 679 | `xdrope_section` is the legacy name HunYuan-VL checkpoints use for the same |
| 680 | field; upstream Transformers normalises it to `mrope_section`. |
| 681 | """ |
| 682 | from vllm.config.utils import getattr_iter |
| 683 | |
| 684 | names = ("mrope_section", "xdrope_section") |
| 685 | |
| 686 | for params in _iter_rope_parameters(config): |
| 687 | for i, name in enumerate(names): |
| 688 | section = params.get(name) |
| 689 | if isinstance(section, (list, tuple)): |
| 690 | if i > 0: |
| 691 | logger.warning_once( |
| 692 | "rope_parameters contains a deprecated key '%s'. " |
| 693 | "Please use the preferred key '%s' instead.", |
| 694 | name, |
| 695 | names[0], |
| 696 | ) |
| 697 | return section |
| 698 | |
| 699 | section = getattr_iter(config, names, None, warn=True) |
| 700 | return section if isinstance(section, (list, tuple)) else None |
| 701 | |
| 702 | |
| 703 | def _uses_mrope(config: PreTrainedConfig) -> bool: |
| 704 | return _mrope_section(config) is not None |
| 705 | |
| 706 | |
| 707 | def uses_mrope(config: PreTrainedConfig) -> bool: |
| 708 | """Detect if the model with this config uses M-ROPE.""" |
| 709 | return ( |
| 710 | _uses_mrope(config) |
| 711 | or _uses_mrope(config.get_text_config()) |
| 712 | or thinker_uses_mrope(config) |
| 713 | ) |
| 714 | |
| 715 | |
| 716 | def thinker_uses_mrope(config: PreTrainedConfig) -> bool: |
| 717 | """Detect if the model contains a thinker config and it uses M-ROPE.""" |
| 718 | thinker_config = getattr(config, "thinker_config", None) |
| 719 | if thinker_config is None: |
| 720 | return False |
| 721 | |
| 722 | thinker_text_config = getattr(thinker_config, "text_config", None) |
| 723 | if thinker_text_config is None: |
| 724 | return False |
| 725 | |
| 726 | return uses_mrope(thinker_text_config) |
| 727 | |
| 728 | |
| 729 | def mrope_num_dims(config: PreTrainedConfig) -> int: |
| 730 | """Number of M-RoPE position channels the model consumes. |
| 731 | |
| 732 | Each section entry sizes one channel, so the section length is the channel |
| 733 | count. Interleaved M-RoPE also accepts a 2 section variant whose positions |
| 734 | are still 3D, so never return fewer than 3. |
| 735 | """ |
| 736 | if not uses_mrope(config): |
| 737 | return 0 |
| 738 | |
| 739 | for candidate in (config.get_text_config(), config): |
| 740 | mrope_section = _mrope_section(candidate) |
| 741 | if mrope_section is not None: |
| 742 | return max(len(mrope_section), 3) |
| 743 | |
| 744 | # Custom configs may declare M-RoPE without exposing the sections. |
| 745 | return 3 |
| 746 | |
| 747 | |
| 748 | def is_encoder_decoder(config: PreTrainedConfig) -> bool: |
| 749 | """Detect if the model with this config is used as an encoder/decoder.""" |
| 750 | |
| 751 | def _is_encoder_decoder(config: PreTrainedConfig) -> bool: |
| 752 | return getattr(config, "is_encoder_decoder", False) |
| 753 | |
| 754 | return _is_encoder_decoder(config) or _is_encoder_decoder(config.get_text_config()) |
| 755 | |
| 756 | |
| 757 | def _maybe_update_auto_config_kwargs(kwargs: dict[str, Any], model_type: str): |
| 758 | """Update kwargs for AutoConfig initialization based on model_type.""" |
| 759 | if model_type in _AUTO_CONFIG_KWARGS_OVERRIDES: |
| 760 | kwargs.update(_AUTO_CONFIG_KWARGS_OVERRIDES[model_type]) |
| 761 | return kwargs |
| 762 | |
| 763 | |
| 764 | def _maybe_remap_hf_config_attrs(config: PreTrainedConfig) -> PreTrainedConfig: |
| 765 | """Remap config attributes to match the expected names.""" |
| 766 | for old_attr, new_attr in _CONFIG_ATTRS_MAPPING.items(): |
| 767 | if hasattr(config, old_attr): |
| 768 | if not hasattr(config, new_attr): |
| 769 | config.update({new_attr: getattr(config, old_attr)}) |
| 770 | logger.debug("Remapped config attribute '%s' to '%s'", old_attr, new_attr) |
| 771 | return config |
| 772 | |
| 773 | |
| 774 | def maybe_override_with_speculators( |
| 775 | model: str, |
| 776 | tokenizer: str | None, |
| 777 | trust_remote_code: bool, |
| 778 | revision: str | None = None, |
| 779 | vllm_speculative_config: dict[str, Any] | None = None, |
| 780 | hf_token: bool | str | None = None, |
| 781 | **kwargs, |
| 782 | ) -> tuple[str, str | None, dict[str, Any] | None]: |
| 783 | """Resolve model configuration when speculators are detected. |
| 784 | |
| 785 | Checks if the provided model is a speculators model and if so, extracts |
| 786 | the target model configuration and builds the speculative config. |
| 787 | |
| 788 | Args: |
| 789 | model: Model name or path |
| 790 | tokenizer: Tokenizer name or path |
| 791 | trust_remote_code: Whether to trust remote code |
| 792 | revision: Model revision |
| 793 | vllm_speculative_config: Existing vLLM speculative config |
| 794 | hf_token: HuggingFace token for authenticated model access |
| 795 | |
| 796 | Returns: |
| 797 | Tuple of (resolved_model, resolved_tokenizer, speculative_config) |
| 798 | |
| 799 | """ |
| 800 | kwargs["local_files_only"] = huggingface_hub.constants.HF_HUB_OFFLINE |
| 801 | config_dict, _ = PreTrainedConfig.get_config_dict( |
| 802 | model, |
| 803 | revision=revision, |
| 804 | token=hf_token, |
| 805 | **without_trust_remote_code(kwargs), |
| 806 | ) |
| 807 | speculators_config = config_dict.get("speculators_config") |
| 808 | |
| 809 | if speculators_config is None: |
| 810 | # No speculators config found, return original values |
| 811 | return model, tokenizer, vllm_speculative_config |
| 812 | |
| 813 | # Speculators format detected - process overrides |
| 814 | from vllm.transformers_utils.configs.speculators.base import SpeculatorsConfig |
| 815 | |
| 816 | speculative_config = SpeculatorsConfig.extract_vllm_speculative_config( |
| 817 | config_dict=config_dict |
| 818 | ) |
| 819 | speculators_method = speculative_config["method"] |
| 820 | |
| 821 | # Apply user --speculative-config overrides (e.g. attention_backend). |
| 822 | if isinstance(vllm_speculative_config, dict): |
| 823 | speculative_config.update(vllm_speculative_config) |
| 824 | |
| 825 | # Lock fields dictated by the speculators format |
| 826 | speculative_config["method"] = speculators_method |
| 827 | speculative_config["model"] = model |
| 828 | |
| 829 | # Override model and tokenizer with the verifier model from config |
| 830 | verifier_model = speculators_config["verifier"]["name_or_path"] |
| 831 | model = tokenizer = verifier_model |
| 832 | |
| 833 | return model, tokenizer, speculative_config |
| 834 | |
| 835 | |
| 836 | def get_config( |
| 837 | model: str | Path, |
| 838 | trust_remote_code: bool, |
| 839 | revision: str | None = None, |
| 840 | code_revision: str | None = None, |
| 841 | config_format: str | ConfigFormat = "auto", |
| 842 | hf_overrides_kw: dict[str, Any] | None = None, |
| 843 | hf_overrides_fn: Callable[[PreTrainedConfig], PreTrainedConfig] | None = None, |
| 844 | **kwargs, |
| 845 | ) -> PreTrainedConfig: |
| 846 | if config_format == "auto": |
| 847 | try: |
| 848 | # First check for Mistral to avoid defaulting to |
| 849 | # Transformers implementation. |
| 850 | if is_mistral_model_repo( |
| 851 | model_name_or_path=str(model), revision=revision |
| 852 | ) and file_or_path_exists( |
| 853 | model=model, config_name=MISTRAL_CONFIG_NAME, revision=revision |
| 854 | ): |
| 855 | config_format = "mistral" |
| 856 | elif file_or_path_exists(model, HF_CONFIG_NAME, revision=revision): |
| 857 | config_format = "hf" |
| 858 | else: |
| 859 | raise ValueError( |
| 860 | "Could not detect config format for no config file found. " |
| 861 | "With config_format 'auto', ensure your model has either " |
| 862 | "config.json (HF format) or params.json (Mistral format). " |
| 863 | "Otherwise please specify your_custom_config_format " |
| 864 | "in engine args for customized config parser." |
| 865 | ) |
| 866 | |
| 867 | except Exception as e: |
| 868 | error_message = ( |
| 869 | "Invalid repository ID or local directory specified:" |
| 870 | " '{model}'.\nPlease verify the following requirements:\n" |
| 871 | "1. Provide a valid Hugging Face repository ID.\n" |
| 872 | "2. Specify a local directory that contains a recognized " |
| 873 | "configuration file.\n" |
| 874 | " - For Hugging Face models: ensure the presence of a " |
| 875 | "'config.json'.\n" |
| 876 | " - For Mistral models: ensure the presence of a " |
| 877 | "'params.json'.\n" |
| 878 | ).format(model=model) |
| 879 | |
| 880 | raise ValueError(error_message) from e |
| 881 | |
| 882 | config_parser = get_config_parser(config_format) |
| 883 | # Retry to tolerate a concurrent HF cache refresh briefly hiding config.json. |
| 884 | config_dict, config = with_retry( |
| 885 | lambda: config_parser.parse( |
| 886 | model, |
| 887 | trust_remote_code=trust_remote_code, |
| 888 | revision=revision, |
| 889 | code_revision=code_revision, |
| 890 | hf_overrides=hf_overrides_kw or hf_overrides_fn, |
| 891 | **kwargs, |
| 892 | ), |
| 893 | f"Error parsing config for {model}", |
| 894 | ) |
| 895 | |
| 896 | # Architecture mapping for models without explicit architectures field |
| 897 | if not config.architectures: |
| 898 | if config.model_type not in MODEL_MAPPING_NAMES: |
| 899 | logger.warning( |
| 900 | "Model config does not have a top-level 'architectures' field: " |
| 901 | "expecting `hf_overrides={'architectures': ['...']}` to be passed " |
| 902 | "in engine args." |
| 903 | ) |
| 904 | else: |
| 905 | model_type = MODEL_MAPPING_NAMES[config.model_type] |
| 906 | config.update({"architectures": [model_type]}) |
| 907 | |
| 908 | # ModelOpt 0.31.0 and after saves the quantization config in the model |
| 909 | # config file. |
| 910 | quantization_config = config_dict.get("quantization_config", None) |
| 911 | |
| 912 | # ModelOpt 0.29.0 and before saves the quantization config in a separate |
| 913 | # "hf_quant_config.json" in the same directory as the model config file. |
| 914 | if quantization_config is None and file_or_path_exists( |
| 915 | model, "hf_quant_config.json", revision |
| 916 | ): |
| 917 | quantization_config = get_hf_file_to_dict( |
| 918 | "hf_quant_config.json", model, revision |
| 919 | ) |
| 920 | |
| 921 | if quantization_config is not None: |
| 922 | config.quantization_config = quantization_config |
| 923 | # auto-enable DeepGEMM UE8M0 if model config requests it |
| 924 | scale_fmt = quantization_config.get("scale_fmt", None) |
| 925 | if scale_fmt in ("ue8m0",): |
| 926 | if not envs.is_set("VLLM_USE_DEEP_GEMM_E8M0"): |
| 927 | os.environ["VLLM_USE_DEEP_GEMM_E8M0"] = "1" |
| 928 | logger.info_once( |
| 929 | ( |
| 930 | "Detected quantization_config.scale_fmt=%s; " |
| 931 | "enabling UE8M0 for DeepGEMM." |
| 932 | ), |
| 933 | scale_fmt, |
| 934 | ) |
| 935 | elif not envs.VLLM_USE_DEEP_GEMM_E8M0: |
| 936 | logger.warning_once( |
| 937 | ( |
| 938 | "Model config requests UE8M0 " |
| 939 | "(quantization_config.scale_fmt=%s), but " |
| 940 | "VLLM_USE_DEEP_GEMM_E8M0=0 is set; " |
| 941 | "UE8M0 for DeepGEMM disabled." |
| 942 | ), |
| 943 | scale_fmt, |
| 944 | ) |
| 945 | |
| 946 | if hf_overrides_kw: |
| 947 | logger.debug("Overriding HF config with %s", hf_overrides_kw) |
| 948 | config.update(hf_overrides_kw) |
| 949 | if hf_overrides_fn: |
| 950 | logger.debug("Overriding HF config with %s", hf_overrides_fn) |
| 951 | config = hf_overrides_fn(config) |
| 952 | |
| 953 | # Exhaustively patch RoPE parameters everywhere they might be |
| 954 | patch_rope_parameters(config) |
| 955 | patch_rope_parameters(config.get_text_config()) |
| 956 | SubConfigs: TypeAlias = dict[str, PreTrainedConfig] |
| 957 | sub_configs: SubConfigs | None = getattr(config, "sub_configs", None) |
| 958 | if sub_configs: |
| 959 | for sub_config in sub_configs: |
| 960 | patch_rope_parameters(getattr(config, sub_config)) |
| 961 | |
| 962 | if trust_remote_code: |
| 963 | maybe_register_config_serialize_by_value() |
| 964 | |
| 965 | return config |
| 966 | |
| 967 | |
| 968 | @cache |
| 969 | def get_pooling_config( |
| 970 | model: str, |
| 971 | revision: str | None = "main", |
| 972 | ) -> dict[str, Any] | None: |
| 973 | """This function gets the pooling and normalize |
| 974 | config from the model - only applies to |
| 975 | sentence-transformers models. |
| 976 | |
| 977 | Args: |
| 978 | model: The name of the Hugging Face model. |
| 979 | revision: The specific version of the model to use. |
| 980 | Defaults to 'main'. |
| 981 | |
| 982 | Returns: |
| 983 | A dictionary containing the pooling type and whether |
| 984 | normalization is used, or None if no pooling configuration is found. |
| 985 | |
| 986 | """ |
| 987 | modules_file_name = "modules.json" |
| 988 | |
| 989 | modules_dict = None |
| 990 | if file_or_path_exists( |
| 991 | model=model, config_name=modules_file_name, revision=revision |
| 992 | ): |
| 993 | modules_dict = get_hf_file_to_dict(modules_file_name, model, revision) |
| 994 | |
| 995 | if modules_dict is None: |
| 996 | return None |
| 997 | |
| 998 | logger.info("Found sentence-transformers modules configuration.") |
| 999 | |
| 1000 | pooling = next( |
| 1001 | (item for item in modules_dict if item["type"] in _ST_POOLING_MODULE_TYPES), |
| 1002 | None, |
| 1003 | ) |
| 1004 | normalize = bool( |
| 1005 | next( |
| 1006 | ( |
| 1007 | item |
| 1008 | for item in modules_dict |
| 1009 | if item["type"] in _ST_NORMALIZE_MODULE_TYPES |
| 1010 | ), |
| 1011 | False, |
| 1012 | ) |
| 1013 | ) |
| 1014 | |
| 1015 | if pooling: |
| 1016 | from vllm.config.pooler import SEQ_POOLING_TYPES, TOK_POOLING_TYPES |
| 1017 | |
| 1018 | pooling_file_name = "{}/config.json".format(pooling["path"]) |
| 1019 | pooling_dict = get_hf_file_to_dict(pooling_file_name, model, revision) or {} |
| 1020 | |
| 1021 | logger.info("Found pooling configuration.") |
| 1022 | |
| 1023 | config: dict[str, Any] = {"use_activation": normalize} |
| 1024 | for key, val in pooling_dict.items(): |
| 1025 | if key == "pooling_mode" and isinstance(val, str): |
| 1026 | pooling_name = val |
| 1027 | elif val is True: |
| 1028 | pooling_name = key |
| 1029 | else: |
| 1030 | continue |
| 1031 | |
| 1032 | pooling_type = parse_pooling_type(pooling_name) |
| 1033 | if pooling_type in SEQ_POOLING_TYPES: |
| 1034 | config["seq_pooling_type"] = pooling_type |
| 1035 | elif pooling_type in TOK_POOLING_TYPES: |
| 1036 | config["tok_pooling_type"] = pooling_type |
| 1037 | else: |
| 1038 | logger.debug("Skipping unrelated field: %r=%r", key, val) |
| 1039 | |
| 1040 | if not {"seq_pooling_type", "tok_pooling_type"} & config.keys(): |
| 1041 | logger.warning( |
| 1042 | "Unable to determine Sentence Transformers pooling type from %s; " |
| 1043 | "unless configured explicitly, vLLM will fall back to the model " |
| 1044 | "architecture default.", |
| 1045 | pooling_file_name, |
| 1046 | ) |
| 1047 | |
| 1048 | return config |
| 1049 | |
| 1050 | return None |
| 1051 | |
| 1052 | |
| 1053 | def parse_pooling_type(pooling_name: str): |
| 1054 | if "pooling_mode_" in pooling_name: |
| 1055 | pooling_name = pooling_name.replace("pooling_mode_", "") |
| 1056 | |
| 1057 | if "_" in pooling_name: |
| 1058 | pooling_name = pooling_name.split("_", 1)[0] |
| 1059 | |
| 1060 | if "lasttoken" in pooling_name: |
| 1061 | pooling_name = "last" |
| 1062 | |
| 1063 | return pooling_name.upper() |
| 1064 | |
| 1065 | |
| 1066 | @cache |
| 1067 | def get_sentence_transformer_tokenizer_config( |
| 1068 | model: str | Path, revision: str | None = "main" |
| 1069 | ) -> dict[str, Any] | None: |
| 1070 | """Returns the tokenization configuration dictionary for a |
| 1071 | given Sentence Transformer BERT model. |
| 1072 | |
| 1073 | Args: |
| 1074 | model (str|Path): The name of the Sentence Transformer BERT model. |
| 1075 | revision (str, optional): The revision of the model to use. Defaults to 'main'. |
| 1076 | |
| 1077 | Returns: |
| 1078 | - dict: A dictionary containing the configuration parameters |
| 1079 | for the Sentence Transformer BERT model. |
| 1080 | |
| 1081 | """ |
| 1082 | sentence_transformer_config_files = [ |
| 1083 | "sentence_bert_config.json", |
| 1084 | "sentence_roberta_config.json", |
| 1085 | "sentence_distilbert_config.json", |
| 1086 | "sentence_camembert_config.json", |
| 1087 | "sentence_albert_config.json", |
| 1088 | "sentence_xlm-roberta_config.json", |
| 1089 | "sentence_xlnet_config.json", |
| 1090 | ] |
| 1091 | encoder_dict = None |
| 1092 | |
| 1093 | for config_file in sentence_transformer_config_files: |
| 1094 | if isinstance( |
| 1095 | try_get_local_file(model=model, file_name=config_file, revision=revision), |
| 1096 | Path, |
| 1097 | ): |
| 1098 | encoder_dict = get_hf_file_to_dict(config_file, model, revision) |
| 1099 | if encoder_dict: |
| 1100 | break |
| 1101 | |
| 1102 | if not encoder_dict and not Path(model).is_absolute(): |
| 1103 | try: |
| 1104 | # If model is on HuggingfaceHub, get the repo files |
| 1105 | repo_files = list_repo_files(model, revision=revision) |
| 1106 | except Exception: |
| 1107 | repo_files = [] |
| 1108 | |
| 1109 | for config_name in sentence_transformer_config_files: |
| 1110 | if config_name in repo_files: |
| 1111 | encoder_dict = get_hf_file_to_dict(config_name, model, revision) |
| 1112 | if encoder_dict: |
| 1113 | break |
| 1114 | |
| 1115 | if not encoder_dict: |
| 1116 | return None |
| 1117 | |
| 1118 | logger.info("Found sentence-transformers tokenize configuration.") |
| 1119 | |
| 1120 | if all(k in encoder_dict for k in ("max_seq_length", "do_lower_case")): |
| 1121 | return encoder_dict |
| 1122 | return None |
| 1123 | |
| 1124 | |
| 1125 | def maybe_register_config_serialize_by_value() -> None: |
| 1126 | """Try to register HF model configuration class to serialize by value |
| 1127 | |
| 1128 | If trust_remote_code is set, and the model's config file specifies an |
| 1129 | `AutoConfig` class, then the config class is typically an instance of |
| 1130 | a custom class imported from the HF modules cache. |
| 1131 | |
| 1132 | Examples: |
| 1133 | |
| 1134 | >>> from transformers import AutoConfig |
| 1135 | >>> klass = AutoConfig.from_pretrained( |
| 1136 | ... "meta-llama/Meta-Llama-3-8B", trust_remote_code=True |
| 1137 | ... ) |
| 1138 | >>> klass.__class__ # transformers.models.llama.configuration_llama.LlamaConfig |
| 1139 | >>> import transformers_modules # error, not initialized |
| 1140 | >>> klass = AutoConfig.from_pretrained( |
| 1141 | ... "deepseek-ai/DeepSeek-V2.5", trust_remote_code=True |
| 1142 | ... ) |
| 1143 | >>> import transformers_modules # success, initialized |
| 1144 | >>> klass.__class__ # transformers_modules.deepseek-ai.DeepSeek-V2.5.98b11844770b2c3ffc18b175c758a803640f4e77.configuration_deepseek.DeepseekV2Config |
| 1145 | |
| 1146 | In the DeepSeek example, the config class is an instance of a custom |
| 1147 | class that is not serializable by default. This class will not be |
| 1148 | importable in spawned workers, and won't exist at all on |
| 1149 | other nodes, which breaks serialization of the config. |
| 1150 | |
| 1151 | In this function we tell the cloudpickle serialization library to pass |
| 1152 | instances of these generated classes by value instead of by reference, |
| 1153 | i.e. the class definition is serialized along with its data so that the |
| 1154 | class module does not need to be importable on the receiving end. |
| 1155 | |
| 1156 | See: https://github.com/cloudpipe/cloudpickle?tab=readme-ov-file#overriding-pickles-serialization-mechanism-for-importable-constructs |
| 1157 | """ # noqa |
| 1158 | try: |
| 1159 | import transformers_modules |
| 1160 | |
| 1161 | transformers_modules_available = True |
| 1162 | except ImportError: |
| 1163 | transformers_modules_available = False |
| 1164 | |
| 1165 | try: |
| 1166 | import multiprocessing |
| 1167 | import pickle |
| 1168 | |
| 1169 | import cloudpickle |
| 1170 | |
| 1171 | from vllm.config import VllmConfig |
| 1172 | |
| 1173 | # Register multiprocessing reducers to handle cross-process |
| 1174 | # serialization of VllmConfig objects that may contain custom configs |
| 1175 | # from transformers_modules |
| 1176 | def _reduce_config(config: VllmConfig): |
| 1177 | return (pickle.loads, (cloudpickle.dumps(config),)) |
| 1178 | |
| 1179 | multiprocessing.reducer.register(VllmConfig, _reduce_config) |
| 1180 | |
| 1181 | # Register transformers_modules with cloudpickle if available |
| 1182 | if transformers_modules_available: |
| 1183 | cloudpickle.register_pickle_by_value(transformers_modules) |
| 1184 | |
| 1185 | # ray vendors its own version of cloudpickle |
| 1186 | from vllm.v1.executor.ray_utils import ray |
| 1187 | |
| 1188 | if ray: |
| 1189 | ray.cloudpickle.register_pickle_by_value(transformers_modules) |
| 1190 | |
| 1191 | except Exception as e: |
| 1192 | logger.warning( |
| 1193 | "Unable to register remote classes used by" |
| 1194 | " trust_remote_code with by-value serialization. This may" |
| 1195 | " lead to a later error. If remote code is not needed" |
| 1196 | " remove `--trust-remote-code`", |
| 1197 | exc_info=e, |
| 1198 | ) |
| 1199 | |
| 1200 | |
| 1201 | def get_hf_image_processor_config( |
| 1202 | model: str | Path, |
| 1203 | hf_token: bool | str | None = None, |
| 1204 | revision: str | None = None, |
| 1205 | **kwargs, |
| 1206 | ) -> dict[str, Any]: |
| 1207 | # ModelScope does not provide an interface for image_processor |
| 1208 | if envs.VLLM_USE_MODELSCOPE: |
| 1209 | return dict() |
| 1210 | return get_image_processor_config( |
| 1211 | model, token=hf_token, revision=revision, **kwargs |
| 1212 | ) |
| 1213 | |
| 1214 | |
| 1215 | def get_hf_text_config(config: PreTrainedConfig): |
| 1216 | """Get the "sub" config relevant to llm for multi modal models. |
| 1217 | No op for pure text models. |
| 1218 | """ |
| 1219 | text_config = config.get_text_config() |
| 1220 | |
| 1221 | if text_config is not config and not hasattr(text_config, "num_attention_heads"): |
| 1222 | raise ValueError( |
| 1223 | "The text_config extracted from the model config does not have " |
| 1224 | "`num_attention_heads` attribute. This indicates a mismatch " |
| 1225 | "between the model config and vLLM's expectations. Please " |
| 1226 | "ensure that the model config is compatible with vLLM." |
| 1227 | ) |
| 1228 | |
| 1229 | return text_config |
| 1230 | |
| 1231 | |
| 1232 | def try_get_generation_config( |
| 1233 | model: str, |
| 1234 | trust_remote_code: bool, |
| 1235 | revision: str | None = None, |
| 1236 | code_revision: str | None = None, |
| 1237 | config_format: str | ConfigFormat = "auto", |
| 1238 | hf_token: bool | str | None = None, |
| 1239 | ) -> GenerationConfig | None: |
| 1240 | try: |
| 1241 | return GenerationConfig.from_pretrained( |
| 1242 | model, |
| 1243 | revision=revision, |
| 1244 | token=hf_token, |
| 1245 | ) |
| 1246 | except OSError: # Not found |
| 1247 | try: |
| 1248 | config = get_config( |
| 1249 | model, |
| 1250 | trust_remote_code=trust_remote_code, |
| 1251 | revision=revision, |
| 1252 | code_revision=code_revision, |
| 1253 | config_format=config_format, |
| 1254 | token=hf_token, |
| 1255 | ) |
| 1256 | return GenerationConfig.from_model_config(config) |
| 1257 | except OSError: # Not found |
| 1258 | return None |
| 1259 | |
| 1260 | |
| 1261 | def try_get_safetensors_metadata( |
| 1262 | model: str, |
| 1263 | *, |
| 1264 | revision: str | None = None, |
| 1265 | ): |
| 1266 | get_safetensors_metadata_partial = partial( |
| 1267 | hf_api().get_safetensors_metadata, model, revision=revision |
| 1268 | ) |
| 1269 | |
| 1270 | try: |
| 1271 | return with_retry( |
| 1272 | get_safetensors_metadata_partial, |
| 1273 | "Error retrieving safetensors", |
| 1274 | fatal_errors=(huggingface_hub.errors.NotASafetensorsRepoError,), |
| 1275 | ) |
| 1276 | except Exception: |
| 1277 | return None |
| 1278 | |
| 1279 | |
| 1280 | def try_get_tokenizer_config( |
| 1281 | pretrained_model_name_or_path: str | os.PathLike, |
| 1282 | trust_remote_code: bool, |
| 1283 | revision: str | None = None, |
| 1284 | ) -> dict[str, Any] | None: |
| 1285 | try: |
| 1286 | return get_tokenizer_config( |
| 1287 | pretrained_model_name_or_path, |
| 1288 | trust_remote_code=trust_remote_code, |
| 1289 | revision=revision, |
| 1290 | ) |
| 1291 | except Exception: |
| 1292 | return None |
| 1293 | |
| 1294 | |
| 1295 | @cache |
| 1296 | def try_get_dense_modules( |
| 1297 | model: str | Path, |
| 1298 | revision: str | None = None, |
| 1299 | ) -> list[dict[str, Any]] | None: |
| 1300 | try: |
| 1301 | modules = get_hf_file_to_dict("modules.json", model, revision) |
| 1302 | if not modules: |
| 1303 | return None |
| 1304 | |
| 1305 | if isinstance(modules, dict): |
| 1306 | modules = modules.get("modules", []) |
| 1307 | |
| 1308 | dense_modules = [m for m in modules if m.get("type") in _DENSE_MODULE_TYPES] |
| 1309 | if not dense_modules: |
| 1310 | return None |
| 1311 | |
| 1312 | layer_configs = [] |
| 1313 | for module in dense_modules: |
| 1314 | folder = module.get("path", "") |
| 1315 | |
| 1316 | config_path = f"{folder}/config.json" if folder else "config.json" |
| 1317 | layer_config = get_hf_file_to_dict(config_path, model, revision) |
| 1318 | if not layer_config: |
| 1319 | continue |
| 1320 | layer_config["folder"] = folder |
| 1321 | layer_configs.append(layer_config) |
| 1322 | return layer_configs |
| 1323 | except Exception: |
| 1324 | return None |
| 1325 | |
| 1326 | |
| 1327 | def _read_safetensors_metadata_in_dir(local_dir: Path) -> dict[str, Any]: |
| 1328 | return { |
| 1329 | param_name: info |
| 1330 | for file_path in local_dir.glob("*.safetensors") |
| 1331 | if file_path.is_file() |
| 1332 | for param_name, info in parse_safetensors_file_metadata(file_path).items() |
| 1333 | } |
| 1334 | |
| 1335 | |
| 1336 | def get_safetensors_params_metadata( |
| 1337 | model: str, |
| 1338 | *, |
| 1339 | revision: str | None = None, |
| 1340 | ) -> dict[str, Any]: |
| 1341 | """Get the safetensors parameters metadata for remote/local model repository.""" |
| 1342 | if (model_path := Path(model)).exists(): |
| 1343 | return _read_safetensors_metadata_in_dir(model_path) |
| 1344 | |
| 1345 | repo_mt = try_get_safetensors_metadata(model, revision=revision) |
| 1346 | if repo_mt and (files_mt := repo_mt.files_metadata): |
| 1347 | return { |
| 1348 | param_name: asdict(info) |
| 1349 | for file_mt in files_mt.values() |
| 1350 | for param_name, info in file_mt.tensors.items() |
| 1351 | } |
| 1352 | |
| 1353 | # Hub fetch failed (e.g. 429, network unreachable). Fall back to the |
| 1354 | # local HF cache: weights may already be cached from a prior run, and |
| 1355 | # weight loading itself uses the same cache. |
| 1356 | try: |
| 1357 | local_dir = hf_api().snapshot_download( |
| 1358 | repo_id=model, |
| 1359 | revision=revision, |
| 1360 | allow_patterns=["*.safetensors"], |
| 1361 | local_files_only=True, |
| 1362 | ) |
| 1363 | except huggingface_hub.errors.LocalEntryNotFoundError as e: |
| 1364 | logger.warning_once( |
| 1365 | "Could not retrieve safetensors metadata for %s " |
| 1366 | "(Hub fetch failed and no local cache snapshot is available): %s.", |
| 1367 | model, |
| 1368 | str(e), |
| 1369 | ) |
| 1370 | return {} |
| 1371 | return _read_safetensors_metadata_in_dir(Path(local_dir)) |
| 1372 | |
| 1373 | |
| 1374 | @cache |
| 1375 | def checkpoint_has_lm_head(model: str, *, revision: str | None = None) -> bool | None: |
| 1376 | """Whether the checkpoint contains an `lm_head` tensor of its own. |
| 1377 | |
| 1378 | Args: |
| 1379 | model: Name or path of the model repository. |
| 1380 | revision: The specific model version to use. |
| 1381 | |
| 1382 | Returns: |
| 1383 | `None` if the checkpoint contents could not be determined, for example |
| 1384 | because it is not stored as safetensors. |
| 1385 | |
| 1386 | """ |
| 1387 | metadata = get_safetensors_params_metadata(model, revision=revision) |
| 1388 | if not metadata: |
| 1389 | return None |
| 1390 | return any(name.endswith("lm_head.weight") for name in metadata) |
| 1391 | |
| 1392 | |
| 1393 | def _download_mistral_config_file(model, revision) -> dict: |
| 1394 | config_file_name = "params.json" |
| 1395 | config_dict = get_hf_file_to_dict(config_file_name, model, revision) |
| 1396 | if config_dict is None: |
| 1397 | raise ValueError( |
| 1398 | f"Failed to load mistral '{config_file_name}' config for model " |
| 1399 | f"{model}. Please check if the model is a mistral-format model " |
| 1400 | f"and if the config file exists." |
| 1401 | ) |
| 1402 | assert isinstance(config_dict, dict) |
| 1403 | return config_dict |
| 1404 | |
| 1405 | |
| 1406 | def _maybe_retrieve_max_pos_from_hf(model, revision, **kwargs) -> int: |
| 1407 | max_position_embeddings = 128_000 |
| 1408 | try: |
| 1409 | trust_remote_code_val = kwargs.get("trust_remote_code", False) |
| 1410 | hf_config = get_config( |
| 1411 | model=model, |
| 1412 | trust_remote_code=trust_remote_code_val, |
| 1413 | revision=revision, |
| 1414 | config_format="hf", |
| 1415 | ) |
| 1416 | if hf_value := hf_config.get_text_config().max_position_embeddings: |
| 1417 | max_position_embeddings = hf_value |
| 1418 | except Exception as e: |
| 1419 | logger.warning( |
| 1420 | "The params.json file is missing 'max_position_embeddings'" |
| 1421 | " and could not get a value from the HF config." |
| 1422 | " Defaulting to 128000", |
| 1423 | exc_info=e, |
| 1424 | ) |
| 1425 | |
| 1426 | return max_position_embeddings |