| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | from collections.abc import Callable |
| 5 | from dataclasses import InitVar, field |
| 6 | from functools import cached_property |
| 7 | from typing import TYPE_CHECKING, Any, Literal, cast, get_args |
| 8 | |
| 9 | import torch |
| 10 | from pydantic import ConfigDict, Field, field_validator, model_validator |
| 11 | |
| 12 | import vllm.envs as envs |
| 13 | from vllm.config.model_arch import ( |
| 14 | ModelArchitectureConfig, |
| 15 | ) |
| 16 | from vllm.config.multimodal import ( |
| 17 | MMCacheType, |
| 18 | MMEncoderTPMode, |
| 19 | MMHasherAlgorithm, |
| 20 | MMProcessorDevice, |
| 21 | MMTensorIPC, |
| 22 | MultiModalConfig, |
| 23 | ) |
| 24 | from vllm.config.pooler import POOLER_CONFIG_LOG_FIELDS, PoolerConfig |
| 25 | from vllm.config.quantization import QuantizationConfigArgs |
| 26 | from vllm.config.scheduler import RunnerType |
| 27 | from vllm.config.utils import config, getattr_iter |
| 28 | from vllm.logger import init_logger |
| 29 | from vllm.platforms import CpuArchEnum, current_platform |
| 30 | from vllm.tasks import PoolingTask, ScoreType, SupportedTask |
| 31 | from vllm.transformers_utils.config import ( |
| 32 | ConfigFormat, |
| 33 | checkpoint_has_lm_head, |
| 34 | get_config, |
| 35 | get_hf_image_processor_config, |
| 36 | get_hf_text_config, |
| 37 | get_pooling_config, |
| 38 | get_sentence_transformer_tokenizer_config, |
| 39 | is_encoder_decoder, |
| 40 | is_rope_parameters_nested, |
| 41 | mrope_num_dims, |
| 42 | try_get_dense_modules, |
| 43 | try_get_generation_config, |
| 44 | try_get_tokenizer_config, |
| 45 | uses_mrope, |
| 46 | ) |
| 47 | from vllm.transformers_utils.model_arch_config_convertor import ( |
| 48 | MODEL_ARCH_CONFIG_CONVERTORS, |
| 49 | ModelArchConfigConvertorBase, |
| 50 | ) |
| 51 | from vllm.transformers_utils.repo_utils import resolve_revision |
| 52 | from vllm.transformers_utils.runai_utils import ObjectStorageModel, is_runai_obj_uri |
| 53 | from vllm.transformers_utils.utils import maybe_model_redirect |
| 54 | from vllm.utils.import_utils import LazyLoader |
| 55 | from vllm.v1.attention.backends.registry import AttentionBackendEnum |
| 56 | |
| 57 | if TYPE_CHECKING: |
| 58 | from transformers import PreTrainedConfig |
| 59 | |
| 60 | import vllm.model_executor.layers.quantization as me_quant |
| 61 | import vllm.model_executor.models as me_models |
| 62 | from vllm.config.load import LoadConfig |
| 63 | from vllm.config.parallel import ParallelConfig |
| 64 | from vllm.model_executor.layers.quantization import QuantizationMethods |
| 65 | from vllm.v1.sample.logits_processor import LogitsProcessor |
| 66 | else: |
| 67 | PreTrainedConfig = Any |
| 68 | |
| 69 | me_quant = LazyLoader( |
| 70 | "model_executor", globals(), "vllm.model_executor.layers.quantization" |
| 71 | ) |
| 72 | me_models = LazyLoader("model_executor", globals(), "vllm.model_executor.models") |
| 73 | LoadConfig = Any |
| 74 | ParallelConfig = Any |
| 75 | QuantizationMethods = str |
| 76 | LogitsProcessor = Any |
| 77 | |
| 78 | logger = init_logger(__name__) |
| 79 | |
| 80 | # Process-local record of which (arch, target) model-class overrides have been |
| 81 | # registered in *this* process. Must not live on ModelConfig: that instance is |
| 82 | # pickled to each worker, so an instance flag would arrive already "registered" |
| 83 | # while the worker's own global ModelRegistry is still untouched. |
| 84 | _REGISTERED_MODEL_CLASS_OVERRIDES: set[tuple[str, str]] = set() |
| 85 | |
| 86 | RunnerOption = Literal["auto", RunnerType] |
| 87 | ConvertType = Literal["none", "embed", "classify"] |
| 88 | ConvertOption = Literal["auto", ConvertType] |
| 89 | TokenizerMode = Literal[ |
| 90 | "auto", |
| 91 | "hf", |
| 92 | "mistral", |
| 93 | "deepseek_v32", |
| 94 | "deepseek_v4", |
| 95 | "inkling", |
| 96 | "kimi_k3", |
| 97 | "cohere", |
| 98 | ] |
| 99 | ModelDType = Literal["auto", "half", "float16", "bfloat16", "float", "float32"] |
| 100 | LogprobsMode = Literal[ |
| 101 | "raw_logits", "raw_logprobs", "processed_logits", "processed_logprobs" |
| 102 | ] |
| 103 | PROCESSED_LOGPROBS_MODES: tuple[LogprobsMode, ...] = ( |
| 104 | "processed_logits", |
| 105 | "processed_logprobs", |
| 106 | ) |
| 107 | HfOverrides = dict[str, Any] | Callable[[PreTrainedConfig], PreTrainedConfig] |
| 108 | ModelImpl = Literal["auto", "vllm", "transformers", "terratorch"] |
| 109 | LayerBlockType = Literal["attention", "linear_attention", "mamba"] |
| 110 | |
| 111 | _ATTENTION_LAYER_TYPES = frozenset( |
| 112 | { |
| 113 | "full_attention", |
| 114 | "deepseek_sparse_attention", |
| 115 | "qwen_sparse_attention", |
| 116 | } |
| 117 | ) |
| 118 | """`layer_types` spellings that consume a full attention KV cache. Sparse |
| 119 | attention still caches every token, so it counts as attention here.""" |
| 120 | |
| 121 | _RUNNER_CONVERTS: dict[RunnerType, list[ConvertType]] = { |
| 122 | "generate": [], |
| 123 | "pooling": ["embed", "classify"], |
| 124 | "draft": [], |
| 125 | } |
| 126 | |
| 127 | |
| 128 | def _modelopt_mixed_has_nvfp4(quant_config: dict[str, Any] | None) -> bool: |
| 129 | """Whether a ModelOpt MIXED_PRECISION checkpoint quantizes any layer to NVFP4. |
| 130 | |
| 131 | ``W4A16_NVFP4`` is excluded on purpose: it quantizes weights only, so there |
| 132 | is no activation quantization for the fusion passes to act on. |
| 133 | """ |
| 134 | layers = (quant_config or {}).get("quantized_layers") |
| 135 | if not isinstance(layers, dict): |
| 136 | return False |
| 137 | return any( |
| 138 | isinstance(info, dict) and info.get("quant_algo") == "NVFP4" |
| 139 | for info in layers.values() |
| 140 | ) |
| 141 | |
| 142 | |
| 143 | AttnTypeStr = Literal[ |
| 144 | "decoder", "encoder", "encoder_only", "encoder_decoder", "attention_free", "hybrid" |
| 145 | ] |
| 146 | |
| 147 | |
| 148 | @config(config=ConfigDict(arbitrary_types_allowed=True)) |
| 149 | class ModelConfig: |
| 150 | """Configuration for the model.""" |
| 151 | |
| 152 | model: str = "Qwen/Qwen3-0.6B" |
| 153 | """Name or path of the Hugging Face model to use. It is also used as the |
| 154 | content for `model_name` tag in metrics output when `served_model_name` is |
| 155 | not specified.""" |
| 156 | model_weights: str = "" |
| 157 | """Original model weights path. Used when the model is pulled from object |
| 158 | storage (e.g., RunAI) to preserve the original URI while `model` points to |
| 159 | the local directory.""" |
| 160 | runner: RunnerOption = "auto" |
| 161 | """The type of model runner to use. Each vLLM instance only supports one |
| 162 | model runner, even if the same model can be used for multiple types.""" |
| 163 | convert: ConvertOption = "auto" |
| 164 | """Convert the model using adapters defined in |
| 165 | [vllm.model_executor.models.adapters][]. The most common use case is to |
| 166 | adapt a text generation model to be used for pooling tasks.""" |
| 167 | tokenizer: str = None # type: ignore[assignment] |
| 168 | """Name or path of the Hugging Face tokenizer to use. If unspecified, model |
| 169 | name or path will be used.""" |
| 170 | tokenizer_mode: TokenizerMode | str = "auto" |
| 171 | """Tokenizer mode: |
| 172 | |
| 173 | - "auto" will use the tokenizer from `mistral_common` for Mistral models |
| 174 | if available, otherwise it will use the "hf" tokenizer. |
| 175 | - "hf" will use the fast tokenizer if available. |
| 176 | - "mistral" will always use the tokenizer from `mistral_common`. |
| 177 | - "deepseek_v32" will always use the tokenizer from `deepseek_v32`. |
| 178 | - "deepseek_v4" will always use the tokenizer from `deepseek_v4`. |
| 179 | - "deepseek_v41" will use the DeepSeek V4.1 prompt encoder. |
| 180 | - "kimi_k3" will always use the "hf" tokenizer but render chat prompts |
| 181 | with Kimi K3's Python XTML encoding instead of a Jinja template. |
| 182 | - "cohere" uses the standard HF tokenizer but renders the chat template |
| 183 | via the `cohere_melody` library (cmd3 / cmd4 templates) instead of |
| 184 | Jinja, and surfaces grounded-citation metadata on responses. |
| 185 | - Other custom values can be supported via plugins. |
| 186 | |
| 187 | To swap the Rust BPE backend that powers HF fast tokenizers for the |
| 188 | [fastokens](https://github.com/crusoecloud/fastokens) implementation, set |
| 189 | `VLLM_USE_FASTOKENS=1` instead — that override applies to any mode that |
| 190 | loads an HF fast tokenizer (`hf`, `deepseek_v32`, `deepseek_v4`, …).""" |
| 191 | trust_remote_code: bool = False |
| 192 | """Trust remote code (e.g., from HuggingFace) when downloading the model |
| 193 | and tokenizer.""" |
| 194 | dtype: ModelDType | torch.dtype = "auto" |
| 195 | """Data type for model weights and activations: |
| 196 | |
| 197 | - "auto" will use FP16 precision for FP32 and FP16 models, and BF16 |
| 198 | precision for BF16 models. |
| 199 | - "half" for FP16. Recommended for AWQ quantization. |
| 200 | - "float16" is the same as "half". |
| 201 | - "bfloat16" for a balance between precision and range. |
| 202 | - "float" is shorthand for FP32 precision. |
| 203 | - "float32" for FP32 precision.""" |
| 204 | seed: int = 0 |
| 205 | """Random seed for reproducibility. |
| 206 | |
| 207 | We must set the global seed because otherwise, |
| 208 | different tensor parallel workers would sample different tokens, |
| 209 | leading to inconsistent results.""" |
| 210 | hf_config: PreTrainedConfig = field(init=False) |
| 211 | """The Hugging Face config of the model.""" |
| 212 | hf_text_config: PreTrainedConfig = field(init=False) |
| 213 | """The Hugging Face config of the text model (same as hf_config for text models).""" |
| 214 | is_submodel_config: bool = field(default=False, init=False) |
| 215 | """Whether this is a submodule view derived by `VllmConfig.with_hf_config` |
| 216 | (e.g. a multimodal model's text stack). Its architecture list is empty, so |
| 217 | deployment-level validation must not run against it.""" |
| 218 | word_embeddings_untied_by_checkpoint: bool = field(default=False, init=False) |
| 219 | """Whether `tie_word_embeddings` was overridden to `False` because the checkpoint |
| 220 | contains an `lm_head` of its own. The two may still turn out to be identical, in |
| 221 | which case they are re-tied once the weights have been loaded.""" |
| 222 | hf_config_path: str | None = None |
| 223 | """Name or path of the Hugging Face config to use. If unspecified, model |
| 224 | name or path will be used.""" |
| 225 | allowed_local_media_path: str = "" |
| 226 | """Allowing API requests to read local images or videos from directories |
| 227 | specified by the server file system. This is a security risk. Should only |
| 228 | be enabled in trusted environments.""" |
| 229 | allowed_media_domains: list[str] | None = None |
| 230 | """If set, only media URLs that belong to this domain can be used for |
| 231 | multi-modal inputs. """ |
| 232 | revision: str | None = None |
| 233 | """The specific model version to use. It can be a branch name, a tag name, |
| 234 | or a commit id. If unspecified, will use the default version.""" |
| 235 | code_revision: str | None = None |
| 236 | """The specific revision to use for the model code on the Hugging Face Hub. |
| 237 | It can be a branch name, a tag name, or a commit id. If unspecified, will |
| 238 | use the default version.""" |
| 239 | tokenizer_revision: str | None = None |
| 240 | """The specific revision to use for the tokenizer on the Hugging Face Hub. |
| 241 | It can be a branch name, a tag name, or a commit id. If unspecified, will |
| 242 | use the default version.""" |
| 243 | max_model_len: int = Field(default=None, ge=-1) # type: ignore[assignment] |
| 244 | """Model context length (prompt and output). If unspecified, will be |
| 245 | automatically derived from the model config. |
| 246 | |
| 247 | When passing via `--max-model-len`, supports k/m/g/K/M/G in human-readable |
| 248 | format. Examples: |
| 249 | |
| 250 | - 1k -> 1000 |
| 251 | - 1K -> 1024 |
| 252 | - 25.6k -> 25,600 |
| 253 | - -1 or 'auto' -> Automatically choose the maximum model length that fits in |
| 254 | GPU memory. This will use the model's maximum context length if it fits, |
| 255 | otherwise it will find the largest length that can be accommodated.""" |
| 256 | spec_target_max_model_len: int | None = None |
| 257 | """Specify the maximum length for spec decoding draft models.""" |
| 258 | quantization: QuantizationMethods | str | None = None |
| 259 | """Method used to quantize the weights. If `None`, we first check the |
| 260 | `quantization_config` attribute in the model config file. If that is |
| 261 | `None`, we assume the model weights are not quantized and use `dtype` to |
| 262 | determine the data type of the weights.""" |
| 263 | quantization_config: dict[str, Any] | QuantizationConfigArgs | None = None |
| 264 | """User-facing quantization configuration. Carries per-layer-kind specs |
| 265 | (linear, moe) and ignore patterns; see :class:`QuantizationConfigArgs`. |
| 266 | Auto-populated from the matching online shorthand when `quantization` is |
| 267 | one of the values in `ONLINE_QUANT_SHORTHAND_NAMES`.""" |
| 268 | allow_deprecated_quantization: bool = False |
| 269 | """Whether to allow deprecated quantization methods.""" |
| 270 | enforce_eager: bool = False |
| 271 | """Whether to always use eager-mode PyTorch. If True, we will disable CUDA |
| 272 | graph and always execute the model in eager mode. If False, we will use |
| 273 | CUDA graph and eager execution in hybrid for maximal performance and |
| 274 | flexibility. |
| 275 | |
| 276 | NOTE: This disables both `torch.compile` and CUDA graphs, and is |
| 277 | equivalent to setting `-cc.mode=none -cc.cudagraph_mode=none`.""" |
| 278 | return_sampling_mask: bool = False |
| 279 | """Whether to return the post-processing token support for each sample.""" |
| 280 | max_logprobs: int = Field(default=20, ge=-1) |
| 281 | """Maximum number of log probabilities to return when `logprobs` is |
| 282 | specified in `SamplingParams`. The default value comes the default for the |
| 283 | OpenAI Chat Completions API. -1 means no cap, i.e. all (output_length * |
| 284 | vocab_size) logprobs are allowed to be returned and it may cause OOM.""" |
| 285 | logprobs_mode: LogprobsMode = "raw_logprobs" |
| 286 | """Indicates the content returned in the logprobs and prompt_logprobs. |
| 287 | Supported mode: |
| 288 | 1) raw_logprobs, 2) processed_logprobs, 3) raw_logits, 4) processed_logits. |
| 289 | Raw means the values before applying any logit processors, like bad words. |
| 290 | Processed means the values after applying all processors, including |
| 291 | temperature and top_k/top_p. |
| 292 | Note: for prompt_logprobs, processed_* and raw_* yield identical results |
| 293 | because prompt tokens do not go through sampling processors. |
| 294 | """ |
| 295 | use_fp64_gumbel: bool = False |
| 296 | """Whether to use FP64 (instead of FP32) random noise for Gumbel-max and |
| 297 | equivalent exponential-race sampling. FP64 preserves lower-tail sampling |
| 298 | events that fp32 uniform/exponential draws can truncate, at the cost of |
| 299 | significantly lower throughput on most GPUs.""" |
| 300 | enable_trace_replay: bool = False |
| 301 | """Whether to allow requests to set |
| 302 | `SamplingParams.trace_decode_token_ids`, which forces decoding to follow a |
| 303 | predetermined token sequence while still computing real logprobs. Reserved |
| 304 | for debugging and RL workflows: enabling it reserves a per-request trace |
| 305 | buffer, so it is off by default.""" |
| 306 | disable_sliding_window: bool = False |
| 307 | """Whether to disable sliding window. If True, we will disable the sliding |
| 308 | window functionality of the model, capping to sliding window size. If the |
| 309 | model does not support sliding window, this argument is ignored.""" |
| 310 | disable_cascade_attn: bool = True |
| 311 | """Disable cascade attention for V1. While cascade attention does not |
| 312 | change the mathematical correctness, disabling it could be useful for |
| 313 | preventing potential numerical issues. This defaults to True, so users |
| 314 | must opt in to cascade attention by setting this to False. Even when this |
| 315 | is set to False, cascade attention will only be used when the heuristic |
| 316 | tells that it's beneficial.""" |
| 317 | skip_tokenizer_init: bool = False |
| 318 | """Skip initialization of tokenizer and detokenizer. Expects valid |
| 319 | `prompt_token_ids` and `None` for prompt from the input. The generated |
| 320 | output will contain token ids.""" |
| 321 | enable_prompt_embeds: bool = False |
| 322 | """If `True`, enables passing text embeddings as inputs via the |
| 323 | `prompt_embeds` key. |
| 324 | |
| 325 | WARNING: The vLLM engine may crash if incorrect shape of embeddings is passed. |
| 326 | Only enable this flag for trusted users!""" |
| 327 | served_model_name: str | list[str] | None = None |
| 328 | """The model name(s) used in the API. If multiple names are provided, the |
| 329 | server will respond to any of the provided names. The model name in the |
| 330 | model field of a response will be the first name in this list. If not |
| 331 | specified, the model name will be the same as the `--model` argument. Noted |
| 332 | that this name(s) will also be used in `model_name` tag content of |
| 333 | prometheus metrics, if multiple names provided, metrics tag will take the |
| 334 | first one.""" |
| 335 | config_format: str | ConfigFormat = "auto" |
| 336 | """The format of the model config to load: |
| 337 | |
| 338 | - "auto" will try to load the config in hf format if available after trying |
| 339 | to load in mistral format. |
| 340 | - "hf" will load the config in hf format. |
| 341 | - "mistral" will load the config in mistral format.""" |
| 342 | hf_token: bool | str | None = None |
| 343 | """The token to use as HTTP bearer authorization for remote files . If |
| 344 | `True`, will use the token generated when running `hf auth login` |
| 345 | (stored in `~/.cache/huggingface/token`).""" |
| 346 | hf_overrides: HfOverrides = field(default_factory=dict) |
| 347 | """If a dictionary, contains arguments to be forwarded to the Hugging Face |
| 348 | config. If a callable, it is called to update the HuggingFace config.""" |
| 349 | model_class_overrides: dict[str, str] = field(default_factory=dict) |
| 350 | """Override the model class used for one or more architectures, mapping the |
| 351 | architecture name to a `"module:class"` target (the same format accepted by |
| 352 | `ModelRegistry.register_model`). This registers the target class at runtime, |
| 353 | e.g. `{"GlmMoeDsaForCausalLM": |
| 354 | "vllm.models.deepseek_v32.nvidia.model:DeepseekV32ForCausalLM"}`. This |
| 355 | argument is for development and debugging purposes only.""" |
| 356 | generation_config: str = "auto" |
| 357 | """The folder path to the generation config. Defaults to `"auto"`, the |
| 358 | generation config will be loaded from model path. If set to `"vllm"`, no |
| 359 | generation config is loaded, vLLM defaults will be used. If set to a folder |
| 360 | path, the generation config will be loaded from the specified folder path. |
| 361 | If `max_new_tokens` is specified in generation config, then it sets a |
| 362 | server-wide limit on the number of output tokens for all requests.""" |
| 363 | override_generation_config: dict[str, Any] = field(default_factory=dict) |
| 364 | """Overrides or sets generation config. e.g. `{"temperature": 0.5}`. If |
| 365 | used with `--generation-config auto`, the override parameters will be |
| 366 | merged with the default config from the model. If used with |
| 367 | `--generation-config vllm`, only the override parameters are used.""" |
| 368 | enable_sleep_mode: bool = False |
| 369 | """Enable sleep mode for the engine (only cuda and |
| 370 | hip platforms are supported).""" |
| 371 | sleep_preserve_parameter_names: list[str] = field(default_factory=list) |
| 372 | """Parameter-name globs to preserve across level-2 sleep. |
| 373 | The sender must omit these parameters; loaders must preserve their storage.""" |
| 374 | sleep_mode_backend: str = "cumem" |
| 375 | """Mechanism used to free and restore GPU state for sleep mode. ``"cumem"`` |
| 376 | (default) uses the built-in ``CuMemAllocator`` and is behavior-compatible |
| 377 | with prior releases. Additional backends (CUDA checkpoint, CRIU, durable |
| 378 | snapshot) may be registered in-tree or by plugins (RFC #34303).""" |
| 379 | enable_nccl_comm_suspend: bool = False |
| 380 | """Enable releasing NCCL communicator memory during sleep mode |
| 381 | (``ncclCommSuspend``/``ncclCommResume``). Experimental; when disabled |
| 382 | (the default) sleep still releases weights/KV-cache memory as before.""" |
| 383 | enable_cumem_allocator: bool = False |
| 384 | """Enable the custom cumem allocator to leverage advanced GPU memory |
| 385 | allocation features such as multi-node NVLink support. |
| 386 | |
| 387 | Sleep mode automatically enables this allocator. Only cuda and hip |
| 388 | platforms are supported. |
| 389 | """ |
| 390 | model_impl: str | ModelImpl = "auto" |
| 391 | """Which implementation of the model to use: |
| 392 | |
| 393 | - "auto" will try to use the vLLM implementation, if it exists, and fall back to the |
| 394 | Transformers implementation if no vLLM implementation is available. |
| 395 | - "vllm" will use the vLLM model implementation. |
| 396 | - "transformers" will use the Transformers model implementation. |
| 397 | - "terratorch" will use the TerraTorch model implementation. |
| 398 | """ |
| 399 | logits_processors: list[str | type[LogitsProcessor]] | None = None |
| 400 | """One or more logits processors' fully-qualified class names or class |
| 401 | definitions""" |
| 402 | io_processor_plugin: str | None = None |
| 403 | """IOProcessor plugin name to load at model startup""" |
| 404 | renderer_num_workers: int = 1 |
| 405 | """Number of worker threads in the renderer thread pool. The pool is |
| 406 | consumed by the async renderer path (e.g. the OpenAI-compatible API |
| 407 | server started by `vllm serve`) to parallelize tokenization, chat |
| 408 | template rendering, and multimodal preprocessing across concurrent |
| 409 | requests. |
| 410 | |
| 411 | The offline `LLM` entrypoint uses the synchronous renderer path and |
| 412 | processes prompts (including multimodal preprocessing) serially, so |
| 413 | this setting has no effect there.""" |
| 414 | |
| 415 | # Pooler config |
| 416 | pooler_config: PoolerConfig | None = None |
| 417 | """Pooler config which controls the behaviour of output pooling in pooling |
| 418 | models.""" |
| 419 | |
| 420 | # Multimodal config and init vars |
| 421 | multimodal_config: MultiModalConfig | None = None |
| 422 | """Configuration for multimodal model. If `None`, this will be inferred |
| 423 | from the architecture of `self.model`.""" |
| 424 | language_model_only: InitVar[bool] = False |
| 425 | limit_mm_per_prompt: InitVar[dict[str, int | dict[str, int]] | None] = None |
| 426 | enable_mm_embeds: InitVar[bool | None] = None |
| 427 | media_io_kwargs: InitVar[dict[str, dict[str, Any]] | None] = None |
| 428 | mm_processor_kwargs: InitVar[dict[str, Any] | None] = None |
| 429 | mm_processor_cache_gb: InitVar[float | None] = None |
| 430 | mm_processor_cache_type: InitVar[MMCacheType | None] = None |
| 431 | mm_hasher_algorithm: InitVar[MMHasherAlgorithm | None] = None |
| 432 | mm_shm_cache_max_object_size_mb: InitVar[int | None] = None |
| 433 | mm_encoder_only: InitVar[bool | None] = None |
| 434 | mm_encoder_tp_mode: InitVar[MMEncoderTPMode | None] = None |
| 435 | mm_encoder_attn_backend: InitVar[AttentionBackendEnum | str | None] = None |
| 436 | mm_encoder_attn_dtype: InitVar[str | None] = None |
| 437 | mm_encoder_fp8_scale_path: InitVar[str | None] = None |
| 438 | mm_encoder_fp8_scale_save_path: InitVar[str | None] = None |
| 439 | mm_encoder_fp8_scale_save_margin: InitVar[float | None] = None |
| 440 | interleave_mm_strings: InitVar[bool | None] = None |
| 441 | skip_mm_profiling: InitVar[bool | None] = None |
| 442 | video_pruning_rate: InitVar[float | None] = None |
| 443 | video_pruning_method: InitVar[str | None] = None |
| 444 | mm_tensor_ipc: InitVar[MMTensorIPC] = None |
| 445 | mm_ipc_gpu_memory_gb: InitVar[float | None] = None |
| 446 | mm_device_do_normalize: InitVar[bool | None] = None |
| 447 | mm_processor_device: InitVar[MMProcessorDevice | None] = None |
| 448 | |
| 449 | def compute_hash(self) -> str: |
| 450 | """WARNING: Whenever a new field is added to this config, |
| 451 | ensure that it is included in the factors list if |
| 452 | it affects the computation graph. |
| 453 | |
| 454 | Provide a hash that uniquely identifies all the configs |
| 455 | that affect the structure of the computation |
| 456 | graph from input ids/embeddings to the final hidden states, |
| 457 | excluding anything before input ids/embeddings and after |
| 458 | the final hidden states. |
| 459 | """ |
| 460 | ignored_factors = { |
| 461 | "convert", |
| 462 | "tokenizer", |
| 463 | "tokenizer_mode", |
| 464 | "seed", |
| 465 | "hf_config_path", |
| 466 | "allowed_local_media_path", |
| 467 | "allowed_media_domains", |
| 468 | "tokenizer_revision", |
| 469 | "spec_target_max_model_len", |
| 470 | "enforce_eager", |
| 471 | "return_sampling_mask", |
| 472 | "logprobs_mode", |
| 473 | "use_fp64_gumbel", |
| 474 | "enable_trace_replay", |
| 475 | "disable_cascade_attn", |
| 476 | "skip_tokenizer_init", |
| 477 | "served_model_name", |
| 478 | "config_format", |
| 479 | "hf_token", |
| 480 | "hf_overrides", |
| 481 | "logits_processors", |
| 482 | "io_processor_plugin", |
| 483 | "pooler_config", |
| 484 | "multimodal_config", |
| 485 | "limit_mm_per_prompt", |
| 486 | "media_io_kwargs", |
| 487 | "mm_processor_kwargs", |
| 488 | "mm_processor_cache_gb", |
| 489 | "mm_processor_cache_type", |
| 490 | "mm_shm_cache_max_object_size_mb", |
| 491 | "mm_encoder_tp_mode", |
| 492 | "interleave_mm_strings", |
| 493 | "skip_mm_profiling", |
| 494 | "mm_ipc_gpu_memory_gb", |
| 495 | } |
| 496 | |
| 497 | from vllm.config.utils import get_hash_factors, hash_factors |
| 498 | |
| 499 | factors = get_hash_factors(self, ignored_factors) |
| 500 | |
| 501 | # NOTE: For some models (e.g, Qwen3-VL), whether the MM code path is enabled |
| 502 | # affects the computation graph of the language model, therefore we add it |
| 503 | # here early. |
| 504 | if self.multimodal_config: |
| 505 | factors["language_model_only"] = self.multimodal_config.language_model_only |
| 506 | return hash_factors(factors) |
| 507 | |
| 508 | def _update_nested( |
| 509 | self, |
| 510 | target: PreTrainedConfig | dict[str, Any], |
| 511 | updates: dict[str, Any], |
| 512 | ) -> None: |
| 513 | """Recursively updates a config or dict with nested updates.""" |
| 514 | for key, value in updates.items(): |
| 515 | if isinstance(value, dict): |
| 516 | # Get the nested target |
| 517 | if isinstance(target, dict): |
| 518 | nested_target = target.get(key) |
| 519 | else: |
| 520 | nested_target = getattr(target, key, None) |
| 521 | |
| 522 | # If nested target exists and can be updated recursively |
| 523 | if nested_target is not None and ( |
| 524 | isinstance(nested_target, dict) |
| 525 | or hasattr(nested_target, "__dict__") |
| 526 | ): |
| 527 | self._update_nested(nested_target, value) |
| 528 | continue |
| 529 | |
| 530 | # Set the value (base case) |
| 531 | if isinstance(target, dict): |
| 532 | target[key] = value |
| 533 | else: |
| 534 | setattr(target, key, value) |
| 535 | |
| 536 | def _apply_dict_overrides( |
| 537 | self, |
| 538 | config: PreTrainedConfig, |
| 539 | overrides: dict[str, Any], |
| 540 | ) -> None: |
| 541 | """Apply dict overrides, handling both nested configs and dict values.""" |
| 542 | from transformers import PreTrainedConfig |
| 543 | |
| 544 | for key, value in overrides.items(): |
| 545 | attr = getattr(config, key, None) |
| 546 | if attr is not None and isinstance(attr, PreTrainedConfig): |
| 547 | # It's a nested config - recursively update it |
| 548 | self._update_nested(attr, value) |
| 549 | else: |
| 550 | # It's a dict-valued parameter - set it directly |
| 551 | setattr(config, key, value) |
| 552 | |
| 553 | def __post_init__( |
| 554 | self, |
| 555 | # Multimodal config init vars |
| 556 | language_model_only: bool, |
| 557 | limit_mm_per_prompt: dict[str, int | dict[str, int]] | None, |
| 558 | enable_mm_embeds: bool | None, |
| 559 | media_io_kwargs: dict[str, dict[str, Any]] | None, |
| 560 | mm_processor_kwargs: dict[str, Any] | None, |
| 561 | mm_processor_cache_gb: float | None, |
| 562 | mm_processor_cache_type: MMCacheType | None, |
| 563 | mm_hasher_algorithm: MMHasherAlgorithm | None, |
| 564 | mm_shm_cache_max_object_size_mb: int | None, |
| 565 | mm_encoder_only: bool | None, |
| 566 | mm_encoder_tp_mode: MMEncoderTPMode | None, |
| 567 | mm_encoder_attn_backend: AttentionBackendEnum | str | None, |
| 568 | mm_encoder_attn_dtype: str | None, |
| 569 | mm_encoder_fp8_scale_path: str | None, |
| 570 | mm_encoder_fp8_scale_save_path: str | None, |
| 571 | mm_encoder_fp8_scale_save_margin: float | None, |
| 572 | interleave_mm_strings: bool | None, |
| 573 | skip_mm_profiling: bool | None, |
| 574 | video_pruning_rate: float | None, |
| 575 | video_pruning_method: str | None, |
| 576 | mm_tensor_ipc: MMTensorIPC, |
| 577 | mm_ipc_gpu_memory_gb: float | None, |
| 578 | mm_device_do_normalize: bool | None, |
| 579 | mm_processor_device: MMProcessorDevice | None, |
| 580 | ) -> None: |
| 581 | # Keep set served_model_name before maybe_model_redirect(self.model) |
| 582 | self.served_model_name = get_served_model_name( |
| 583 | self.model, self.served_model_name |
| 584 | ) |
| 585 | requested_revision = self.revision |
| 586 | self.model = maybe_model_redirect(self.model) |
| 587 | # The tokenizer is consistent with the model by default. |
| 588 | if self.tokenizer is None: |
| 589 | self.tokenizer = self.model |
| 590 | if self.tokenizer_revision is None: |
| 591 | self.tokenizer_revision = self.revision |
| 592 | self.tokenizer = maybe_model_redirect(self.tokenizer) |
| 593 | |
| 594 | if isinstance(self.hf_config_path, str): |
| 595 | self.hf_config_path = maybe_model_redirect(self.hf_config_path) |
| 596 | |
| 597 | if callable(self.hf_overrides): |
| 598 | hf_overrides_kw: dict[str, Any] = {} |
| 599 | hf_overrides_fn = self.hf_overrides |
| 600 | dict_overrides: dict[str, Any] = {} |
| 601 | else: |
| 602 | # Separate dict overrides from flat ones |
| 603 | # We'll determine how to apply dict overrides after loading the config |
| 604 | hf_overrides_kw = {} |
| 605 | dict_overrides = {} |
| 606 | for key, value in self.hf_overrides.items(): |
| 607 | if isinstance(value, dict): |
| 608 | dict_overrides[key] = value |
| 609 | else: |
| 610 | hf_overrides_kw[key] = value |
| 611 | hf_overrides_fn = None |
| 612 | |
| 613 | self.maybe_pull_model_tokenizer_for_runai(self.model, self.tokenizer) |
| 614 | |
| 615 | # If loading model/tokenizer from HF Hub, resolve the revision once |
| 616 | # to prevent resolving it multiple times downstream. A resolved revision |
| 617 | # only pins the repo it was resolved for, so each repo needs its own call. |
| 618 | self.revision = resolve_revision( |
| 619 | self.model, |
| 620 | self.revision, |
| 621 | self.hf_token, |
| 622 | ) |
| 623 | |
| 624 | # The config can live in another repo, which `self.revision` does not pin. |
| 625 | # It stays `None` if the config comes from `self.model`, so that call sites |
| 626 | # fall back to `self.revision` the same way they fall back to `self.model`. |
| 627 | self._hf_config_revision = None |
| 628 | if self.hf_config_path and self.hf_config_path != self.model: |
| 629 | self._hf_config_revision = resolve_revision( |
| 630 | self.hf_config_path, |
| 631 | self.revision, |
| 632 | self.hf_token, |
| 633 | ) |
| 634 | |
| 635 | if ( |
| 636 | self.tokenizer == self.model |
| 637 | and self.tokenizer_revision == requested_revision |
| 638 | ): |
| 639 | self.tokenizer_revision = self.revision |
| 640 | else: |
| 641 | self.tokenizer_revision = resolve_revision( |
| 642 | self.tokenizer, |
| 643 | self.tokenizer_revision, |
| 644 | self.hf_token, |
| 645 | ) |
| 646 | |
| 647 | if self.enable_sleep_mode: |
| 648 | if not current_platform.is_sleep_mode_available(): |
| 649 | raise ValueError("Sleep mode is not supported on current platform.") |
| 650 | if current_platform.is_cuda_alike() and not self.enable_cumem_allocator: |
| 651 | logger.info_once( |
| 652 | "Enabling cumem allocator because sleep mode requires it." |
| 653 | ) |
| 654 | self.enable_cumem_allocator = True |
| 655 | if ( |
| 656 | self.enable_cumem_allocator |
| 657 | and not current_platform.is_cumem_allocator_available() |
| 658 | ): |
| 659 | raise ValueError("cumem allocator is not supported on current platform.") |
| 660 | |
| 661 | hf_config = get_config( |
| 662 | self.hf_config_path or self.model, |
| 663 | self.trust_remote_code, |
| 664 | self._hf_config_revision or self.revision, |
| 665 | self.code_revision, |
| 666 | self.config_format, |
| 667 | hf_overrides_kw=hf_overrides_kw, |
| 668 | hf_overrides_fn=hf_overrides_fn, |
| 669 | token=self.hf_token, |
| 670 | ) |
| 671 | self.hf_config = hf_config |
| 672 | if dict_overrides: |
| 673 | self._apply_dict_overrides(hf_config, dict_overrides) |
| 674 | self.hf_text_config = get_hf_text_config(self.hf_config) |
| 675 | self.model_arch_config = self.get_model_arch_config() |
| 676 | self.attention_chunk_size = getattr( |
| 677 | self.hf_text_config, "attention_chunk_size", None |
| 678 | ) |
| 679 | self.encoder_config = self._get_encoder_config() |
| 680 | # Image-processor metadata is only consumed by multimodal models. |
| 681 | # Probing it for text-only models causes avoidable Hub requests. |
| 682 | self.hf_image_processor_config: dict[str, Any] = {} |
| 683 | |
| 684 | architectures = self.architectures |
| 685 | registry = self.registry |
| 686 | is_generative_model = registry.is_text_generation_model(architectures, self) |
| 687 | is_pooling_model = registry.is_pooling_model(architectures, self) |
| 688 | |
| 689 | self.runner_type = self._get_runner_type( |
| 690 | architectures, self.runner, self.convert |
| 691 | ) |
| 692 | self.convert_type = self._get_convert_type( |
| 693 | architectures, self.runner_type, self.convert |
| 694 | ) |
| 695 | |
| 696 | if ( |
| 697 | is_pooling_model |
| 698 | and not is_generative_model |
| 699 | and self.runner_type in ("draft", "generate") |
| 700 | ): |
| 701 | raise ValueError( |
| 702 | f"Embedding models do not support `--runner {self.runner_type}`. " |
| 703 | "Use `--runner pooling` or `--runner auto` for embedding models." |
| 704 | ) |
| 705 | if self.runner_type == "generate" and not is_generative_model: |
| 706 | generate_converts = _RUNNER_CONVERTS["generate"] |
| 707 | if self.convert_type not in generate_converts: |
| 708 | # Currently we don't have any converters for generative models |
| 709 | raise ValueError("This model does not support `--runner generate`.") |
| 710 | if self.runner_type == "pooling" and not is_pooling_model: |
| 711 | pooling_converts = _RUNNER_CONVERTS["pooling"] |
| 712 | if self.convert_type not in pooling_converts: |
| 713 | convert_option = "<" + "|".join(pooling_converts) + ">" |
| 714 | raise ValueError( |
| 715 | "This model does not support `--runner pooling`. " |
| 716 | f"You can pass `--convert {convert_option} to adapt " |
| 717 | "it into a pooling model." |
| 718 | ) |
| 719 | |
| 720 | # Note: Initialize these attributes early because transformers fallback |
| 721 | # may fail to load dynamic modules in child processes |
| 722 | model_info, arch = registry.inspect_model_cls(architectures, self) |
| 723 | self._model_info = model_info |
| 724 | self._architecture = arch |
| 725 | logger.info("Resolved architecture: %s", arch) |
| 726 | |
| 727 | # Set default tokenizer modes based on model architecture |
| 728 | if self.tokenizer_mode == "auto": |
| 729 | if self.model_impl == "terratorch": |
| 730 | self.tokenizer_mode = "terratorch" |
| 731 | elif arch == "MoonshotKimiaForCausalLM": |
| 732 | self.tokenizer_mode = "kimi_audio" |
| 733 | elif arch == "KimiK3ForConditionalGeneration": |
| 734 | self.tokenizer_mode = "kimi_k3" |
| 735 | elif arch == "DeepseekV32ForCausalLM": |
| 736 | self.tokenizer_mode = "deepseek_v32" |
| 737 | elif arch in ( |
| 738 | "DeepseekV4ForCausalLM", |
| 739 | "DeepseekV4ForConditionalGeneration", |
| 740 | ): |
| 741 | self.tokenizer_mode = "deepseek_v4" |
| 742 | elif arch == "DeepseekV41ForCausalLM": |
| 743 | self.tokenizer_mode = "deepseek_v41" |
| 744 | elif arch in ("InklingForCausalLM", "InklingForConditionalGeneration"): |
| 745 | self.tokenizer_mode = "inkling" |
| 746 | |
| 747 | if self.tokenizer_mode != "auto": |
| 748 | logger.info( |
| 749 | "Defaulting to tokenizer_mode=%r for %s", |
| 750 | self.tokenizer_mode, |
| 751 | arch, |
| 752 | ) |
| 753 | |
| 754 | # Init pooler config if needed |
| 755 | if self.runner_type == "pooling": |
| 756 | if self.pooler_config is None: |
| 757 | self.pooler_config = PoolerConfig() |
| 758 | pooler_config_sources: dict[str, str] = {} |
| 759 | else: |
| 760 | pooler_config_sources = { |
| 761 | k: "user" |
| 762 | for k in POOLER_CONFIG_LOG_FIELDS |
| 763 | if getattr(self.pooler_config, k) is not None |
| 764 | } |
| 765 | |
| 766 | base_config = get_pooling_config(self.model, self.revision) |
| 767 | if base_config is not None: |
| 768 | # Only set values that are not overridden by the user |
| 769 | for k, v in base_config.items(): |
| 770 | if getattr(self.pooler_config, k) is None: |
| 771 | setattr(self.pooler_config, k, v) |
| 772 | pooler_config_sources[k] = "sentence_transformers" |
| 773 | |
| 774 | default_seq_pooling_type = self._model_info.default_seq_pooling_type |
| 775 | if self.pooler_config.seq_pooling_type is None: |
| 776 | self.pooler_config.seq_pooling_type = default_seq_pooling_type |
| 777 | pooler_config_sources["seq_pooling_type"] = "model_default" |
| 778 | default_tok_pooling_type = self._model_info.default_tok_pooling_type |
| 779 | if self.pooler_config.tok_pooling_type is None: |
| 780 | self.pooler_config.tok_pooling_type = default_tok_pooling_type |
| 781 | pooler_config_sources["tok_pooling_type"] = "model_default" |
| 782 | pooler_config_sources.setdefault("use_activation", "pooler_default") |
| 783 | self._pooler_config_sources = pooler_config_sources |
| 784 | |
| 785 | self.dtype: torch.dtype = _get_and_verify_dtype( |
| 786 | self.model, |
| 787 | self.hf_config, |
| 788 | self.dtype, |
| 789 | is_pooling_model=self.runner_type == "pooling", |
| 790 | revision=self.revision, |
| 791 | config_format=self.config_format, |
| 792 | ) |
| 793 | |
| 794 | # Some checkpoints set sliding_window to 0 to indicate that sliding window is |
| 795 | # disabled, but vLLM uses None for that. Convert 0 to None to avoid errors. |
| 796 | # Set before get_and_verify_max_len to ensure that max_model_len does not get |
| 797 | # capped to 0. |
| 798 | if self.get_sliding_window() == 0: |
| 799 | self.disable_sliding_window = True |
| 800 | self.hf_text_config.sliding_window = None |
| 801 | |
| 802 | self.original_max_model_len = self.max_model_len |
| 803 | self.max_model_len = self.get_and_verify_max_len(self.max_model_len) |
| 804 | |
| 805 | if self.is_encoder_decoder: |
| 806 | mm_processor_cache_gb = 0 |
| 807 | logger.info("Encoder-decoder model detected, disabling mm processor cache.") |
| 808 | |
| 809 | # Init multimodal config if needed |
| 810 | if self._model_info.supports_multimodal: |
| 811 | self.hf_image_processor_config = get_hf_image_processor_config( |
| 812 | self.model, hf_token=self.hf_token, revision=self.revision |
| 813 | ) |
| 814 | if ( |
| 815 | mm_encoder_tp_mode == "data" |
| 816 | and not self._model_info.supports_multimodal_encoder_tp_data |
| 817 | ): |
| 818 | logger.warning_once( |
| 819 | "This model does not support `--mm-encoder-tp-mode data`. " |
| 820 | "Falling back to `--mm-encoder-tp-mode weights`." |
| 821 | ) |
| 822 | mm_encoder_tp_mode = "weights" |
| 823 | |
| 824 | mm_processor_kwargs = MultiModalConfig.fold_mm_processor_device( |
| 825 | mm_processor_kwargs, mm_processor_device |
| 826 | ) |
| 827 | |
| 828 | mm_config_kwargs = dict( |
| 829 | language_model_only=language_model_only, |
| 830 | limit_per_prompt=limit_mm_per_prompt, |
| 831 | enable_mm_embeds=enable_mm_embeds, |
| 832 | media_io_kwargs=media_io_kwargs, |
| 833 | mm_processor_kwargs=mm_processor_kwargs, |
| 834 | mm_processor_cache_gb=mm_processor_cache_gb, |
| 835 | mm_processor_cache_type=mm_processor_cache_type, |
| 836 | mm_hasher_algorithm=mm_hasher_algorithm, |
| 837 | mm_shm_cache_max_object_size_mb=mm_shm_cache_max_object_size_mb, |
| 838 | mm_encoder_only=mm_encoder_only, |
| 839 | mm_encoder_tp_mode=mm_encoder_tp_mode, |
| 840 | mm_encoder_attn_backend=mm_encoder_attn_backend, |
| 841 | mm_encoder_attn_dtype=mm_encoder_attn_dtype, |
| 842 | mm_encoder_fp8_scale_path=mm_encoder_fp8_scale_path, |
| 843 | mm_encoder_fp8_scale_save_path=mm_encoder_fp8_scale_save_path, |
| 844 | mm_encoder_fp8_scale_save_margin=mm_encoder_fp8_scale_save_margin, |
| 845 | interleave_mm_strings=interleave_mm_strings, |
| 846 | skip_mm_profiling=skip_mm_profiling, |
| 847 | video_pruning_rate=video_pruning_rate, |
| 848 | video_pruning_method=video_pruning_method, |
| 849 | mm_tensor_ipc=mm_tensor_ipc, |
| 850 | mm_ipc_gpu_memory_gb=mm_ipc_gpu_memory_gb, |
| 851 | mm_device_do_normalize=self._resolve_mm_device_do_normalize( |
| 852 | mm_device_do_normalize |
| 853 | ), |
| 854 | ) |
| 855 | |
| 856 | mm_config_kwargs = { |
| 857 | k: v for k, v in mm_config_kwargs.items() if v is not None |
| 858 | } |
| 859 | |
| 860 | self.multimodal_config = MultiModalConfig(**mm_config_kwargs) # type: ignore[arg-type] |
| 861 | |
| 862 | pruning_spec = self.multimodal_config.get_video_pruning_spec() |
| 863 | supported_pruning = self._model_info.supported_video_pruning_methods |
| 864 | if ( |
| 865 | pruning_spec is not None |
| 866 | and supported_pruning |
| 867 | and pruning_spec[0] not in supported_pruning |
| 868 | ): |
| 869 | raise ValueError( |
| 870 | f"Video pruning method '{pruning_spec[0]}' is not " |
| 871 | f"supported by {self._model_info.architecture} " |
| 872 | f"(supported methods: {supported_pruning})." |
| 873 | ) |
| 874 | |
| 875 | if ( |
| 876 | self.renderer_num_workers > 1 |
| 877 | and self.multimodal_config.mm_processor_cache_gb > 0 |
| 878 | and self.runner_type == "pooling" |
| 879 | ): |
| 880 | raise ValueError( |
| 881 | "Cannot use --renderer-num-workers > 1 with the " |
| 882 | "multimodal processor cache enabled for pooling models. " |
| 883 | "Pooling preprocessing runs on the renderer workers, and " |
| 884 | "the cache is not thread-safe. Please set " |
| 885 | "--renderer-num-workers 1 (the default), or " |
| 886 | "disable the cache with --mm-processor-cache-gb 0." |
| 887 | ) |
| 888 | |
| 889 | # Rebuild after multimodal_config exists so text-only mm_prefix |
| 890 | # clearing is applied (and cached for later with_hf_config calls). |
| 891 | self.model_arch_config = self.get_model_arch_config() |
| 892 | |
| 893 | if self.disable_sliding_window: |
| 894 | # Set after get_and_verify_max_len to ensure that max_model_len |
| 895 | # can be correctly capped to sliding window size |
| 896 | self.hf_text_config.sliding_window = None |
| 897 | |
| 898 | # Avoid running try_verify_and_update_config multiple times |
| 899 | self.config_updated = False |
| 900 | self._try_verify_and_update_model_config() |
| 901 | self._verify_quantization() |
| 902 | self._verify_cuda_graph() |
| 903 | |
| 904 | def _supports_multimodal_inputs(self) -> bool: |
| 905 | """Checks if the model supports multimodal inputs. |
| 906 | Returns True if the model is multimodal with any non-zero supported |
| 907 | modalities, otherwise returns False, effectively running in |
| 908 | text-only mode. |
| 909 | """ |
| 910 | if not self.is_multimodal_model: |
| 911 | return False |
| 912 | |
| 913 | from vllm.multimodal import MULTIMODAL_REGISTRY |
| 914 | |
| 915 | mm_config = self.get_multimodal_config() |
| 916 | try: |
| 917 | info = MULTIMODAL_REGISTRY.get_processing_info(self) |
| 918 | except ValueError: |
| 919 | # Speculative drafters for multimodal targets (e.g. Qwen3_5MTP, |
| 920 | # Exaone4_5_MTP, MiMoV2OmniMTP) declare `SupportsMultiModal` so |
| 921 | # that they can consume the embeddings merged by the target model, |
| 922 | # but they never run a multi-modal processor of their own. Running |
| 923 | # in text-only mode is the expected outcome for them, not a |
| 924 | # misconfiguration worth warning about. |
| 925 | if self.runner_type != "draft": |
| 926 | logger.warning_once( |
| 927 | "Model %s is treated as multimodal but has no registered " |
| 928 | "multimodal processor; running in text-only mode.", |
| 929 | self.model, |
| 930 | ) |
| 931 | return False |
| 932 | |
| 933 | # Check if all supported modalities have limit == 0 |
| 934 | if all( |
| 935 | mm_config.get_limit_per_prompt(modality) == 0 |
| 936 | for modality in info.supported_mm_limits |
| 937 | ): |
| 938 | # If enable_mm_embeds is True, we still need MM infrastructure |
| 939 | # to process pre-computed embeddings even though encoder won't run |
| 940 | if mm_config.enable_mm_embeds: |
| 941 | return True |
| 942 | |
| 943 | logger.info_once( |
| 944 | "All limits of multimodal modalities supported by the model " |
| 945 | "are set to 0, running in text-only mode." |
| 946 | ) |
| 947 | return False |
| 948 | |
| 949 | return True |
| 950 | |
| 951 | def _cached_supports_multimodal_inputs(self) -> bool: |
| 952 | cache = getattr(self, "_supports_multimodal_inputs_cache", None) |
| 953 | if cache is None: |
| 954 | self._supports_multimodal_inputs_cache = cache = {} |
| 955 | |
| 956 | mm_config = self.multimodal_config |
| 957 | if mm_config is None: |
| 958 | mm_cache_key = None |
| 959 | else: |
| 960 | limits_per_prompt = { |
| 961 | modality: mm_config.get_limit_per_prompt(modality) |
| 962 | for modality in mm_config.limit_per_prompt |
| 963 | } |
| 964 | mm_cache_key = ( |
| 965 | mm_config.language_model_only, |
| 966 | tuple(limits_per_prompt.items()), |
| 967 | mm_config.enable_mm_embeds, |
| 968 | ) |
| 969 | |
| 970 | cache_key = (self.is_multimodal_model, self.runner_type, mm_cache_key) |
| 971 | if cache_key in cache: |
| 972 | return cache[cache_key] |
| 973 | |
| 974 | supports_mm = self._supports_multimodal_inputs() |
| 975 | cache[cache_key] = supports_mm |
| 976 | return supports_mm |
| 977 | |
| 978 | def _supports_multimodal_for_mm_prefix(self) -> bool: |
| 979 | """Whether multimodal inputs can still appear for this deployment. |
| 980 | |
| 981 | This runs more than once per config: once early in ``__post_init__`` |
| 982 | (before ``multimodal_config`` exists), again after it is created, and |
| 983 | then for every ``get_model_arch_config`` regeneration -- notably |
| 984 | ``with_hf_config``, which deep-copies this ``ModelConfig`` and swaps |
| 985 | ``hf_config`` for a text-only submodule (e.g. ``Gemma4ForCausalLM``). |
| 986 | |
| 987 | The result is cached for correctness, not just to save work: on the |
| 988 | ``with_hf_config`` copy the submodule architecture has no registered |
| 989 | multimodal processor, so re-querying the registry would raise and be |
| 990 | treated as text-only, wrongly clearing ``is_mm_prefix_lm`` even when a |
| 991 | vision modality is still enabled (e.g. ``image=0`` but video allowed). |
| 992 | The deep-copied cache preserves the top-level decision instead. |
| 993 | """ |
| 994 | if self.multimodal_config is None: |
| 995 | # Early call before multimodal init — do not clear mm_prefix yet. |
| 996 | return True |
| 997 | |
| 998 | supports_mm = self._cached_supports_multimodal_inputs() |
| 999 | if not supports_mm: |
| 1000 | logger.info_once( |
| 1001 | "Disabled mm_prefix attention mode because multimodal inputs " |
| 1002 | "are configuration-disabled. Attention backends without " |
| 1003 | "mm_prefix support may now be selected." |
| 1004 | ) |
| 1005 | |
| 1006 | return supports_mm |
| 1007 | |
| 1008 | def get_model_arch_config(self) -> ModelArchitectureConfig: |
| 1009 | convertor_cls = MODEL_ARCH_CONFIG_CONVERTORS.get( |
| 1010 | self.hf_config.model_type, ModelArchConfigConvertorBase |
| 1011 | ) |
| 1012 | convertor = convertor_cls( |
| 1013 | self.hf_config, |
| 1014 | self.hf_text_config, |
| 1015 | revision=getattr(self, "revision", None), |
| 1016 | ) |
| 1017 | return convertor.convert( |
| 1018 | supports_multimodal=self._supports_multimodal_for_mm_prefix() |
| 1019 | ) |
| 1020 | |
| 1021 | @field_validator("tokenizer", "max_model_len", mode="wrap") |
| 1022 | @classmethod |
| 1023 | def _skip_none_validation(cls, value: Any, handler: Callable) -> Any: |
| 1024 | """Skip validation if the value is `None` when initialisation is delayed.""" |
| 1025 | if value is None: |
| 1026 | return value |
| 1027 | return handler(value) |
| 1028 | |
| 1029 | @field_validator("tokenizer_mode", mode="after") |
| 1030 | def _lowercase_tokenizer_mode(cls, tokenizer_mode: str) -> str: |
| 1031 | return tokenizer_mode.lower() |
| 1032 | |
| 1033 | @field_validator("quantization", mode="before") |
| 1034 | @classmethod |
| 1035 | def validate_quantization_before(cls, value: Any) -> Any: |
| 1036 | if isinstance(value, str): |
| 1037 | return value.lower() |
| 1038 | return value |
| 1039 | |
| 1040 | @model_validator(mode="after") |
| 1041 | def validate_model_config_after(self: "ModelConfig") -> "ModelConfig": |
| 1042 | """Called after __post_init__.""" |
| 1043 | if not isinstance(self.tokenizer, str): |
| 1044 | raise ValueError( |
| 1045 | f"tokenizer must be a string, got " |
| 1046 | f"{type(self.tokenizer).__name__}: {self.tokenizer!r}. " |
| 1047 | "Please provide a valid tokenizer path or HuggingFace model ID." |
| 1048 | ) |
| 1049 | if not isinstance(self.max_model_len, int) or self.max_model_len < 1: |
| 1050 | raise ValueError( |
| 1051 | f"max_model_len must be a positive integer, " |
| 1052 | f"got {type(self.max_model_len).__name__}: {self.max_model_len!r}. " |
| 1053 | "Example: max_model_len=2048" |
| 1054 | ) |
| 1055 | if self.enable_prompt_embeds and self.is_encoder_decoder: |
| 1056 | # No encoder-decoder model accepts `inputs_embeds`; their decoders |
| 1057 | # embed `input_ids` internally. |
| 1058 | raise ValueError( |
| 1059 | "--enable-prompt-embeds is not supported with encoder-decoder models." |
| 1060 | ) |
| 1061 | return self |
| 1062 | |
| 1063 | def _resolve_mm_device_do_normalize( |
| 1064 | self, mm_device_do_normalize: bool | None |
| 1065 | ) -> bool: |
| 1066 | if mm_device_do_normalize is None: |
| 1067 | if envs.VLLM_USE_RUST_FRONTEND: |
| 1068 | logger.debug( |
| 1069 | "VLLM_USE_RUST_FRONTEND is set. " |
| 1070 | "Rust frontend does not currently support mm_device_do_normalize, " |
| 1071 | "forcing mm_device_do_normalize = False." |
| 1072 | ) |
| 1073 | mm_device_do_normalize = False |
| 1074 | else: |
| 1075 | mm_device_do_normalize = ( |
| 1076 | self._model_info.supports_mm_device_do_normalize |
| 1077 | ) |
| 1078 | logger.debug( |
| 1079 | "mm_device_do_normalize is %s by default.", |
| 1080 | "enabled" if mm_device_do_normalize else "disabled", |
| 1081 | ) |
| 1082 | else: |
| 1083 | if mm_device_do_normalize and envs.VLLM_USE_RUST_FRONTEND: |
| 1084 | logger.warning( |
| 1085 | "VLLM_USE_RUST_FRONTEND is set. " |
| 1086 | "Rust frontend does not currently support mm_device_do_normalize, " |
| 1087 | "forcing mm_device_do_normalize = False." |
| 1088 | ) |
| 1089 | mm_device_do_normalize = False |
| 1090 | |
| 1091 | if ( |
| 1092 | mm_device_do_normalize |
| 1093 | and not self._model_info.supports_mm_device_do_normalize |
| 1094 | ): |
| 1095 | logger.warning( |
| 1096 | "Model does not support mm_device_do_normalize, " |
| 1097 | "forcing mm_device_do_normalize = False." |
| 1098 | ) |
| 1099 | mm_device_do_normalize = False |
| 1100 | |
| 1101 | logger.debug( |
| 1102 | "mm_device_do_normalize is %s.", |
| 1103 | "enabled" if mm_device_do_normalize else "disabled", |
| 1104 | ) |
| 1105 | |
| 1106 | return mm_device_do_normalize |
| 1107 | |
| 1108 | def _get_transformers_backend_cls(self) -> str: |
| 1109 | """Determine which Transformers modeling backend class will be used if |
| 1110 | `model_impl` is set to `transformers` or `auto`.""" |
| 1111 | cls = "Transformers" |
| 1112 | # If 'hf_config is not hf_text_config' it's a nested config, i.e. multimodal |
| 1113 | cls += "MultiModal" if self.hf_config is not self.hf_text_config else "" |
| 1114 | cls += "MoE" if self.is_moe else "" |
| 1115 | # Check if the architecture we're wrapping has defaults |
| 1116 | runner = None |
| 1117 | task = None |
| 1118 | # architectures is empty for with_hf_config() submodel views. |
| 1119 | if self.architectures and ( |
| 1120 | defaults := try_match_architecture_defaults(self.architectures[0]) |
| 1121 | ): |
| 1122 | _, (runner, task) = defaults |
| 1123 | # User specified value take precedence |
| 1124 | if self.runner != "auto": |
| 1125 | runner = self.runner |
| 1126 | # Only consider Transformers modeling backend pooling classes if we're wrapping |
| 1127 | # an architecture that defaults to pooling. Otherwise, we return the LM class |
| 1128 | # and use adapters. |
| 1129 | if runner == "pooling" and task in {"embed", "classify"}: |
| 1130 | if task == "embed": |
| 1131 | cls += "EmbeddingModel" |
| 1132 | elif task == "classify": |
| 1133 | cls += "ForSequenceClassification" |
| 1134 | else: |
| 1135 | cls += "ForCausalLM" |
| 1136 | return cls |
| 1137 | |
| 1138 | def using_transformers_backend(self) -> bool: |
| 1139 | """Check if the model is using the Transformers modeling backend class.""" |
| 1140 | used_cls = self._model_info.architecture |
| 1141 | transformers_backend_cls = self._get_transformers_backend_cls() |
| 1142 | return used_cls == transformers_backend_cls |
| 1143 | |
| 1144 | @property |
| 1145 | def registry(self): |
| 1146 | self._maybe_register_model_class_overrides() |
| 1147 | return me_models.ModelRegistry |
| 1148 | |
| 1149 | def _maybe_register_model_class_overrides(self) -> None: |
| 1150 | # Apply ``model_class_overrides`` here because this property is the |
| 1151 | # single chokepoint through which every model-class inspect/resolve |
| 1152 | # passes, in both the engine front-end and every worker process. The |
| 1153 | # guard is process-local (see ``_REGISTERED_MODEL_CLASS_OVERRIDES``), so |
| 1154 | # each worker re-registers into its own ModelRegistry exactly once |
| 1155 | # rather than trusting a pickled-in instance flag. |
| 1156 | if not self.model_class_overrides: |
| 1157 | return |
| 1158 | pending = [ |
| 1159 | (arch, target) |
| 1160 | for arch, target in self.model_class_overrides.items() |
| 1161 | if (arch, target) not in _REGISTERED_MODEL_CLASS_OVERRIDES |
| 1162 | ] |
| 1163 | if not pending: |
| 1164 | return |
| 1165 | logger.warning_once( |
| 1166 | "Applying model_class_overrides %s. This is intended for " |
| 1167 | "development/debugging.", |
| 1168 | str(self.model_class_overrides), |
| 1169 | ) |
| 1170 | for arch, target in pending: |
| 1171 | me_models.ModelRegistry.register_model(arch, target) |
| 1172 | _REGISTERED_MODEL_CLASS_OVERRIDES.add((arch, target)) |
| 1173 | |
| 1174 | @property |
| 1175 | def architectures(self) -> list[str]: |
| 1176 | return self.model_arch_config.architectures |
| 1177 | |
| 1178 | @property |
| 1179 | def architecture(self) -> str: |
| 1180 | """The architecture vllm actually used.""" |
| 1181 | return self._architecture |
| 1182 | |
| 1183 | def maybe_pull_model_tokenizer_for_runai(self, model: str, tokenizer: str) -> None: |
| 1184 | """Pull model/tokenizer from Object Storage to temporary |
| 1185 | directory when needed. |
| 1186 | |
| 1187 | Args: |
| 1188 | model: Model name or path |
| 1189 | tokenizer: Tokenizer name or path |
| 1190 | |
| 1191 | """ |
| 1192 | # Skip if model_weights is already set (model already pulled) |
| 1193 | if self.model_weights: |
| 1194 | return |
| 1195 | |
| 1196 | if not (is_runai_obj_uri(model) or is_runai_obj_uri(tokenizer)): |
| 1197 | return |
| 1198 | |
| 1199 | if is_runai_obj_uri(model): |
| 1200 | object_storage_model = ObjectStorageModel(url=model) |
| 1201 | object_storage_model.pull_files( |
| 1202 | model, allow_pattern=["*.model", "*.py", "*.json"] |
| 1203 | ) |
| 1204 | self.model_weights = model |
| 1205 | self.model = object_storage_model.dir |
| 1206 | |
| 1207 | # If tokenizer is same as model, download to same directory |
| 1208 | if model == tokenizer: |
| 1209 | object_storage_model.pull_files( |
| 1210 | model, |
| 1211 | ignore_pattern=[ |
| 1212 | "*.pt", |
| 1213 | "*.safetensors", |
| 1214 | "*.bin", |
| 1215 | "*.tensors", |
| 1216 | "*.pth", |
| 1217 | ], |
| 1218 | ) |
| 1219 | self.tokenizer = object_storage_model.dir |
| 1220 | return |
| 1221 | |
| 1222 | # Only download tokenizer if needed and not already handled |
| 1223 | if is_runai_obj_uri(tokenizer): |
| 1224 | object_storage_tokenizer = ObjectStorageModel(url=tokenizer) |
| 1225 | object_storage_tokenizer.pull_files( |
| 1226 | tokenizer, |
| 1227 | ignore_pattern=["*.pt", "*.safetensors", "*.bin", "*.tensors", "*.pth"], |
| 1228 | ) |
| 1229 | self.tokenizer = object_storage_tokenizer.dir |
| 1230 | |
| 1231 | def _get_encoder_config(self) -> dict[str, Any] | None: |
| 1232 | return get_sentence_transformer_tokenizer_config(self.model, self.revision) |
| 1233 | |
| 1234 | def maybe_untie_word_embeddings(self) -> None: |
| 1235 | """Stop trusting `tie_word_embeddings` when the checkpoint disagrees. |
| 1236 | |
| 1237 | A config may claim the word embeddings are tied while the checkpoint |
| 1238 | ships an `lm_head` of its own. Tying regardless would silently discard |
| 1239 | that tensor, so build the `lm_head` as if untied and let it load. The |
| 1240 | two are compared once loaded, and re-tied if they turn out to match, by |
| 1241 | [maybe_retie_word_embeddings][vllm.model_executor.model_loader.weight_tying.maybe_retie_word_embeddings]. |
| 1242 | |
| 1243 | Transformers makes the same decision in `PreTrainedModel.tie_weights`, |
| 1244 | where it can compare the two tensors directly. |
| 1245 | """ |
| 1246 | if not getattr(self.hf_config, "tie_word_embeddings", False): |
| 1247 | return |
| 1248 | if not checkpoint_has_lm_head(self.model, revision=self.revision): |
| 1249 | return |
| 1250 | |
| 1251 | logger.debug( |
| 1252 | "The config for %s says the word embeddings are tied, but the checkpoint " |
| 1253 | "contains an lm_head. Loading it to find out whether they really are tied.", |
| 1254 | self.model, |
| 1255 | ) |
| 1256 | # Both levels must be updated: `VllmConfig.with_hf_config` reads the top |
| 1257 | # level, and the text config is what the language model itself sees. |
| 1258 | self.hf_config.tie_word_embeddings = False |
| 1259 | self.hf_config.get_text_config().tie_word_embeddings = False |
| 1260 | self.word_embeddings_untied_by_checkpoint = True |
| 1261 | |
| 1262 | def _get_default_runner_type( |
| 1263 | self, |
| 1264 | architectures: list[str], |
| 1265 | ) -> RunnerType: |
| 1266 | registry = self.registry |
| 1267 | |
| 1268 | # Some Sentence Transformers models use *ForCausalLM archs |
| 1269 | if get_pooling_config(self.model, self.revision): |
| 1270 | return "pooling" |
| 1271 | |
| 1272 | for arch in architectures: |
| 1273 | if arch in registry.get_supported_archs(): |
| 1274 | if registry.is_pooling_model(architectures, self): |
| 1275 | return "pooling" |
| 1276 | if registry.is_text_generation_model(architectures, self): |
| 1277 | return "generate" |
| 1278 | |
| 1279 | match = try_match_architecture_defaults(arch) |
| 1280 | if match: |
| 1281 | _, (runner_type, _) = match |
| 1282 | return runner_type |
| 1283 | |
| 1284 | return "generate" |
| 1285 | |
| 1286 | def _get_runner_type( |
| 1287 | self, |
| 1288 | architectures: list[str], |
| 1289 | runner: RunnerOption, |
| 1290 | convert: ConvertOption, |
| 1291 | ) -> RunnerType: |
| 1292 | if runner != "auto": |
| 1293 | return runner |
| 1294 | |
| 1295 | if convert in {"auto", "none"}: |
| 1296 | runner_type = self._get_default_runner_type(architectures) |
| 1297 | else: |
| 1298 | runner_type = "pooling" |
| 1299 | |
| 1300 | # Don't log the most common case |
| 1301 | if runner_type != "generate": |
| 1302 | logger.info( |
| 1303 | "Resolved `--runner auto` to `--runner %s`. " |
| 1304 | "Pass the value explicitly to silence this message.", |
| 1305 | runner_type, |
| 1306 | ) |
| 1307 | |
| 1308 | return runner_type |
| 1309 | |
| 1310 | def _get_default_convert_type( |
| 1311 | self, |
| 1312 | architectures: list[str], |
| 1313 | runner_type: RunnerType, |
| 1314 | ) -> ConvertType: |
| 1315 | registry = self.registry |
| 1316 | |
| 1317 | for arch in architectures: |
| 1318 | if arch in registry.get_supported_archs(): |
| 1319 | if runner_type == "generate" and registry.is_text_generation_model( |
| 1320 | architectures, self |
| 1321 | ): |
| 1322 | return "none" |
| 1323 | if runner_type == "pooling" and registry.is_pooling_model( |
| 1324 | architectures, self |
| 1325 | ): |
| 1326 | return "none" |
| 1327 | |
| 1328 | match = try_match_architecture_defaults(arch, runner_type=runner_type) |
| 1329 | if match: |
| 1330 | _, (_, convert_type) = match |
| 1331 | return convert_type |
| 1332 | |
| 1333 | # This is to handle Sentence Transformers models that use *ForCausalLM |
| 1334 | # and also multi-modal pooling models which are not defined as |
| 1335 | # Sentence Transformers models |
| 1336 | if runner_type == "pooling": |
| 1337 | return "embed" |
| 1338 | |
| 1339 | return "none" |
| 1340 | |
| 1341 | def _get_convert_type( |
| 1342 | self, |
| 1343 | architectures: list[str], |
| 1344 | runner_type: RunnerType, |
| 1345 | convert: ConvertOption, |
| 1346 | ) -> ConvertType: |
| 1347 | if convert != "auto": |
| 1348 | return convert |
| 1349 | |
| 1350 | convert_type = self._get_default_convert_type(architectures, runner_type) |
| 1351 | |
| 1352 | # Don't log the most common case |
| 1353 | if convert_type != "none": |
| 1354 | logger.info( |
| 1355 | "Resolved `--convert auto` to `--convert %s`. " |
| 1356 | "Pass the value explicitly to silence this message.", |
| 1357 | convert_type, |
| 1358 | ) |
| 1359 | |
| 1360 | return convert_type |
| 1361 | |
| 1362 | def _verify_quantization(self) -> None: |
| 1363 | supported_quantization = me_quant.QUANTIZATION_METHODS |
| 1364 | if self.quantization is not None: |
| 1365 | self.quantization = cast(me_quant.QuantizationMethods, self.quantization) |
| 1366 | |
| 1367 | # Parse quantization method from the HF model config, if available. |
| 1368 | quant_cfg = self.model_arch_config.quantization_config |
| 1369 | |
| 1370 | if quant_cfg is not None: |
| 1371 | quant_method = quant_cfg["quant_method"] |
| 1372 | # Quantization methods which are overrides (i.e. they have a |
| 1373 | # `override_quantization_method` method) must be checked in order |
| 1374 | # of preference (this is particularly important for GPTQ). |
| 1375 | overrides = [ |
| 1376 | "auto_gptq", |
| 1377 | "gptq", |
| 1378 | "gptq_marlin", |
| 1379 | "auto_awq", |
| 1380 | "awq", |
| 1381 | "awq_marlin", |
| 1382 | "inc", |
| 1383 | "moe_wna16", |
| 1384 | "modelopt", |
| 1385 | "modelopt_fp4", |
| 1386 | "modelopt_mxfp8", |
| 1387 | "mxfp8", |
| 1388 | "modelopt_mixed", |
| 1389 | # Ensure heavy backends are probed last to avoid unnecessary |
| 1390 | # imports during override detection (e.g., MXFP4 imports Triton) |
| 1391 | "mxfp4", |
| 1392 | "gpt_oss_mxfp4", |
| 1393 | "deepseek_v4_fp8", |
| 1394 | "humming", |
| 1395 | ] |
| 1396 | # if the user specifies humming, we should always use humming |
| 1397 | if self.quantization == "humming": |
| 1398 | overrides = ["humming"] + overrides |
| 1399 | quantization_methods = [ |
| 1400 | q for q in supported_quantization if q not in overrides |
| 1401 | ] |
| 1402 | # Any custom overrides will be in quantization_methods so we place |
| 1403 | # them at the start of the list so custom overrides have preference |
| 1404 | # over the built-in ones. |
| 1405 | quantization_methods = quantization_methods + overrides |
| 1406 | |
| 1407 | # Detect which checkpoint is it |
| 1408 | for name in quantization_methods: |
| 1409 | method = me_quant.get_quantization_config(name) |
| 1410 | quantization_override = method.override_quantization_method( |
| 1411 | quant_cfg, self.quantization, hf_config=self.hf_config |
| 1412 | ) |
| 1413 | if quantization_override is not None: |
| 1414 | # Raise error if the override is not custom (custom would |
| 1415 | # be in QUANTIZATION_METHODS but not QuantizationMethods) |
| 1416 | # and hasn't been added to the overrides list. |
| 1417 | if ( |
| 1418 | name in get_args(me_quant.QuantizationMethods) |
| 1419 | and name not in overrides |
| 1420 | ): |
| 1421 | raise ValueError( |
| 1422 | f"Quantization method {name} is an override but " |
| 1423 | "is has not been added to the `overrides` list " |
| 1424 | "above. This is necessary to ensure that the " |
| 1425 | "overrides are checked in order of preference." |
| 1426 | ) |
| 1427 | quant_method = quantization_override |
| 1428 | self.quantization = quantization_override |
| 1429 | break |
| 1430 | |
| 1431 | quant_method = quant_method if quant_method != "" else None |
| 1432 | # Verify quantization configurations. |
| 1433 | if self.quantization is None: |
| 1434 | self.quantization = quant_method |
| 1435 | elif self.quantization != quant_method: |
| 1436 | raise ValueError( |
| 1437 | "Quantization method specified in the model config " |
| 1438 | f"({quant_method}) does not match the quantization " |
| 1439 | f"method specified in the `quantization` argument " |
| 1440 | f"({self.quantization})." |
| 1441 | ) |
| 1442 | |
| 1443 | if self.quantization is not None: |
| 1444 | if self.quantization not in supported_quantization: |
| 1445 | raise ValueError( |
| 1446 | f"Unknown quantization method: {self.quantization}. Must " |
| 1447 | f"be one of {supported_quantization}." |
| 1448 | ) |
| 1449 | current_platform.verify_quantization(self.quantization) |
| 1450 | |
| 1451 | if self.quantization in me_quant.DEPRECATED_QUANTIZATION_METHODS: |
| 1452 | if self.allow_deprecated_quantization: |
| 1453 | logger.warning( |
| 1454 | "The quantization method %s is deprecated " |
| 1455 | "and will be removed in future versions of vLLM.", |
| 1456 | self.quantization, |
| 1457 | ) |
| 1458 | else: |
| 1459 | raise ValueError( |
| 1460 | "The quantization method %s is deprecated " |
| 1461 | "and will be removed in future versions of vLLM. To bypass, " |
| 1462 | "set `--allow-deprecated-quantization`.", |
| 1463 | self.quantization, |
| 1464 | ) |
| 1465 | |
| 1466 | def _verify_cuda_graph(self) -> None: |
| 1467 | # CUDAGraph capture not supported for encoder-decoder models on ROCm |
| 1468 | unsupported_rocm = self.is_encoder_decoder |
| 1469 | if unsupported_rocm and not self.enforce_eager and current_platform.is_rocm(): |
| 1470 | logger.warning( |
| 1471 | "CUDA graph is not supported for %s on ROCm yet, fallback " |
| 1472 | "to eager mode.", |
| 1473 | self.model_arch_config.model_type, |
| 1474 | ) |
| 1475 | self.enforce_eager = True |
| 1476 | |
| 1477 | def _verify_with_expert_parallelism(self) -> None: |
| 1478 | if not self.is_moe: |
| 1479 | raise ValueError( |
| 1480 | "Number of experts in the model must be greater than 0 " |
| 1481 | "when expert parallelism is enabled." |
| 1482 | ) |
| 1483 | |
| 1484 | def _try_verify_and_update_model_config(self): |
| 1485 | # Avoid running try_verify_and_update_config multiple times |
| 1486 | if getattr(self, "config_updated", False): |
| 1487 | return |
| 1488 | |
| 1489 | architecture = self.architecture |
| 1490 | if architecture is None: |
| 1491 | return |
| 1492 | |
| 1493 | from vllm.model_executor.models.config import ( |
| 1494 | MODELS_CONFIG_MAP, |
| 1495 | ) |
| 1496 | |
| 1497 | cls = MODELS_CONFIG_MAP.get(architecture, None) |
| 1498 | if cls is not None: |
| 1499 | cls.verify_and_update_model_config(self) |
| 1500 | |
| 1501 | def verify_dual_chunk_attention_config( |
| 1502 | self, |
| 1503 | load_config: LoadConfig, |
| 1504 | ) -> None: |
| 1505 | if hasattr(self.hf_config, "dual_chunk_attention_config"): |
| 1506 | # Try loading the sparse attention config |
| 1507 | from vllm.model_executor.model_loader.weight_utils import ( |
| 1508 | get_sparse_attention_config, |
| 1509 | ) |
| 1510 | |
| 1511 | sparse_attn_config = get_sparse_attention_config(self, load_config) |
| 1512 | if sparse_attn_config: |
| 1513 | self.hf_config.dual_chunk_attention_config[ |
| 1514 | "sparse_attention_config" |
| 1515 | ] = sparse_attn_config |
| 1516 | if ( |
| 1517 | "sparse_attention_enabled" |
| 1518 | not in self.hf_config.dual_chunk_attention_config |
| 1519 | ): |
| 1520 | self.hf_config.dual_chunk_attention_config[ |
| 1521 | "sparse_attention_enabled" |
| 1522 | ] = True |
| 1523 | |
| 1524 | def verify_with_parallel_config( |
| 1525 | self, |
| 1526 | parallel_config: ParallelConfig, |
| 1527 | ) -> None: |
| 1528 | total_num_attention_heads = self.model_arch_config.total_num_attention_heads |
| 1529 | tensor_parallel_size = parallel_config.tensor_parallel_size |
| 1530 | if total_num_attention_heads % tensor_parallel_size != 0: |
| 1531 | raise ValueError( |
| 1532 | f"Total number of attention heads ({total_num_attention_heads})" |
| 1533 | " must be divisible by tensor parallel size " |
| 1534 | f"({tensor_parallel_size})." |
| 1535 | ) |
| 1536 | |
| 1537 | if parallel_config.enable_expert_parallel: |
| 1538 | self._verify_with_expert_parallelism() |
| 1539 | |
| 1540 | pipeline_parallel_size = parallel_config.pipeline_parallel_size |
| 1541 | if pipeline_parallel_size > 1 and not self.registry.is_pp_supported_model( |
| 1542 | self.architectures, self |
| 1543 | ): |
| 1544 | raise NotImplementedError( |
| 1545 | "Pipeline parallelism is not supported for this model. " |
| 1546 | "Supported models implement the `SupportsPP` interface." |
| 1547 | ) |
| 1548 | |
| 1549 | decode_context_parallel_size = parallel_config.decode_context_parallel_size |
| 1550 | if decode_context_parallel_size > 1 and not self.use_mla: |
| 1551 | total_num_kv_heads = self.get_total_num_kv_heads() |
| 1552 | if tensor_parallel_size <= total_num_kv_heads: |
| 1553 | raise ValueError( |
| 1554 | "Decode context parallelism for GQA/MQA requires " |
| 1555 | f"`--tensor-parallel-size` ({tensor_parallel_size}) to be " |
| 1556 | "greater than the model's total number of KV heads " |
| 1557 | f"({total_num_kv_heads}). Increase `--tensor-parallel-size` " |
| 1558 | "or set `--decode-context-parallel-size 1`." |
| 1559 | ) |
| 1560 | |
| 1561 | max_dcp_size = tensor_parallel_size // total_num_kv_heads |
| 1562 | if decode_context_parallel_size > max_dcp_size: |
| 1563 | raise ValueError( |
| 1564 | "`--decode-context-parallel-size` " |
| 1565 | f"({decode_context_parallel_size}) exceeds the maximum " |
| 1566 | f"supported value ({max_dcp_size}) for " |
| 1567 | f"`--tensor-parallel-size` ({tensor_parallel_size}) and " |
| 1568 | f"{total_num_kv_heads} model KV heads." |
| 1569 | ) |
| 1570 | |
| 1571 | num_q_per_kv = total_num_attention_heads // total_num_kv_heads |
| 1572 | if num_q_per_kv % decode_context_parallel_size != 0: |
| 1573 | raise ValueError( |
| 1574 | "The model's number of query heads per KV head " |
| 1575 | f"({num_q_per_kv}) must be divisible by " |
| 1576 | "`--decode-context-parallel-size` " |
| 1577 | f"({decode_context_parallel_size}) for GQA/MQA." |
| 1578 | ) |
| 1579 | |
| 1580 | # torch_shm uses a single IPC queue to rank 0; DP>1 is |
| 1581 | # incompatible because API servers can't know which |
| 1582 | # CoreEngine the scheduler will assign work to. TP>1 is |
| 1583 | # also not supported because this requires broadcasting |
| 1584 | # MM tensors between all TP ranks. |
| 1585 | if ( |
| 1586 | self.multimodal_config is not None |
| 1587 | and self.multimodal_config.mm_tensor_ipc == "torch_shm" |
| 1588 | and parallel_config.world_size_across_dp > 1 |
| 1589 | ): |
| 1590 | raise ValueError( |
| 1591 | "mm_tensor_ipc='torch_shm' is not supported with " |
| 1592 | "data_parallel_size > 1 or tensor_parallel_size > 1 " |
| 1593 | "or pipeline_parallel_size > 1." |
| 1594 | ) |
| 1595 | |
| 1596 | def get_sliding_window(self) -> int | None: |
| 1597 | """Get the sliding window size from the HF text config if present.""" |
| 1598 | return getattr(self.hf_text_config, "sliding_window", None) |
| 1599 | |
| 1600 | def get_vocab_size(self) -> int: |
| 1601 | return self.model_arch_config.vocab_size |
| 1602 | |
| 1603 | def get_hidden_size(self) -> int: |
| 1604 | return self.model_arch_config.hidden_size |
| 1605 | |
| 1606 | def get_inputs_embeds_size(self) -> int: |
| 1607 | # The size of inputs_embeds is usually identical to the size |
| 1608 | # of the hidden states, however there are exceptions, such as |
| 1609 | # embedding models like CLIP and SigLIP |
| 1610 | names = ("projection_dim", "projection_size") |
| 1611 | return getattr_iter( |
| 1612 | self.hf_text_config, names, default_factory=self.get_hidden_size |
| 1613 | ) |
| 1614 | |
| 1615 | @property |
| 1616 | def is_deepseek_mla(self) -> bool: |
| 1617 | return self.model_arch_config.is_deepseek_mla |
| 1618 | |
| 1619 | @cached_property |
| 1620 | def is_mm_prefix_lm(self) -> bool: |
| 1621 | return self.model_arch_config.is_mm_prefix_lm |
| 1622 | |
| 1623 | @property |
| 1624 | def rswa_window(self) -> int | None: |
| 1625 | return self.model_arch_config.rswa_window |
| 1626 | |
| 1627 | def get_head_size(self) -> int: |
| 1628 | return self.model_arch_config.head_size |
| 1629 | |
| 1630 | def get_total_num_kv_heads(self) -> int: |
| 1631 | """Returns the total number of KV heads.""" |
| 1632 | return self.model_arch_config.total_num_kv_heads |
| 1633 | |
| 1634 | def get_num_kv_heads( |
| 1635 | self, |
| 1636 | parallel_config: ParallelConfig, |
| 1637 | arch_config: ModelArchitectureConfig | None = None, |
| 1638 | ) -> int: |
| 1639 | """Returns the number of KV heads per GPU. |
| 1640 | |
| 1641 | Pass ``arch_config`` (from ``model_arch_config[layer_idx]``) to size a |
| 1642 | single layer of a heterogeneous model rather than the model as a whole. |
| 1643 | """ |
| 1644 | if self.use_mla: |
| 1645 | # When using MLA during decode it becomes MQA |
| 1646 | return 1 |
| 1647 | |
| 1648 | arch_config = arch_config or self.model_arch_config |
| 1649 | total_num_kv_heads = arch_config.total_num_kv_heads |
| 1650 | # If tensor parallelism is used, we divide the number of KV heads by |
| 1651 | # the tensor parallel size. We will replicate the KV heads in the |
| 1652 | # case where the number of KV heads is smaller than the tensor |
| 1653 | # parallel size so each GPU has at least one KV head. |
| 1654 | return max(1, total_num_kv_heads // parallel_config.tensor_parallel_size) |
| 1655 | |
| 1656 | def get_num_attention_heads( |
| 1657 | self, |
| 1658 | parallel_config: ParallelConfig, |
| 1659 | arch_config: ModelArchitectureConfig | None = None, |
| 1660 | ) -> int: |
| 1661 | """Returns the number of attention heads per GPU. |
| 1662 | |
| 1663 | Pass ``arch_config`` (from ``model_arch_config[layer_idx]``) to size a |
| 1664 | single layer of a heterogeneous model rather than the model as a whole. |
| 1665 | """ |
| 1666 | arch_config = arch_config or self.model_arch_config |
| 1667 | num_heads = arch_config.total_num_attention_heads |
| 1668 | return num_heads // parallel_config.tensor_parallel_size |
| 1669 | |
| 1670 | def get_num_experts(self) -> int: |
| 1671 | return self.model_arch_config.num_experts |
| 1672 | |
| 1673 | def get_num_experts_per_tok(self) -> int: |
| 1674 | return self.model_arch_config.num_experts_per_token |
| 1675 | |
| 1676 | def get_total_num_hidden_layers(self) -> int: |
| 1677 | return self.model_arch_config.total_num_hidden_layers |
| 1678 | |
| 1679 | def get_layers_start_end_indices( |
| 1680 | self, parallel_config: ParallelConfig |
| 1681 | ) -> tuple[int, int]: |
| 1682 | from vllm.distributed.utils import get_pp_indices |
| 1683 | |
| 1684 | total_num_hidden_layers = self.get_total_num_hidden_layers() |
| 1685 | |
| 1686 | # the layout order is: DP x PP x TP |
| 1687 | pp_rank = ( |
| 1688 | parallel_config.rank // parallel_config.tensor_parallel_size |
| 1689 | ) % parallel_config.pipeline_parallel_size |
| 1690 | pp_size = parallel_config.pipeline_parallel_size |
| 1691 | start, end = get_pp_indices(total_num_hidden_layers, pp_rank, pp_size) |
| 1692 | return start, end |
| 1693 | |
| 1694 | def get_num_layers(self, parallel_config: ParallelConfig) -> int: |
| 1695 | start, end = self.get_layers_start_end_indices(parallel_config) |
| 1696 | return end - start |
| 1697 | |
| 1698 | def get_num_layers_by_block_type( |
| 1699 | self, |
| 1700 | parallel_config: ParallelConfig, |
| 1701 | block_type: LayerBlockType = "attention", |
| 1702 | ) -> int: |
| 1703 | # This function relies on 'layers_block_type' in hf_config, |
| 1704 | # for w/o this attribute, we will need to have workarounds like so |
| 1705 | attn_block_type = block_type == "attention" |
| 1706 | is_transformer = ( |
| 1707 | not self.is_hybrid and not self.has_noops and not self.is_attention_free |
| 1708 | ) |
| 1709 | start, end = self.get_layers_start_end_indices(parallel_config) |
| 1710 | |
| 1711 | if is_transformer: |
| 1712 | # Handle the basic case first |
| 1713 | return end - start if attn_block_type else 0 |
| 1714 | elif self.is_attention_free: |
| 1715 | # Attention free |
| 1716 | # Note that this code assumes there |
| 1717 | # is only one type of attention-free block type. |
| 1718 | return 0 if attn_block_type else end - start |
| 1719 | elif self.has_noops: |
| 1720 | block_configs = self.hf_config.block_configs |
| 1721 | return sum(not bc.attention.no_op for bc in block_configs[start:end]) |
| 1722 | else: |
| 1723 | # Hybrid model Jamba |
| 1724 | layers_block_type_value = getattr( |
| 1725 | self.hf_text_config, "layers_block_type", None |
| 1726 | ) |
| 1727 | if layers_block_type_value is not None: |
| 1728 | if self.model_arch_config.text_model_type == "zamba2": |
| 1729 | if attn_block_type: |
| 1730 | return sum( |
| 1731 | t == "hybrid" for t in layers_block_type_value[start:end] |
| 1732 | ) |
| 1733 | else: |
| 1734 | return self.get_num_layers(parallel_config) |
| 1735 | return sum(t == block_type for t in layers_block_type_value[start:end]) |
| 1736 | |
| 1737 | # Hybrid model Minimax |
| 1738 | attn_type_list = getattr(self.hf_config, "attn_type_list", None) |
| 1739 | if attn_type_list: |
| 1740 | return sum(t == 1 for t in attn_type_list[start:end]) |
| 1741 | |
| 1742 | # Hybrid model Qwen3Next Qwen3.5 Series |
| 1743 | layer_types_value = getattr(self.hf_text_config, "layer_types", None) |
| 1744 | if layer_types_value is not None: |
| 1745 | if block_type == "attention": |
| 1746 | return sum( |
| 1747 | t in _ATTENTION_LAYER_TYPES |
| 1748 | for t in layer_types_value[start:end] |
| 1749 | ) |
| 1750 | elif block_type == "linear_attention": |
| 1751 | return sum( |
| 1752 | t == "linear_attention" for t in layer_types_value[start:end] |
| 1753 | ) |
| 1754 | else: |
| 1755 | return sum(t == block_type for t in layer_types_value[start:end]) |
| 1756 | |
| 1757 | if ( |
| 1758 | layers_block_type_value is None |
| 1759 | and attn_type_list is None |
| 1760 | and layer_types_value is None |
| 1761 | ): |
| 1762 | raise ValueError( |
| 1763 | "The model is an hybrid without a layers_block_type or an " |
| 1764 | "attn_type_list, or a layer_types in the hf_config, " |
| 1765 | f"cannot determine the num of {block_type} layers" |
| 1766 | ) |
| 1767 | raise AssertionError(f"Unsupported block type: {block_type}") |
| 1768 | |
| 1769 | def get_mamba_chunk_size(self) -> int: |
| 1770 | """Returns the mamba chunk size if it exists.""" |
| 1771 | # used by e.g. Bamba, FalconH1, Granite |
| 1772 | chunk_size = getattr(self.hf_text_config, "mamba_chunk_size", None) |
| 1773 | if chunk_size is None: |
| 1774 | # used by e.g. Mamba2, NemotronH, Zamba |
| 1775 | chunk_size = getattr(self.hf_text_config, "chunk_size", None) |
| 1776 | |
| 1777 | # Since Mamba1 does not have a chunk notion |
| 1778 | # we use a default chunk size of 2048. |
| 1779 | if chunk_size is None: |
| 1780 | chunk_size = 2048 |
| 1781 | |
| 1782 | return chunk_size |
| 1783 | |
| 1784 | def get_multimodal_config(self) -> MultiModalConfig: |
| 1785 | """Get the multimodal configuration of the model. |
| 1786 | |
| 1787 | Raises: |
| 1788 | ValueError: If the model is not multimodal. |
| 1789 | |
| 1790 | """ |
| 1791 | if self.multimodal_config is None: |
| 1792 | raise ValueError("The model is not multimodal.") |
| 1793 | |
| 1794 | return self.multimodal_config |
| 1795 | |
| 1796 | def try_get_generation_config(self) -> dict[str, Any]: |
| 1797 | """This method attempts to retrieve the non-default values of the |
| 1798 | generation config for this model. |
| 1799 | |
| 1800 | The generation config can contain information about special tokens, as |
| 1801 | well as sampling parameters. Which is why this method exists separately |
| 1802 | to `get_diff_sampling_param`. |
| 1803 | |
| 1804 | Returns: |
| 1805 | A dictionary containing the non-default generation config. |
| 1806 | |
| 1807 | """ |
| 1808 | if self.generation_config in {"auto", "vllm"}: |
| 1809 | config = try_get_generation_config( |
| 1810 | self.hf_config_path or self.model, |
| 1811 | trust_remote_code=self.trust_remote_code, |
| 1812 | revision=self._hf_config_revision or self.revision, |
| 1813 | code_revision=self.code_revision, |
| 1814 | config_format=self.config_format, |
| 1815 | hf_token=self.hf_token, |
| 1816 | ) |
| 1817 | else: |
| 1818 | config = try_get_generation_config( |
| 1819 | self.generation_config, |
| 1820 | trust_remote_code=self.trust_remote_code, |
| 1821 | code_revision=self.code_revision, |
| 1822 | config_format=self.config_format, |
| 1823 | hf_token=self.hf_token, |
| 1824 | ) |
| 1825 | |
| 1826 | if config is None: |
| 1827 | return {} |
| 1828 | |
| 1829 | return config.to_diff_dict() |
| 1830 | |
| 1831 | def get_diff_sampling_param(self) -> dict[str, Any]: |
| 1832 | """This method returns a dictionary containing the non-default sampling |
| 1833 | parameters with `override_generation_config` applied. |
| 1834 | |
| 1835 | The default sampling parameters are: |
| 1836 | |
| 1837 | - vLLM's neutral defaults if `self.generation_config="vllm"` |
| 1838 | - the model's defaults if `self.generation_config="auto"` |
| 1839 | - as defined in `generation_config.json` if |
| 1840 | `self.generation_config="path/to/generation_config/dir"` |
| 1841 | |
| 1842 | Returns: |
| 1843 | A dictionary containing the non-default sampling parameters. |
| 1844 | |
| 1845 | """ |
| 1846 | src = self.generation_config |
| 1847 | |
| 1848 | config = {} if src == "vllm" else self.try_get_generation_config() |
| 1849 | |
| 1850 | # Overriding with given generation config |
| 1851 | config.update(self.override_generation_config) |
| 1852 | |
| 1853 | available_params = [ |
| 1854 | "repetition_penalty", |
| 1855 | "presence_penalty", |
| 1856 | "frequency_penalty", |
| 1857 | "temperature", |
| 1858 | "top_k", |
| 1859 | "top_p", |
| 1860 | "min_p", |
| 1861 | "max_new_tokens", |
| 1862 | ] |
| 1863 | if any(p in config for p in available_params): |
| 1864 | diff_sampling_param = { |
| 1865 | p: config.get(p) for p in available_params if config.get(p) is not None |
| 1866 | } |
| 1867 | # Huggingface definition of max_new_tokens is equivalent |
| 1868 | # to vLLM's max_tokens |
| 1869 | if "max_new_tokens" in diff_sampling_param: |
| 1870 | diff_sampling_param["max_tokens"] = diff_sampling_param.pop( |
| 1871 | "max_new_tokens" |
| 1872 | ) |
| 1873 | else: |
| 1874 | diff_sampling_param = {} |
| 1875 | |
| 1876 | if diff_sampling_param and src != "vllm": |
| 1877 | logger.warning_once( |
| 1878 | "Default vLLM sampling parameters have been overridden by %s: `%s`. " |
| 1879 | "If this is not intended, please relaunch vLLM instance " |
| 1880 | "with `--generation-config vllm`.", |
| 1881 | "the model's `generation_config.json`" if src == "auto" else src, |
| 1882 | str(diff_sampling_param), |
| 1883 | scope="local", |
| 1884 | ) |
| 1885 | |
| 1886 | return diff_sampling_param |
| 1887 | |
| 1888 | def get_pooling_task( |
| 1889 | self, supported_tasks: tuple[SupportedTask, ...] |
| 1890 | ) -> PoolingTask | None: |
| 1891 | if self.pooler_config is None: |
| 1892 | return None |
| 1893 | |
| 1894 | pooling_task = self.pooler_config.task |
| 1895 | |
| 1896 | if pooling_task is not None: |
| 1897 | if self.pooler_config.task in supported_tasks: |
| 1898 | return self.pooler_config.task |
| 1899 | else: |
| 1900 | raise RuntimeError( |
| 1901 | f"Unsupported task: {pooling_task!r} " |
| 1902 | f"Supported tasks: {supported_tasks}" |
| 1903 | ) |
| 1904 | |
| 1905 | if "token_classify" in supported_tasks: |
| 1906 | for architecture in self.architectures: |
| 1907 | if "ForTokenClassification" in architecture: |
| 1908 | return "token_classify" |
| 1909 | |
| 1910 | priority: list[PoolingTask] = [ |
| 1911 | "embed&token_classify", |
| 1912 | "embed", |
| 1913 | "classify", |
| 1914 | "token_embed", |
| 1915 | "token_classify", |
| 1916 | "plugin", |
| 1917 | ] |
| 1918 | for task in priority: |
| 1919 | if task in supported_tasks: |
| 1920 | return task |
| 1921 | return None |
| 1922 | |
| 1923 | @cached_property |
| 1924 | def is_encoder_decoder(self) -> bool: |
| 1925 | """Extract the HF encoder/decoder model flag.""" |
| 1926 | return is_encoder_decoder(self.hf_config) |
| 1927 | |
| 1928 | @cached_property |
| 1929 | def is_diffusion(self) -> bool: |
| 1930 | """Detect discrete diffusion (dLLM) models from HF config.""" |
| 1931 | return getattr(self.hf_config, "canvas_length", None) is not None |
| 1932 | |
| 1933 | @property |
| 1934 | def uses_alibi(self) -> bool: |
| 1935 | cfg = self.hf_text_config |
| 1936 | |
| 1937 | return ( |
| 1938 | getattr(cfg, "alibi", False) # Falcon |
| 1939 | or "BloomForCausalLM" in self.architectures # Bloom |
| 1940 | or getattr(cfg, "position_encoding_type", "") == "alibi" # codellm_1b_alibi |
| 1941 | or ( |
| 1942 | hasattr(cfg, "attn_config") # MPT |
| 1943 | and ( |
| 1944 | ( |
| 1945 | isinstance(cfg.attn_config, dict) |
| 1946 | and cfg.attn_config.get("alibi", False) |
| 1947 | ) |
| 1948 | or ( |
| 1949 | not isinstance(cfg.attn_config, dict) |
| 1950 | and getattr(cfg.attn_config, "alibi", False) |
| 1951 | ) |
| 1952 | ) |
| 1953 | ) |
| 1954 | ) |
| 1955 | |
| 1956 | @property |
| 1957 | def uses_mrope(self) -> bool: |
| 1958 | return uses_mrope(self.hf_config) |
| 1959 | |
| 1960 | @property |
| 1961 | def mrope_num_dims(self) -> int: |
| 1962 | return mrope_num_dims(self.hf_config) |
| 1963 | |
| 1964 | @property |
| 1965 | def is_multimodal_model(self) -> bool: |
| 1966 | return self.multimodal_config is not None |
| 1967 | |
| 1968 | @property |
| 1969 | def is_multimodal_raw_input_only_model(self) -> bool: |
| 1970 | return self._model_info.supports_multimodal_raw_input_only |
| 1971 | |
| 1972 | @property |
| 1973 | def supports_multimodal_inputs(self) -> bool: |
| 1974 | return self._cached_supports_multimodal_inputs() |
| 1975 | |
| 1976 | @property |
| 1977 | def requires_raw_input_tokens(self) -> bool: |
| 1978 | return self._model_info.requires_raw_input_tokens |
| 1979 | |
| 1980 | @property |
| 1981 | def score_type(self) -> ScoreType: |
| 1982 | """Scoring API handles score/rerank for: |
| 1983 | |
| 1984 | - "classify" task (score_type: cross-encoder models) |
| 1985 | - "embed" task (score_type: bi-encoder models) |
| 1986 | - "token_embed" task (score_type: late interaction models) |
| 1987 | """ |
| 1988 | # fixme: self._model_info.score_type is the score type before |
| 1989 | # as_seq_cls_model, which is "bi-encoder", rather than the |
| 1990 | # score type after as_seq_cls_model, which is "cross-encoder". |
| 1991 | # Therefore, the following logic is required. |
| 1992 | return ( |
| 1993 | "cross-encoder" |
| 1994 | if self.convert_type == "classify" |
| 1995 | else self._model_info.score_type |
| 1996 | ) |
| 1997 | |
| 1998 | @property |
| 1999 | def is_pp_supported(self) -> bool: |
| 2000 | return self._model_info.supports_pp |
| 2001 | |
| 2002 | @property |
| 2003 | def is_attention_free(self) -> bool: |
| 2004 | return self._model_info.is_attention_free |
| 2005 | |
| 2006 | @property |
| 2007 | def is_hybrid(self) -> bool: |
| 2008 | if not self._model_info.is_hybrid: |
| 2009 | return False |
| 2010 | # Handle granite-4.0-micro case which uses hybrid config but does not |
| 2011 | # actually contain any non-attention layers. |
| 2012 | layer_types = getattr(self.hf_config, "layer_types", None) |
| 2013 | return layer_types is None or not all( |
| 2014 | layer in ("attention", "full_attention") for layer in layer_types |
| 2015 | ) |
| 2016 | |
| 2017 | @property |
| 2018 | def has_noops(self) -> bool: |
| 2019 | return self._model_info.has_noops |
| 2020 | |
| 2021 | @property |
| 2022 | def has_inner_state(self): |
| 2023 | return self._model_info.has_inner_state |
| 2024 | |
| 2025 | @property |
| 2026 | def supports_mamba_prefix_caching(self) -> bool: |
| 2027 | return self._model_info.supports_mamba_prefix_caching |
| 2028 | |
| 2029 | @property |
| 2030 | def supports_replayssm(self) -> bool: |
| 2031 | return self._model_info.supports_replayssm |
| 2032 | |
| 2033 | @property |
| 2034 | def use_mla(self) -> bool: |
| 2035 | if envs.VLLM_MLA_DISABLE: |
| 2036 | return False |
| 2037 | if self.using_transformers_backend(): |
| 2038 | # kv_lora_rank indicates that a Transformers model implementation uses MLA |
| 2039 | return getattr(self.hf_text_config, "kv_lora_rank", None) is not None |
| 2040 | # Manually maintained list of model types for vLLM model implementations |
| 2041 | |
| 2042 | # Bidirectional DeepSeek variants (is_causal=False, used by some |
| 2043 | # embedding models) must use the non-MLA attention path, since the |
| 2044 | # MLA kernels only support causal attention. |
| 2045 | if not getattr(self.hf_text_config, "is_causal", True): |
| 2046 | return False |
| 2047 | return self.is_deepseek_mla |
| 2048 | |
| 2049 | @property |
| 2050 | def is_matryoshka(self) -> bool: |
| 2051 | return bool(getattr(self.hf_config, "matryoshka_dimensions", None)) or getattr( |
| 2052 | self.hf_config, "is_matryoshka", False |
| 2053 | ) |
| 2054 | |
| 2055 | @property |
| 2056 | def matryoshka_dimensions(self): |
| 2057 | return getattr(self.hf_config, "matryoshka_dimensions", None) |
| 2058 | |
| 2059 | @property |
| 2060 | def use_sep_token(self) -> bool: |
| 2061 | # cross_encoder models defaults to using separating token. |
| 2062 | # `llm as reranker` defaults to not using separating token. |
| 2063 | |
| 2064 | use_pad_token = getattr(self.hf_config, "use_pad_token", None) |
| 2065 | if use_pad_token is not None: |
| 2066 | logger.warning_once( |
| 2067 | "use_pad_token has been deprecated; please use use_sep_token instead." |
| 2068 | ) |
| 2069 | return use_pad_token |
| 2070 | |
| 2071 | return getattr(self.hf_config, "use_sep_token", True) |
| 2072 | |
| 2073 | @property |
| 2074 | def head_dtype(self) -> torch.dtype: |
| 2075 | """The "head" refers to the last Linear layer(s) of an LLM, |
| 2076 | such as the lm_head in a generation model, |
| 2077 | or the score or classifier in a classification model. |
| 2078 | |
| 2079 | - Pooling models default to an fp32 head; use |
| 2080 | --hf-overrides '{"head_dtype": "model"}' to disable it. |
| 2081 | - Generation models default to the model dtype; set |
| 2082 | --hf-overrides '{"head_dtype": "float32"}' to run the lm_head in |
| 2083 | fp32, which is required for RL training-inference consistency |
| 2084 | (the trainer computes logits in fp32). |
| 2085 | """ |
| 2086 | head_dtype = _get_head_dtype( |
| 2087 | config=self.hf_config, dtype=self.dtype, runner_type=self.runner_type |
| 2088 | ) |
| 2089 | |
| 2090 | if head_dtype not in current_platform.supported_dtypes: |
| 2091 | logger.warning_once( |
| 2092 | "The current platform does not support [%s] head dtype, " |
| 2093 | "fallback to model dtype [%s].", |
| 2094 | head_dtype, |
| 2095 | self.dtype, |
| 2096 | ) |
| 2097 | return self.dtype |
| 2098 | |
| 2099 | logger.debug_once("head dtype: %s", head_dtype) |
| 2100 | return head_dtype |
| 2101 | |
| 2102 | @property |
| 2103 | def embedding_size(self): |
| 2104 | # Check for embedding_size set by model config (e.g., Voyage models) |
| 2105 | override = getattr(self.hf_config, "embedding_size", None) |
| 2106 | if override is not None: |
| 2107 | return override |
| 2108 | dense_modules = try_get_dense_modules(self.model, revision=self.revision) |
| 2109 | if dense_modules is not None: |
| 2110 | return dense_modules[-1]["out_features"] |
| 2111 | return self.get_hidden_size() |
| 2112 | |
| 2113 | def get_and_verify_max_len(self, max_model_len: int): |
| 2114 | # Consider max_model_len in tokenizer_config only when |
| 2115 | # pooling models use absolute position_embedding. |
| 2116 | tokenizer_config = None |
| 2117 | if ( |
| 2118 | self.runner_type == "pooling" |
| 2119 | and getattr(self.hf_config, "position_embedding_type", "") == "absolute" |
| 2120 | ): |
| 2121 | tokenizer_config = try_get_tokenizer_config( |
| 2122 | self.tokenizer, |
| 2123 | trust_remote_code=self.trust_remote_code, |
| 2124 | revision=self.tokenizer_revision, |
| 2125 | ) |
| 2126 | max_model_len = _get_and_verify_max_len( |
| 2127 | hf_config=self.hf_text_config, |
| 2128 | model_arch_config=self.model_arch_config, |
| 2129 | tokenizer_config=tokenizer_config, |
| 2130 | max_model_len=max_model_len, |
| 2131 | disable_sliding_window=self.disable_sliding_window, |
| 2132 | sliding_window=self.get_sliding_window(), |
| 2133 | spec_target_max_model_len=self.spec_target_max_model_len, |
| 2134 | encoder_config=self.encoder_config, |
| 2135 | ) |
| 2136 | logger.info("Using max model len %s", max_model_len) |
| 2137 | return max_model_len |
| 2138 | |
| 2139 | @property |
| 2140 | def attn_type(self) -> AttnTypeStr: |
| 2141 | """Determine the attention type based on model configuration.""" |
| 2142 | if self.pooler_config is not None: |
| 2143 | seq_pooling_type = self._model_info.default_seq_pooling_type |
| 2144 | if seq_pooling_type == "CLS": |
| 2145 | return "encoder_only" |
| 2146 | else: |
| 2147 | is_causal = getattr(self.hf_config, "is_causal", True) |
| 2148 | return "encoder_only" if not is_causal else self._model_info.attn_type |
| 2149 | elif self.is_hybrid: |
| 2150 | return "hybrid" |
| 2151 | elif self.is_attention_free: |
| 2152 | return "attention_free" |
| 2153 | elif self.is_encoder_decoder: |
| 2154 | return "encoder_decoder" |
| 2155 | else: |
| 2156 | return "decoder" |
| 2157 | |
| 2158 | @property |
| 2159 | def is_chunked_prefill_supported(self) -> bool: |
| 2160 | attn_type = self.attn_type |
| 2161 | |
| 2162 | if pooler_config := self.pooler_config: |
| 2163 | # for pooling models |
| 2164 | if attn_type == "encoder_only": |
| 2165 | logger.debug( |
| 2166 | "Pooling models with bidirectional attn " |
| 2167 | "do not support chunked prefill." |
| 2168 | ) |
| 2169 | return False |
| 2170 | |
| 2171 | if attn_type == "decoder": |
| 2172 | if ( |
| 2173 | pooler_config.seq_pooling_type in ("MEAN", "CLS") |
| 2174 | or pooler_config.tok_pooling_type == "STEP" |
| 2175 | ): |
| 2176 | logger.debug( |
| 2177 | "Pooling models with causal attn and %s/%s pooling " |
| 2178 | "do not support chunked prefill.", |
| 2179 | pooler_config.seq_pooling_type, |
| 2180 | pooler_config.tok_pooling_type, |
| 2181 | ) |
| 2182 | return False |
| 2183 | else: |
| 2184 | logger.debug( |
| 2185 | "Pooling models with causal attn and %s/%s pooling " |
| 2186 | "support chunked prefill.", |
| 2187 | pooler_config.seq_pooling_type, |
| 2188 | pooler_config.tok_pooling_type, |
| 2189 | ) |
| 2190 | return True |
| 2191 | |
| 2192 | # vllm currently does not have pooling models using hybrid, |
| 2193 | # attention_free or encoder_decoder attn types. |
| 2194 | return attn_type != "encoder_decoder" |
| 2195 | else: |
| 2196 | # for generative models |
| 2197 | if attn_type == "encoder_decoder": |
| 2198 | logger.debug("Encoder decoder models do not support chunked prefill.") |
| 2199 | return False |
| 2200 | |
| 2201 | logger.debug("Generative models support chunked prefill.") |
| 2202 | return True |
| 2203 | |
| 2204 | @property |
| 2205 | def is_prefix_caching_supported(self) -> bool: |
| 2206 | attn_type = self.attn_type |
| 2207 | |
| 2208 | if pooler_config := self.pooler_config: |
| 2209 | # for pooling models |
| 2210 | if attn_type == "encoder_only": |
| 2211 | logger.debug( |
| 2212 | "Pooling models with bidirectional attn " |
| 2213 | "do not support prefix caching." |
| 2214 | ) |
| 2215 | return False |
| 2216 | |
| 2217 | if attn_type == "decoder": |
| 2218 | if ( |
| 2219 | pooler_config.seq_pooling_type in ("MEAN", "CLS") |
| 2220 | or pooler_config.tok_pooling_type == "STEP" |
| 2221 | ): |
| 2222 | logger.debug( |
| 2223 | "Pooling models with causal attn and %s/%s pooling " |
| 2224 | "do not support prefix caching.", |
| 2225 | pooler_config.seq_pooling_type, |
| 2226 | pooler_config.tok_pooling_type, |
| 2227 | ) |
| 2228 | return False |
| 2229 | else: |
| 2230 | logger.debug( |
| 2231 | "Pooling models with causal attn and %s/%s pooling " |
| 2232 | "support prefix caching.", |
| 2233 | pooler_config.seq_pooling_type, |
| 2234 | pooler_config.tok_pooling_type, |
| 2235 | ) |
| 2236 | return True |
| 2237 | |
| 2238 | # vllm currently does not have pooling models using hybrid, |
| 2239 | # attention_free or encoder_decoder attn types. |
| 2240 | return False |
| 2241 | else: |
| 2242 | # for generative models |
| 2243 | if attn_type == "hybrid": |
| 2244 | logger.debug("Generative hybrid models support prefix caching.") |
| 2245 | return True |
| 2246 | elif attn_type == "attention_free": |
| 2247 | logger.debug( |
| 2248 | "Attention free models do not support prefix caching since the " |
| 2249 | "feature is still experimental." |
| 2250 | ) |
| 2251 | return False |
| 2252 | elif attn_type == "encoder_decoder": |
| 2253 | logger.debug("Encoder decoder models do not support prefix caching.") |
| 2254 | return False |
| 2255 | else: # attn_type == "decoder" |
| 2256 | logger.debug("Generative models support prefix caching.") |
| 2257 | return True |
| 2258 | |
| 2259 | @property |
| 2260 | def is_moe(self) -> bool: |
| 2261 | return self.get_num_experts() > 0 |
| 2262 | |
| 2263 | @property |
| 2264 | def is_quantized(self) -> bool: |
| 2265 | return getattr(self.hf_config, "quantization_config", None) is not None |
| 2266 | |
| 2267 | def is_nvfp4_quantized(self) -> bool: |
| 2268 | quant_config = self.model_arch_config.quantization_config |
| 2269 | |
| 2270 | # ModelOpt NVFP4 checkpoints resolve to modelopt_fp4 quantization method |
| 2271 | if self.quantization in ("modelopt_fp4",): |
| 2272 | return True |
| 2273 | |
| 2274 | # A checkpoint mixing NVFP4 with another algorithm declares |
| 2275 | # quant_algo MIXED_PRECISION and resolves to modelopt_mixed, so the |
| 2276 | # per-layer algorithms decide. |
| 2277 | if self.quantization == "modelopt_mixed": |
| 2278 | return _modelopt_mixed_has_nvfp4(quant_config) |
| 2279 | |
| 2280 | # For Compressed Tensors we look for `"format": "nvfp4-pack-quantized"` |
| 2281 | # in the quantization config |
| 2282 | return ( |
| 2283 | self.quantization == "compressed-tensors" |
| 2284 | and quant_config is not None |
| 2285 | and "nvfp4" in quant_config.get("format", "").lower() |
| 2286 | ) |
| 2287 | |
| 2288 | |
| 2289 | def get_served_model_name(model: str, served_model_name: str | list[str] | None): |
| 2290 | """If the input is a non-empty list, the first model_name in |
| 2291 | `served_model_name` is taken. |
| 2292 | If the input is a non-empty string, it is used directly. |
| 2293 | For cases where the input is either an empty string or an |
| 2294 | empty list, the fallback is to use `self.model`. |
| 2295 | """ |
| 2296 | if not served_model_name: |
| 2297 | return model |
| 2298 | if isinstance(served_model_name, list): |
| 2299 | return served_model_name[0] |
| 2300 | return served_model_name |
| 2301 | |
| 2302 | |
| 2303 | # Some model suffixes are based on auto classes from Transformers: |
| 2304 | # https://huggingface.co/docs/transformers/en/model_doc/auto |
| 2305 | # NOTE: Items higher on this list priority over lower ones |
| 2306 | _SUFFIX_TO_DEFAULTS: list[tuple[str, tuple[RunnerType, ConvertType]]] = [ |
| 2307 | ("ForCausalLM", ("generate", "none")), |
| 2308 | ("ForConditionalGeneration", ("generate", "none")), |
| 2309 | ("ChatModel", ("generate", "none")), |
| 2310 | ("LMHeadModel", ("generate", "none")), |
| 2311 | ("ForTextEncoding", ("pooling", "embed")), |
| 2312 | ("EmbeddingModel", ("pooling", "embed")), |
| 2313 | ("ForSequenceClassification", ("pooling", "classify")), |
| 2314 | ("ForTokenClassification", ("pooling", "classify")), |
| 2315 | ("ForAudioClassification", ("pooling", "classify")), |
| 2316 | ("ForImageClassification", ("pooling", "classify")), |
| 2317 | ("ForVideoClassification", ("pooling", "classify")), |
| 2318 | ("ClassificationModel", ("pooling", "classify")), |
| 2319 | ("ForRewardModeling", ("pooling", "embed")), |
| 2320 | ("RewardModel", ("pooling", "embed")), |
| 2321 | # Let other `*Model`s take priority |
| 2322 | ("Model", ("pooling", "embed")), |
| 2323 | ] |
| 2324 | |
| 2325 | |
| 2326 | def iter_architecture_defaults(): |
| 2327 | yield from _SUFFIX_TO_DEFAULTS |
| 2328 | |
| 2329 | |
| 2330 | def try_match_architecture_defaults( |
| 2331 | architecture: str, |
| 2332 | *, |
| 2333 | runner_type: RunnerType | None = None, |
| 2334 | convert_type: ConvertType | None = None, |
| 2335 | ) -> tuple[str, tuple[RunnerType, ConvertType]] | None: |
| 2336 | for suffix, ( |
| 2337 | default_runner_type, |
| 2338 | default_convert_type, |
| 2339 | ) in iter_architecture_defaults(): |
| 2340 | if ( |
| 2341 | (runner_type is None or runner_type == default_runner_type) |
| 2342 | and (convert_type is None or convert_type == default_convert_type) |
| 2343 | and architecture.endswith(suffix) |
| 2344 | ): |
| 2345 | return suffix, (default_runner_type, default_convert_type) |
| 2346 | |
| 2347 | return None |
| 2348 | |
| 2349 | |
| 2350 | _STR_DTYPE_TO_TORCH_DTYPE = { |
| 2351 | "half": torch.float16, |
| 2352 | "float16": torch.float16, |
| 2353 | "float": torch.float32, |
| 2354 | "float32": torch.float32, |
| 2355 | "bfloat16": torch.bfloat16, |
| 2356 | } |
| 2357 | |
| 2358 | |
| 2359 | def str_dtype_to_torch_dtype(type: str): |
| 2360 | return _STR_DTYPE_TO_TORCH_DTYPE.get(type) |
| 2361 | |
| 2362 | |
| 2363 | # model_type -> reason |
| 2364 | _FLOAT16_NOT_SUPPORTED_MODELS = { |
| 2365 | "gemma2": "Numerical instability. Please use bfloat16 or float32 instead.", |
| 2366 | "gemma3": "Numerical instability. Please use bfloat16 or float32 instead.", |
| 2367 | "gemma3_text": "Numerical instability. Please use bfloat16 or float32 instead.", |
| 2368 | "glm4": "Numerical instability. Please use bfloat16 or float32 instead.", |
| 2369 | } |
| 2370 | |
| 2371 | |
| 2372 | def _is_valid_dtype(model_type: str, dtype: torch.dtype): |
| 2373 | if model_type in _FLOAT16_NOT_SUPPORTED_MODELS and dtype == torch.float16: # noqa: E501, SIM103 |
| 2374 | return False |
| 2375 | |
| 2376 | return True |
| 2377 | |
| 2378 | |
| 2379 | def _check_valid_dtype(model_type: str, dtype: torch.dtype): |
| 2380 | if model_type in _FLOAT16_NOT_SUPPORTED_MODELS and dtype == torch.float16: |
| 2381 | reason = _FLOAT16_NOT_SUPPORTED_MODELS[model_type] |
| 2382 | raise ValueError( |
| 2383 | f"The model type {model_type!r} does not support float16. Reason: {reason}" |
| 2384 | ) |
| 2385 | |
| 2386 | return True |
| 2387 | |
| 2388 | |
| 2389 | def _resolve_auto_dtype( |
| 2390 | model_type: str, |
| 2391 | config_dtype: torch.dtype, |
| 2392 | *, |
| 2393 | is_pooling_model: bool, |
| 2394 | ): |
| 2395 | supported_dtypes = [ |
| 2396 | dtype |
| 2397 | for dtype in current_platform.supported_dtypes |
| 2398 | if _is_valid_dtype(model_type, dtype) |
| 2399 | ] |
| 2400 | |
| 2401 | if is_pooling_model and torch.float16 in supported_dtypes: |
| 2402 | preferred_dtype = torch.float16 |
| 2403 | else: |
| 2404 | preferred_dtype = supported_dtypes[0] |
| 2405 | |
| 2406 | # Downcast for float32 models |
| 2407 | if config_dtype == torch.float32: |
| 2408 | config_dtype = preferred_dtype |
| 2409 | |
| 2410 | if config_dtype in supported_dtypes: |
| 2411 | return config_dtype |
| 2412 | |
| 2413 | # Ensure device compatibility |
| 2414 | device_name = current_platform.get_device_name() |
| 2415 | device_capability = current_platform.get_device_capability() |
| 2416 | |
| 2417 | if device_capability is None: |
| 2418 | device_str = f"{device_name!r}" |
| 2419 | else: |
| 2420 | version_str = device_capability.as_version_str() |
| 2421 | device_str = f"{device_name!r} (with compute capability {version_str})" |
| 2422 | |
| 2423 | logger.warning( |
| 2424 | "Your device %s doesn't support %s. Falling back to %s for compatibility.", |
| 2425 | device_str, |
| 2426 | config_dtype, |
| 2427 | preferred_dtype, |
| 2428 | ) |
| 2429 | |
| 2430 | return preferred_dtype |
| 2431 | |
| 2432 | |
| 2433 | def _get_and_verify_dtype( |
| 2434 | model_id: str, |
| 2435 | config: PreTrainedConfig, |
| 2436 | dtype: str | torch.dtype, |
| 2437 | *, |
| 2438 | is_pooling_model: bool, |
| 2439 | revision: str | None = None, |
| 2440 | config_format: str | ConfigFormat = "hf", |
| 2441 | ) -> torch.dtype: |
| 2442 | config_dtype = ModelArchConfigConvertorBase.get_torch_dtype( |
| 2443 | config, model_id, revision=revision, config_format=config_format |
| 2444 | ) |
| 2445 | model_type = config.model_type |
| 2446 | |
| 2447 | if isinstance(dtype, str): |
| 2448 | dtype = dtype.lower() |
| 2449 | if dtype == "auto": |
| 2450 | # Set default dtype from model config |
| 2451 | torch_dtype = _resolve_auto_dtype( |
| 2452 | model_type, |
| 2453 | config_dtype, |
| 2454 | is_pooling_model=is_pooling_model, |
| 2455 | ) |
| 2456 | if ( |
| 2457 | current_platform.is_cpu() |
| 2458 | and current_platform.get_cpu_architecture() == CpuArchEnum.POWERPC |
| 2459 | and torch_dtype in (torch.float16, torch.float32) |
| 2460 | ): |
| 2461 | torch_dtype = torch.bfloat16 |
| 2462 | else: |
| 2463 | if dtype not in _STR_DTYPE_TO_TORCH_DTYPE: |
| 2464 | raise ValueError(f"Unknown dtype: {dtype!r}") |
| 2465 | torch_dtype = _STR_DTYPE_TO_TORCH_DTYPE[dtype] |
| 2466 | elif isinstance(dtype, torch.dtype): |
| 2467 | torch_dtype = dtype |
| 2468 | else: |
| 2469 | raise ValueError(f"Unknown dtype: {dtype}") |
| 2470 | |
| 2471 | _check_valid_dtype(model_type, torch_dtype) |
| 2472 | |
| 2473 | if torch_dtype != config_dtype: |
| 2474 | if torch_dtype == torch.float32: |
| 2475 | # Upcasting to float32 is allowed. |
| 2476 | logger.info("Upcasting %s to %s.", config_dtype, torch_dtype) |
| 2477 | elif config_dtype == torch.float32: |
| 2478 | # Downcasting from float32 to float16 or bfloat16 is allowed. |
| 2479 | logger.info("Downcasting %s to %s.", config_dtype, torch_dtype) |
| 2480 | else: |
| 2481 | # Casting between float16 and bfloat16 is allowed with a warning. |
| 2482 | logger.warning("Casting %s to %s.", config_dtype, torch_dtype) |
| 2483 | |
| 2484 | return torch_dtype |
| 2485 | |
| 2486 | |
| 2487 | def _get_head_dtype( |
| 2488 | config: PreTrainedConfig, dtype: torch.dtype, runner_type: str |
| 2489 | ) -> torch.dtype: |
| 2490 | head_dtype: str | torch.dtype | None = getattr(config, "head_dtype", None) |
| 2491 | |
| 2492 | if head_dtype == "model": |
| 2493 | return dtype |
| 2494 | elif isinstance(head_dtype, str): |
| 2495 | head_dtype = head_dtype.lower() |
| 2496 | if head_dtype not in _STR_DTYPE_TO_TORCH_DTYPE: |
| 2497 | raise ValueError(f"Unknown dtype: {head_dtype!r}") |
| 2498 | return _STR_DTYPE_TO_TORCH_DTYPE[head_dtype] |
| 2499 | elif isinstance(head_dtype, torch.dtype): |
| 2500 | return head_dtype |
| 2501 | elif head_dtype is None: |
| 2502 | if torch.float32 not in current_platform.supported_dtypes: |
| 2503 | return dtype |
| 2504 | if runner_type == "pooling": |
| 2505 | return torch.float32 |
| 2506 | return dtype |
| 2507 | else: |
| 2508 | raise ValueError(f"Unknown dtype: {head_dtype}") |
| 2509 | |
| 2510 | |
| 2511 | def _get_and_verify_max_len( |
| 2512 | hf_config: PreTrainedConfig, |
| 2513 | model_arch_config: ModelArchitectureConfig, |
| 2514 | tokenizer_config: dict | None, |
| 2515 | max_model_len: int | None, |
| 2516 | disable_sliding_window: bool, |
| 2517 | sliding_window: int | None, |
| 2518 | spec_target_max_model_len: int | None = None, |
| 2519 | encoder_config: dict[str, Any] | None = None, |
| 2520 | ) -> int: |
| 2521 | """Get and verify the model's maximum length.""" |
| 2522 | (derived_max_model_len, max_len_key) = ( |
| 2523 | model_arch_config.derived_max_model_len_and_key |
| 2524 | ) |
| 2525 | |
| 2526 | # If sliding window is manually disabled, max_length should be less |
| 2527 | # than the sliding window length in the model config. |
| 2528 | if ( |
| 2529 | disable_sliding_window |
| 2530 | and sliding_window is not None |
| 2531 | and sliding_window < derived_max_model_len |
| 2532 | ): |
| 2533 | max_len_key = "sliding_window" |
| 2534 | derived_max_model_len = sliding_window |
| 2535 | |
| 2536 | # Consider model_max_length in tokenizer_config |
| 2537 | if tokenizer_config: |
| 2538 | tokenizer_model_max_length = tokenizer_config.get( |
| 2539 | "model_max_length", derived_max_model_len |
| 2540 | ) |
| 2541 | derived_max_model_len = min(derived_max_model_len, tokenizer_model_max_length) |
| 2542 | |
| 2543 | # If none of the keys were found in the config, use a default and |
| 2544 | # log a warning. |
| 2545 | if derived_max_model_len == float("inf"): |
| 2546 | if max_model_len is not None: |
| 2547 | # If max_model_len is specified, we use it. |
| 2548 | return max_model_len |
| 2549 | |
| 2550 | if spec_target_max_model_len is not None: |
| 2551 | # If this is a speculative draft model, we use the max model len |
| 2552 | # from the target model. |
| 2553 | return spec_target_max_model_len |
| 2554 | |
| 2555 | default_max_len = 2048 |
| 2556 | logger.warning( |
| 2557 | "The model's config.json does not contain any of the keys " |
| 2558 | "to determine the original maximum length of the model. " |
| 2559 | "Assuming the model's maximum length is %d.", |
| 2560 | default_max_len, |
| 2561 | ) |
| 2562 | derived_max_model_len = default_max_len |
| 2563 | |
| 2564 | # In Transformers v5 rope_parameters could be TypedDict or dict[str, TypedDict]. |
| 2565 | # To simplify the verification, we convert it to dict[str, TypedDict]. |
| 2566 | rope_parameters = getattr(hf_config, "rope_parameters", None) |
| 2567 | if rope_parameters and not is_rope_parameters_nested(rope_parameters): |
| 2568 | rope_parameters = {"": rope_parameters} |
| 2569 | if rope_parameters is not None: |
| 2570 | # Layers without RoPE do not contribute to context length scaling. |
| 2571 | rope_parameters = { |
| 2572 | layer_type: rp |
| 2573 | for layer_type, rp in rope_parameters.items() |
| 2574 | if rp is not None |
| 2575 | } |
| 2576 | |
| 2577 | # NOTE(woosuk): Gemma3's max_model_len (128K) is already scaled by RoPE |
| 2578 | # scaling, so we skip applying the scaling factor again. |
| 2579 | if rope_parameters is not None and "gemma3" not in hf_config.model_type: |
| 2580 | scaling_factor = 1.0 |
| 2581 | for rp in rope_parameters.values(): |
| 2582 | # No need to consider "type" key because of patch_rope_parameters when |
| 2583 | # loading HF config |
| 2584 | rope_type = rp["rope_type"] |
| 2585 | |
| 2586 | # YaRN variants leave max_position_embeddings already scaled, as |
| 2587 | # Transformers' _compute_yarn_parameters assumes, so `factor` must |
| 2588 | # not be applied to it again. |
| 2589 | if rope_type not in ( |
| 2590 | "su", |
| 2591 | "longrope", |
| 2592 | "llama3", |
| 2593 | "yarn", |
| 2594 | "deepseek_yarn", |
| 2595 | "deepseek_llama_scaling", |
| 2596 | ): |
| 2597 | # NOTE: rope_type == "default" does not define factor https://github.com/huggingface/transformers/blob/v4.45.2/src/transformers/modeling_rope_utils.py |
| 2598 | # NOTE: This assumes all layer types have the same scaling factor. |
| 2599 | scaling_factor = rp.get("factor", scaling_factor) |
| 2600 | if scaling_factor is None: |
| 2601 | # Fallback the factor to 1.0 if a user assigned `null` |
| 2602 | logger.warning_once( |
| 2603 | "The model's RoPE configuration has a null scaling " |
| 2604 | "factor which is unexpected. This likely indicates a bug " |
| 2605 | "in the model's HuggingFace config.json. Please notify the " |
| 2606 | "model vendor. Falling back the value to 1.0. " |
| 2607 | ) |
| 2608 | scaling_factor = 1.0 |
| 2609 | # Do this outside loop since all layer types should have the same scaling |
| 2610 | derived_max_model_len *= scaling_factor |
| 2611 | |
| 2612 | if encoder_config and "max_seq_length" in encoder_config: |
| 2613 | derived_max_model_len = encoder_config["max_seq_length"] |
| 2614 | |
| 2615 | # If the user didn't specify `max_model_len` or specified -1 (auto-fit), |
| 2616 | # then use that derived from the model config as a default value. |
| 2617 | # When -1 is specified, the engine will later auto-fit to available memory. |
| 2618 | if max_model_len is None or max_model_len == -1: |
| 2619 | # For LongRoPE, default to original_max_position_embeddings to avoid |
| 2620 | # performance degradation for shorter sequences |
| 2621 | if rope_parameters is not None and any( |
| 2622 | rp["rope_type"] == "longrope" for rp in rope_parameters.values() |
| 2623 | ): |
| 2624 | max_model_len = int( |
| 2625 | getattr( |
| 2626 | hf_config, "original_max_position_embeddings", derived_max_model_len |
| 2627 | ) |
| 2628 | ) |
| 2629 | else: |
| 2630 | max_model_len = int(derived_max_model_len) |
| 2631 | max_model_len = current_platform.check_max_model_len(max_model_len) |
| 2632 | |
| 2633 | # If the user specified a max length, make sure it is smaller than the |
| 2634 | # derived length from the HF model config. |
| 2635 | elif max_model_len > derived_max_model_len: |
| 2636 | # Some models might have a separate key for specifying model_max_length |
| 2637 | # that will be bigger than derived_max_model_len. We compare user input |
| 2638 | # with model_max_length and allow this override when it's smaller. |
| 2639 | model_max_length = getattr(hf_config, "model_max_length", None) |
| 2640 | if model_max_length is None or max_model_len > model_max_length: |
| 2641 | msg = ( |
| 2642 | f"User-specified max_model_len ({max_model_len}) is greater " |
| 2643 | f"than the derived max_model_len ({max_len_key}=" |
| 2644 | f"{derived_max_model_len} or model_max_length=" |
| 2645 | f"{model_max_length} in model's config.json)." |
| 2646 | ) |
| 2647 | warning = ( |
| 2648 | "VLLM_ALLOW_LONG_MAX_MODEL_LEN must be used with extreme " |
| 2649 | "caution. If the model uses relative position encoding (RoPE), " |
| 2650 | "positions exceeding derived_max_model_len lead to nan. If the " |
| 2651 | "model uses absolute position encoding, positions exceeding " |
| 2652 | "derived_max_model_len will cause a CUDA array out-of-bounds " |
| 2653 | "error." |
| 2654 | ) |
| 2655 | if envs.VLLM_ALLOW_LONG_MAX_MODEL_LEN: |
| 2656 | logger.warning_once("%s %s", msg, warning) |
| 2657 | else: |
| 2658 | raise ValueError( |
| 2659 | f"{msg} To allow overriding this maximum, set " |
| 2660 | f"the env var VLLM_ALLOW_LONG_MAX_MODEL_LEN=1. {warning}" |
| 2661 | ) |
| 2662 | return int(max_model_len) |