| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Utility functions for vLLM config dataclasses.""" |
| 4 | |
| 5 | import ast |
| 6 | import enum |
| 7 | import hashlib |
| 8 | import inspect |
| 9 | import json |
| 10 | import os |
| 11 | import pathlib |
| 12 | import textwrap |
| 13 | from collections.abc import Callable, Mapping, Sequence, Set |
| 14 | from dataclasses import MISSING, field, fields, is_dataclass |
| 15 | from itertools import pairwise |
| 16 | from typing import TYPE_CHECKING, Any, Protocol, TypeVar, cast, get_type_hints, overload |
| 17 | |
| 18 | import torch |
| 19 | from pydantic import ConfigDict |
| 20 | from pydantic.dataclasses import dataclass |
| 21 | from pydantic.fields import Field as PydanticField |
| 22 | from pydantic.fields import FieldInfo |
| 23 | from typing_extensions import dataclass_transform, runtime_checkable |
| 24 | |
| 25 | import vllm.envs as envs |
| 26 | from vllm.logger import init_logger |
| 27 | |
| 28 | logger = init_logger(__name__) |
| 29 | |
| 30 | if TYPE_CHECKING: |
| 31 | from _typeshed import DataclassInstance |
| 32 | else: |
| 33 | DataclassInstance = Any |
| 34 | |
| 35 | ConfigType = type[DataclassInstance] |
| 36 | ConfigT = TypeVar("ConfigT", bound=DataclassInstance) |
| 37 | |
| 38 | |
| 39 | @overload |
| 40 | @dataclass_transform(field_specifiers=(PydanticField,)) |
| 41 | def config(cls: type[ConfigT]) -> type[ConfigT]: ... |
| 42 | |
| 43 | |
| 44 | @overload |
| 45 | @dataclass_transform(field_specifiers=(PydanticField,)) |
| 46 | def config( |
| 47 | *, config: ConfigDict | None = None, **kwargs: Any |
| 48 | ) -> Callable[[type[ConfigT]], type[ConfigT]]: ... |
| 49 | |
| 50 | |
| 51 | @dataclass_transform(field_specifiers=(PydanticField,)) |
| 52 | def config( |
| 53 | cls: type[ConfigT] | None = None, |
| 54 | *, |
| 55 | config: ConfigDict | None = None, |
| 56 | **kwargs: Any, |
| 57 | ) -> type[ConfigT] | Callable[[type[ConfigT]], type[ConfigT]]: |
| 58 | """Decorator to create a pydantic dataclass with default config. The default config |
| 59 | for the dataclass forbids extra fields. |
| 60 | |
| 61 | All config classes in vLLM should use this decorator. |
| 62 | |
| 63 | Args: |
| 64 | cls: The class to decorate |
| 65 | config: The pydantic ConfigDict to use. If provided, it will be merged with |
| 66 | the default config. |
| 67 | **kwargs: Additional arguments to pass to pydantic.dataclass. |
| 68 | |
| 69 | """ |
| 70 | # Extra fields are forbidden by default |
| 71 | merged_config = ConfigDict(extra="forbid") |
| 72 | if config is not None: |
| 73 | merged_config.update(config) |
| 74 | |
| 75 | def decorator(cls: type[ConfigT]) -> type[ConfigT]: |
| 76 | return dataclass(cls, config=merged_config, **kwargs) # type: ignore[return-value] |
| 77 | |
| 78 | # Called with arguments: @config(config=...) |
| 79 | if cls is None: |
| 80 | return decorator |
| 81 | # Called without arguments: @config |
| 82 | return decorator(cls) |
| 83 | |
| 84 | |
| 85 | def get_field(cls: ConfigType, name: str) -> Any: |
| 86 | """Get the default factory field of a dataclass by name. Used for getting |
| 87 | default factory fields in `EngineArgs`.""" |
| 88 | if not is_dataclass(cls): |
| 89 | raise TypeError("The given class is not a dataclass.") |
| 90 | try: |
| 91 | named_field = next(f for f in fields(cls) if f.name == name) |
| 92 | except StopIteration as e: |
| 93 | raise ValueError(f"Field '{name}' not found in {cls.__name__}.") from e |
| 94 | |
| 95 | # The arguments to copy to the new field |
| 96 | default = named_field.default |
| 97 | default_factory = named_field.default_factory |
| 98 | init = named_field.init |
| 99 | |
| 100 | # Handle pydantic.Field |
| 101 | if isinstance(default, FieldInfo): |
| 102 | if default.init is not None: |
| 103 | init = default.init |
| 104 | if default.default_factory is not None: |
| 105 | default_factory = cast(Callable[[], Any], default.default_factory) |
| 106 | default = MISSING |
| 107 | else: |
| 108 | default = default.default |
| 109 | |
| 110 | if default is MISSING and default_factory is MISSING: |
| 111 | logger.warning_once( |
| 112 | "%s.%s has no default or default factory.", cls.__name__, name |
| 113 | ) |
| 114 | return field(default=default, default_factory=default_factory, init=init) |
| 115 | |
| 116 | |
| 117 | def is_init_field(cls: ConfigType, name: str) -> bool: |
| 118 | return get_field(cls, name).init |
| 119 | |
| 120 | |
| 121 | def replace(dataclass_instance: ConfigT, /, **kwargs) -> ConfigT: |
| 122 | """Like [`dataclasses.replace`](https://docs.python.org/3/library/dataclasses.html#dataclasses.replace), |
| 123 | but compatible with Pydantic dataclasses which use `pydantic.fields.Field` instead |
| 124 | of `dataclasses.field`""" |
| 125 | cls = type(dataclass_instance) |
| 126 | dataclass_dict = dataclass_instance.__dict__ |
| 127 | dataclass_dict = {k: v for k, v in dataclass_dict.items() if is_init_field(cls, k)} |
| 128 | dataclass_dict.update(kwargs) |
| 129 | return cls(**dataclass_dict) |
| 130 | |
| 131 | |
| 132 | def getattr_iter( |
| 133 | object: object, |
| 134 | names: Sequence[str], |
| 135 | default: Any | None = None, |
| 136 | default_factory: Callable[[], Any] | None = None, |
| 137 | warn: bool = False, |
| 138 | ) -> Any: |
| 139 | """A helper function that retrieves an attribute from an object which may |
| 140 | have multiple possible names. This is useful when fetching attributes from |
| 141 | arbitrary `transformers.PreTrainedConfig` instances. |
| 142 | |
| 143 | In the case where the first name in `names` is the preferred name, and |
| 144 | any other names are deprecated aliases, setting `warn=True` will log a |
| 145 | warning when a deprecated name is used. |
| 146 | """ |
| 147 | for i, name in enumerate(names): |
| 148 | if hasattr(object, name): |
| 149 | if warn and i > 0: |
| 150 | logger.warning_once( |
| 151 | "%s contains a deprecated attribute name '%s'. " |
| 152 | "Please use the preferred attribute name '%s' instead.", |
| 153 | type(object).__name__, |
| 154 | name, |
| 155 | names[0], |
| 156 | ) |
| 157 | return getattr(object, name) |
| 158 | return default_factory() if default_factory is not None else default |
| 159 | |
| 160 | |
| 161 | def _get_own_attr_docs(cls: type[Any], out: dict[str, str]) -> None: |
| 162 | """Get any docstrings placed after attribute assignments in a class body. |
| 163 | |
| 164 | https://davidism.com/mit-license/ |
| 165 | """ |
| 166 | try: |
| 167 | source = inspect.getsource(cls) |
| 168 | except (OSError, TypeError): |
| 169 | # ``object`` and classes without retrievable source (e.g. builtins). |
| 170 | return |
| 171 | |
| 172 | cls_node = ast.parse(textwrap.dedent(source)).body[0] |
| 173 | if not isinstance(cls_node, ast.ClassDef): |
| 174 | return |
| 175 | |
| 176 | # Consider each pair of nodes. |
| 177 | for a, b in pairwise(cls_node.body): |
| 178 | # Must be an assignment then a constant string. |
| 179 | if ( |
| 180 | not isinstance(a, (ast.Assign, ast.AnnAssign)) |
| 181 | or not isinstance(b, ast.Expr) |
| 182 | or not isinstance(b.value, ast.Constant) |
| 183 | or not isinstance(b.value.value, str) |
| 184 | ): |
| 185 | continue |
| 186 | |
| 187 | doc = inspect.cleandoc(b.value.value) |
| 188 | |
| 189 | # An assignment can have multiple targets (a = b = v), but an |
| 190 | # annotated assignment only has one target. |
| 191 | targets = a.targets if isinstance(a, ast.Assign) else [a.target] |
| 192 | |
| 193 | for target in targets: |
| 194 | # Must be assigning to a plain name. |
| 195 | if not isinstance(target, ast.Name): |
| 196 | continue |
| 197 | |
| 198 | out[target.id] = doc |
| 199 | |
| 200 | |
| 201 | def get_attr_docs(cls: type[Any]) -> dict[str, str]: |
| 202 | """Get any docstrings placed after attribute assignments in a class body or |
| 203 | its bases.""" |
| 204 | if not isinstance(cls, type): |
| 205 | raise TypeError("Given object was not a class.") |
| 206 | |
| 207 | out: dict[str, str] = {} |
| 208 | |
| 209 | # Walk the MRO from the most-base class to ``cls`` so that a docstring |
| 210 | # redefined on a subclass overrides the one inherited from a base class. |
| 211 | for base in reversed(cls.__mro__): |
| 212 | _get_own_attr_docs(base, out) |
| 213 | |
| 214 | return out |
| 215 | |
| 216 | |
| 217 | @runtime_checkable |
| 218 | class SupportsHash(Protocol): |
| 219 | def compute_hash(self) -> str: ... |
| 220 | |
| 221 | |
| 222 | _config_hash_cache: dict[int, str] = {} |
| 223 | |
| 224 | |
| 225 | def compute_hash_cached(config: SupportsHash) -> str: |
| 226 | """Cache config.compute_hash() by object identity. |
| 227 | |
| 228 | Config objects (ModelConfig, etc.) are long-lived singletons that never |
| 229 | mutate after construction, but compute_hash() is expensive (JSON |
| 230 | serialization + SHA-256). This utility avoids recomputing the hash on |
| 231 | every forward pass while keeping a single consistent key type for all |
| 232 | lookup paths. |
| 233 | """ |
| 234 | key = id(config) |
| 235 | result = _config_hash_cache.get(key) |
| 236 | if result is None: |
| 237 | result = config.compute_hash() |
| 238 | _config_hash_cache[key] = result |
| 239 | return result |
| 240 | |
| 241 | |
| 242 | class SupportsMetricsInfo(Protocol): |
| 243 | def metrics_info(self) -> dict[str, str]: ... |
| 244 | |
| 245 | |
| 246 | def update_config(config: ConfigT, overrides: Mapping[str, Any]) -> ConfigT: |
| 247 | return _update_config(config, overrides, type(config).__name__) |
| 248 | |
| 249 | |
| 250 | def _update_config( |
| 251 | config: ConfigT, overrides: Mapping[str, Any], config_path: str |
| 252 | ) -> ConfigT: |
| 253 | processed_overrides: dict[str, Any] = {} |
| 254 | field_types = get_type_hints(type(config)) |
| 255 | for field_name, value in overrides.items(): |
| 256 | field_path = f"{config_path}.{field_name}" |
| 257 | if not hasattr(config, field_name): |
| 258 | raise ValueError(f"{field_path} is not a valid config field") |
| 259 | |
| 260 | current_value = getattr(config, field_name) |
| 261 | if is_dataclass(current_value): |
| 262 | expected_type = field_types[field_name] |
| 263 | if isinstance(value, Mapping): |
| 264 | value = _update_config( |
| 265 | current_value, # type: ignore[type-var] |
| 266 | value, |
| 267 | field_path, |
| 268 | ) |
| 269 | elif not isinstance(value, expected_type): |
| 270 | expected_type_name = getattr( |
| 271 | expected_type, "__name__", str(expected_type) |
| 272 | ) |
| 273 | raise ValueError( |
| 274 | f"Override for {field_path} must be a mapping or " |
| 275 | f"{expected_type_name}, got {type(value).__name__}" |
| 276 | ) |
| 277 | |
| 278 | processed_overrides[field_name] = value |
| 279 | return replace(config, **processed_overrides) |
| 280 | |
| 281 | |
| 282 | def normalize_value(x): |
| 283 | """Return a stable, JSON-serializable canonical form for hashing. |
| 284 | Order: primitives, special types (Enum, callable, torch.dtype, Path), then |
| 285 | generic containers (Mapping/Set/Sequence) with recursion. |
| 286 | """ |
| 287 | # Fast path |
| 288 | if x is None or isinstance(x, (bool, int, float, str)): |
| 289 | return x |
| 290 | |
| 291 | # Enums: tag with FQN to avoid primitive collisions. |
| 292 | # Ex: Enum(1) vs int(1) -> ("module.QualName", value). |
| 293 | if isinstance(x, enum.Enum): |
| 294 | enum_type = f"{x.__class__.__module__}.{x.__class__.__qualname__}" |
| 295 | return (enum_type, normalize_value(x.value)) |
| 296 | |
| 297 | # Classes (types) are accepted and canonicalized by their fully-qualified |
| 298 | # name (module.qualname) for a stable identifier. |
| 299 | # Instances are only accepted if they expose uuid(); otherwise they are |
| 300 | # rejected to avoid under-hashing object state. |
| 301 | |
| 302 | # Callables: accept classes only; reject funcs/lambdas/methods. |
| 303 | # Used by LogitsProcessor types and ModelConfig.hf_overrides. |
| 304 | if isinstance(x, type): |
| 305 | module = getattr(x, "__module__", "") |
| 306 | qual = getattr(x, "__qualname__", getattr(x, "__name__", "")) |
| 307 | return ".".join([p for p in (module, qual) if p]) or repr(x) |
| 308 | |
| 309 | # Prefer stable uuid identifiers for objects that provide them, even if |
| 310 | # they are callable instances (e.g., InductorPass wrappers). |
| 311 | if hasattr(x, "uuid") and callable(getattr(x, "uuid", None)): |
| 312 | return x.uuid() |
| 313 | |
| 314 | if callable(x): |
| 315 | raise TypeError("normalize_value: function or callable instance unsupported") |
| 316 | |
| 317 | # Torch dtype: stringify (torch.float64 -> "torch.float64"). |
| 318 | # We rely on the string form here; dtype-bearing fields that need additional |
| 319 | # disambiguation should encode that at the config layer. |
| 320 | if isinstance(x, torch.dtype): |
| 321 | return str(x) |
| 322 | |
| 323 | # Bytes |
| 324 | if isinstance(x, (bytes, bytearray)): |
| 325 | return x.hex() |
| 326 | |
| 327 | # Paths (canonicalize) |
| 328 | if isinstance(x, pathlib.Path): |
| 329 | try: |
| 330 | return str(x.expanduser().resolve()) |
| 331 | except Exception: |
| 332 | return str(x) |
| 333 | |
| 334 | # PreTrainedConfig (must be before dataclass branch as these are now dataclasses) |
| 335 | if hasattr(x, "to_json_string") and callable(x.to_json_string): |
| 336 | try: |
| 337 | return x.to_json_string() |
| 338 | except (TypeError, ValueError): |
| 339 | # to_json_string() may fail for trust-remote-code configs |
| 340 | # with non-JSON-serializable nested objects. Fall back to |
| 341 | # normalizing the dict representation recursively. |
| 342 | if hasattr(x, "to_dict") and callable(x.to_dict): |
| 343 | return normalize_value(x.to_dict()) |
| 344 | raise |
| 345 | |
| 346 | # Dataclasses: represent as (FQN, sorted(field,value) tuple) for stability. |
| 347 | if is_dataclass(x): |
| 348 | type_fqn = f"{x.__class__.__module__}.{x.__class__.__qualname__}" |
| 349 | items = tuple( |
| 350 | (f.name, normalize_value(getattr(x, f.name))) |
| 351 | for f in sorted(fields(x), key=lambda f: f.name) |
| 352 | ) |
| 353 | return (type_fqn, items) |
| 354 | |
| 355 | # Containers (generic) |
| 356 | if isinstance(x, Mapping): |
| 357 | return tuple(sorted((str(k), normalize_value(v)) for k, v in x.items())) |
| 358 | if isinstance(x, Set): |
| 359 | return tuple(sorted(repr(normalize_value(v)) for v in x)) |
| 360 | if isinstance(x, Sequence) and not isinstance(x, (str, bytes, bytearray)): |
| 361 | return tuple(normalize_value(v) for v in x) |
| 362 | |
| 363 | # Unsupported type: e.g., modules, generators, open files, or objects |
| 364 | # without a stable JSON/UUID representation. Hard-error to avoid |
| 365 | # under-hashing. |
| 366 | # If you hit this, either reshape your config to use supported primitives |
| 367 | # and containers, or extend normalize_value to provide a stable encoding |
| 368 | # (e.g., via uuid() or to_json_string()) for this type. |
| 369 | raise TypeError( |
| 370 | f"normalize_value: unsupported type '{type(x).__name__}'. " |
| 371 | "Ensure config values use supported primitives/containers or add a " |
| 372 | "stable representation for this type." |
| 373 | ) |
| 374 | |
| 375 | |
| 376 | def get_hash_factors(config: ConfigT, ignored_factors: set[str]) -> dict[str, object]: |
| 377 | """Gets the factors used for hashing a config class. |
| 378 | - Includes all dataclass fields not in `ignored_factors`. |
| 379 | - Errors on non-normalizable values. |
| 380 | """ |
| 381 | factors: dict[str, object] = {} |
| 382 | for dc_field in fields(config): |
| 383 | factor = dc_field.name |
| 384 | if factor in ignored_factors: |
| 385 | continue |
| 386 | value = getattr(config, factor, None) |
| 387 | try: |
| 388 | factors[factor] = normalize_value(value) |
| 389 | except TypeError as e: |
| 390 | raise TypeError( |
| 391 | f"get_hash_factors: unsupported type for key '{factor}' " |
| 392 | f"({type(value).__name__})" |
| 393 | ) from e |
| 394 | return factors |
| 395 | |
| 396 | |
| 397 | def hash_factors(items: dict[str, object]) -> str: |
| 398 | """Return a SHA-256 hex digest of the canonical items structure.""" |
| 399 | return hashlib.sha256(json.dumps(items, sort_keys=True).encode()).hexdigest() |
| 400 | |
| 401 | |
| 402 | @dataclass |
| 403 | class Range: |
| 404 | """A range of numbers. |
| 405 | Inclusive of start, inclusive of end. |
| 406 | """ |
| 407 | |
| 408 | start: int |
| 409 | end: int |
| 410 | |
| 411 | def is_single_size(self) -> bool: |
| 412 | return self.start == self.end |
| 413 | |
| 414 | def __contains__(self, size: int) -> bool: |
| 415 | # Inclusive of start, inclusive of end |
| 416 | return self.start <= size <= self.end |
| 417 | |
| 418 | def __eq__(self, other: object) -> bool: |
| 419 | if not isinstance(other, Range): |
| 420 | return False |
| 421 | return self.start == other.start and self.end == other.end |
| 422 | |
| 423 | def __hash__(self) -> int: |
| 424 | return hash((self.start, self.end)) |
| 425 | |
| 426 | def __str__(self) -> str: |
| 427 | return f"({self.start}, {self.end})" |
| 428 | |
| 429 | def __repr__(self) -> str: |
| 430 | return self.__str__() |
| 431 | |
| 432 | |
| 433 | def handle_deprecated( |
| 434 | config: ConfigT, |
| 435 | old_name: str, |
| 436 | new_name_or_names: str | list[str], |
| 437 | removal_version: str, |
| 438 | ) -> None: |
| 439 | old_val = getattr(config, old_name) |
| 440 | if old_val is None: |
| 441 | return |
| 442 | |
| 443 | if isinstance(new_name_or_names, str): |
| 444 | new_names = [new_name_or_names] |
| 445 | else: |
| 446 | new_names = new_name_or_names |
| 447 | |
| 448 | msg = ( |
| 449 | f"{old_name} is deprecated and will be removed in {removal_version}. " |
| 450 | f"Use {', '.join(new_names)} instead." |
| 451 | ) |
| 452 | logger.warning(msg) |
| 453 | |
| 454 | for new_name in new_names: |
| 455 | setattr(config, new_name, old_val) |
| 456 | |
| 457 | |
| 458 | def get_from_deprecated_env_if_set( |
| 459 | env_name: str, |
| 460 | removal_version: str, |
| 461 | field_name: str | None = None, |
| 462 | ) -> str | None: |
| 463 | """Get value from deprecated environment variable with warning. |
| 464 | |
| 465 | Args: |
| 466 | env_name: Name of the deprecated environment variable |
| 467 | removal_version: Version when it will be removed |
| 468 | field_name: Name of the field to suggest as alternative |
| 469 | |
| 470 | Returns: |
| 471 | The environment variable value if set, None otherwise |
| 472 | |
| 473 | """ |
| 474 | if envs.is_set(env_name): |
| 475 | value = os.environ.get(env_name) |
| 476 | alt_msg = f" Please use {field_name} instead." if field_name else "" |
| 477 | logger.warning_once( |
| 478 | "Using %s environment variable is deprecated and will be removed in %s.%s", |
| 479 | env_name, |
| 480 | removal_version, |
| 481 | alt_msg, |
| 482 | ) |
| 483 | return value |
| 484 | return None |
| 485 | |
| 486 | |
| 487 | def set_from_deprecated_env_if_set( |
| 488 | config: ConfigT, |
| 489 | env_name: str, |
| 490 | removal_version: str, |
| 491 | field_name: str, |
| 492 | to_bool: bool = False, |
| 493 | to_int: bool = False, |
| 494 | ) -> None: |
| 495 | """Set object field from deprecated environment variable with warning. |
| 496 | |
| 497 | Args: |
| 498 | config: Config object to set the field on |
| 499 | env_name: Name of the deprecated environment variable |
| 500 | removal_version: Version when the env var will be removed |
| 501 | field_name: Name of the field to set |
| 502 | to_bool: Whether to convert the environment variable value to boolean |
| 503 | to_int: Whether to convert the environment variable value to integer |
| 504 | Returns: |
| 505 | None |
| 506 | |
| 507 | """ |
| 508 | if to_bool and to_int: |
| 509 | raise ValueError("Cannot convert to both boolean and integer.") |
| 510 | |
| 511 | env_value = get_from_deprecated_env_if_set(env_name, removal_version, field_name) |
| 512 | if env_value is not None: |
| 513 | field_value: str | bool | int = env_value |
| 514 | if to_bool: |
| 515 | field_value = env_value.lower() in ("1", "true") |
| 516 | elif to_int: |
| 517 | field_value = int(env_value) |
| 518 | setattr(config, field_name, field_value) |