| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Utilities for selecting and loading models.""" |
| 4 | |
| 5 | import inspect |
| 6 | import warnings |
| 7 | from contextlib import contextmanager, nullcontext |
| 8 | from typing import Any |
| 9 | |
| 10 | import torch |
| 11 | from torch import nn |
| 12 | from typing_extensions import assert_never |
| 13 | |
| 14 | import vllm.envs as envs |
| 15 | from vllm.config import ( |
| 16 | LoadConfig, |
| 17 | ModelConfig, |
| 18 | VllmConfig, |
| 19 | replace, |
| 20 | set_current_vllm_config, |
| 21 | ) |
| 22 | from vllm.logger import init_logger |
| 23 | from vllm.model_executor.layers.attention import is_deferred_attention_layer |
| 24 | from vllm.model_executor.layers.quantization.base_config import ( |
| 25 | QuantizationConfig, |
| 26 | QuantizeMethodBase, |
| 27 | ) |
| 28 | from vllm.model_executor.model_loader.reload import ( |
| 29 | record_metadata_for_reloading, |
| 30 | set_torchao_reload_attrs, |
| 31 | ) |
| 32 | from vllm.model_executor.model_loader.weight_cache.utils import ( |
| 33 | is_draft_model_cacheable, |
| 34 | ) |
| 35 | from vllm.model_executor.model_loader.weight_tying import maybe_retie_word_embeddings |
| 36 | from vllm.model_executor.models.interfaces import SupportsQuant |
| 37 | from vllm.model_executor.utils import is_weights_pre_processed |
| 38 | from vllm.tracing import instrument |
| 39 | from vllm.utils.mem_utils import release_device_memory_under_pressure |
| 40 | from vllm.utils.platform_utils import is_pin_memory_available |
| 41 | from vllm.utils.torch_utils import get_accelerator_view_from_cpu_tensor |
| 42 | |
| 43 | logger = init_logger(__name__) |
| 44 | |
| 45 | |
| 46 | def get_draft_load_config(vllm_config: VllmConfig) -> LoadConfig: |
| 47 | """Get load config for the speculative draft model.""" |
| 48 | speculative_config = vllm_config.speculative_config |
| 49 | if ( |
| 50 | speculative_config is not None |
| 51 | and speculative_config.draft_load_config is not None |
| 52 | ): |
| 53 | return speculative_config.draft_load_config |
| 54 | load_config = vllm_config.load_config |
| 55 | if load_config is not None and load_config.load_format != "ipc_cache": |
| 56 | return load_config |
| 57 | kwargs = ( |
| 58 | # Route the draft to the daemon's draft group. |
| 59 | { |
| 60 | "model_loader_extra_config": { |
| 61 | **load_config.model_loader_extra_config, |
| 62 | "is_draft": True, |
| 63 | } |
| 64 | } |
| 65 | if is_draft_model_cacheable(speculative_config) |
| 66 | # No daemon draft group for this method; load from disk instead of |
| 67 | # hitting the target daemon with a mismatching fingerprint. |
| 68 | else {"load_format": "auto", "model_loader_extra_config": {}} |
| 69 | ) |
| 70 | return replace(load_config, **kwargs) |
| 71 | |
| 72 | |
| 73 | @instrument(span_name="Initialize model") |
| 74 | def initialize_model( |
| 75 | vllm_config: VllmConfig, |
| 76 | *, |
| 77 | prefix: str = "", |
| 78 | model_class: type[nn.Module] | None = None, |
| 79 | model_config: ModelConfig | None = None, |
| 80 | ) -> nn.Module: |
| 81 | """Initialize a model with the given configurations.""" |
| 82 | if model_config is None: |
| 83 | model_config = vllm_config.model_config |
| 84 | if model_class is None: |
| 85 | model_class, _ = get_model_architecture(model_config) |
| 86 | |
| 87 | if vllm_config.quant_config is not None: |
| 88 | configure_quant_config(vllm_config.quant_config, model_class) |
| 89 | |
| 90 | signatures = inspect.signature(model_class.__init__) |
| 91 | all_params = [param.name for param in signatures.parameters.values()] |
| 92 | if "vllm_config" in all_params and "prefix" in all_params: |
| 93 | # new-style model class |
| 94 | with set_current_vllm_config(vllm_config, check_compile=True, prefix=prefix): |
| 95 | model = model_class(vllm_config=vllm_config, prefix=prefix) |
| 96 | record_metadata_for_reloading(model) |
| 97 | return model |
| 98 | |
| 99 | msg = ( |
| 100 | "vLLM model class should accept `vllm_config` and `prefix` as " |
| 101 | "input arguments. Possibly you have an old-style model class" |
| 102 | " registered from out of tree and it is used for new vLLM version. " |
| 103 | "Check https://docs.vllm.ai/en/latest/design/arch_overview.html " |
| 104 | "for the design and update the model class accordingly." |
| 105 | ) |
| 106 | warnings.warn(msg, DeprecationWarning, stacklevel=2) |
| 107 | |
| 108 | logger.warning( |
| 109 | "Trying to guess the arguments for old-style model class %s", |
| 110 | model_class, |
| 111 | ) |
| 112 | # try to be compatible with old-style model class |
| 113 | kwargs: dict[str, Any] = {} |
| 114 | if "prefix" in all_params: |
| 115 | kwargs["prefix"] = prefix |
| 116 | if "config" in all_params: |
| 117 | kwargs["config"] = model_config.hf_config |
| 118 | if "cache_config" in all_params: |
| 119 | kwargs["cache_config"] = vllm_config.cache_config |
| 120 | if "quant_config" in all_params: |
| 121 | kwargs["quant_config"] = vllm_config.quant_config |
| 122 | if "lora_config" in all_params: |
| 123 | kwargs["lora_config"] = vllm_config.lora_config |
| 124 | if "scheduler_config" in all_params: |
| 125 | kwargs["scheduler_config"] = vllm_config.scheduler_config |
| 126 | with set_current_vllm_config(vllm_config, check_compile=True, prefix=prefix): |
| 127 | model = model_class(**kwargs) |
| 128 | record_metadata_for_reloading(model) |
| 129 | |
| 130 | return model |
| 131 | |
| 132 | |
| 133 | def process_weights_after_loading( |
| 134 | model: nn.Module, model_config: ModelConfig, target_device: torch.device |
| 135 | ) -> None: |
| 136 | """Post-process loaded weights into runtime format. |
| 137 | |
| 138 | Under ``weights_already_processed`` (weight cache IPC loader), quant |
| 139 | methods skip tensor transforms and must declare |
| 140 | ``supports_pre_processed_weights``, otherwise this raises ``RuntimeError``. |
| 141 | """ |
| 142 | # Reclaim memory when an explicit lm_head has been |
| 143 | # loaded, but it is identical to the input embeddings. |
| 144 | maybe_retie_word_embeddings(model, model_config) |
| 145 | |
| 146 | for name, module in model.named_modules(): |
| 147 | quant_method = getattr(module, "quant_method", None) |
| 148 | if isinstance(quant_method, QuantizeMethodBase): |
| 149 | if ( |
| 150 | is_weights_pre_processed() |
| 151 | and not quant_method.supports_pre_processed_weights |
| 152 | ): |
| 153 | raise RuntimeError( |
| 154 | f"layer {name or '<root>'}: {type(quant_method).__name__} " |
| 155 | "does not support pre-processed weights" |
| 156 | ) |
| 157 | # When quant methods need to process weights after loading |
| 158 | # (for repacking, quantizing, etc), they typically expect parameters |
| 159 | # to be on the global target device. This scope is for the |
| 160 | # case where cpu offloading is used, where we will move the |
| 161 | # parameters onto device for processing and back off after. |
| 162 | # Methods that can process weights in place (e.g. PLE scale |
| 163 | # validation) set requires_device_loading=False to skip this move. |
| 164 | loading_context = ( |
| 165 | device_loading_context(module, target_device) |
| 166 | if quant_method.requires_device_loading |
| 167 | else nullcontext() |
| 168 | ) |
| 169 | with loading_context: |
| 170 | quant_method.process_weights_after_loading(module) |
| 171 | # process_weights_after_loading may swap in freshly-created |
| 172 | # Parameters (e.g. FP8 requantization), which are stamped with the |
| 173 | # global rank in BasevLLMParameter.__init__. Re-reconcile their TP |
| 174 | # state to the layer so a later weight reload / RL weight-refit |
| 175 | # narrows replicated (disable_tp) weights at the correct offset. |
| 176 | if hasattr(module, "update_param_tp_status"): |
| 177 | module.update_param_tp_status() |
| 178 | # Repacking transients above can leave large amounts of memory in |
| 179 | # the caching allocator, which starves the OS on UMA devices. |
| 180 | release_device_memory_under_pressure(target_device) |
| 181 | |
| 182 | # Initialize post-load attention weights for any attention layer and MM |
| 183 | # encoder. NOTE: Happens after other modules so we can easily decompress |
| 184 | # weights. |
| 185 | for _, module in model.named_modules(): |
| 186 | if is_deferred_attention_layer(module): |
| 187 | # TODO(lucas): see if there is a way to unify the signatures |
| 188 | # of process_weights_after_loading |
| 189 | with device_loading_context(module, target_device): |
| 190 | module.process_weights_after_loading(model_config.dtype) |
| 191 | |
| 192 | # Model-level post-load hook, after the per-layer quant finalize. |
| 193 | if hasattr(model, "process_weights_after_loading"): |
| 194 | model.process_weights_after_loading() |
| 195 | |
| 196 | # Needed for torchao model reloading via model.reload_weights |
| 197 | # @kylesayrs @jerryzh168 this can be removed if callers move to `reload_weights` |
| 198 | if model_config.quantization == "torchao": |
| 199 | set_torchao_reload_attrs(model, model_config) |
| 200 | |
| 201 | |
| 202 | @contextmanager |
| 203 | def device_loading_context(module: torch.nn.Module, target_device: torch.device): |
| 204 | if target_device.type == "cpu": |
| 205 | # If target is CPU, no need to move anything |
| 206 | yield module |
| 207 | return |
| 208 | |
| 209 | cpu_params: set[str] = set() |
| 210 | uva_offloaded_parameters: list[str] = [] |
| 211 | |
| 212 | # Store which parameters are on CPU and move them to the GPU |
| 213 | for name, p in module.named_parameters(): |
| 214 | if p.device.type == "cpu": |
| 215 | cpu_params.add(name) |
| 216 | p.data = p.data.to(target_device) |
| 217 | if getattr(p, "_vllm_is_uva_offloaded", False): |
| 218 | uva_offloaded_parameters.append(name) |
| 219 | # Parameters already on target device are not touched |
| 220 | |
| 221 | try: |
| 222 | yield module |
| 223 | |
| 224 | finally: |
| 225 | use_pin_memory = ( |
| 226 | is_pin_memory_available() |
| 227 | and not envs.VLLM_WEIGHT_OFFLOADING_DISABLE_PIN_MEMORY |
| 228 | ) |
| 229 | # Restore the CPU-resident parameters, ignoring new parameters. |
| 230 | for name, p in module.named_parameters(): |
| 231 | if name in cpu_params: |
| 232 | p.data = torch.empty_like( |
| 233 | p.data, device="cpu", pin_memory=use_pin_memory |
| 234 | ).copy_(p.data) |
| 235 | |
| 236 | # parameter is UVA offloaded, but was replaced with a new device tensor |
| 237 | # re-offload it to CPU using UVA |
| 238 | if name in uva_offloaded_parameters and not getattr( |
| 239 | p, "_vllm_is_uva_offloaded", False |
| 240 | ): |
| 241 | cpu_data = torch.empty_like( |
| 242 | p.data, device="cpu", pin_memory=use_pin_memory |
| 243 | ).copy_(p.data) |
| 244 | p.data = get_accelerator_view_from_cpu_tensor(cpu_data) |
| 245 | p._vllm_is_uva_offloaded = True |
| 246 | |
| 247 | |
| 248 | _MODEL_ARCH_BY_HASH = dict[int, tuple[type[nn.Module], str]]() |
| 249 | """Caches the outputs of `_get_model_architecture`.""" |
| 250 | |
| 251 | |
| 252 | def _get_model_architecture(model_config: ModelConfig) -> tuple[type[nn.Module], str]: |
| 253 | from vllm.model_executor.models.adapters import as_embedding_model, as_seq_cls_model |
| 254 | |
| 255 | architectures = getattr(model_config.hf_config, "architectures", None) or [] |
| 256 | |
| 257 | model_cls, arch = model_config.registry.resolve_model_cls( |
| 258 | architectures, |
| 259 | model_config=model_config, |
| 260 | ) |
| 261 | |
| 262 | if arch == model_config._get_transformers_backend_cls(): |
| 263 | assert model_config.model_impl != "vllm" |
| 264 | if model_config.model_impl == "auto": |
| 265 | logger.warning_once( |
| 266 | "%s has no vLLM implementation, falling back to Transformers " |
| 267 | "implementation. Some features may not be supported and " |
| 268 | "performance may not be optimal.", |
| 269 | arch, |
| 270 | ) |
| 271 | |
| 272 | convert_type = model_config.convert_type |
| 273 | if convert_type == "none": |
| 274 | pass |
| 275 | elif convert_type == "embed": |
| 276 | logger.debug_once("Converting to embedding model.") |
| 277 | model_cls = as_embedding_model(model_cls) |
| 278 | elif convert_type == "classify": |
| 279 | logger.debug_once("Converting to sequence classification model.") |
| 280 | model_cls = as_seq_cls_model(model_cls) |
| 281 | else: |
| 282 | assert_never(convert_type) |
| 283 | |
| 284 | return model_cls, arch |
| 285 | |
| 286 | |
| 287 | def get_model_architecture(model_config: ModelConfig) -> tuple[type[nn.Module], str]: |
| 288 | key = hash( |
| 289 | ( |
| 290 | model_config.model, |
| 291 | model_config.convert_type, |
| 292 | model_config.runner_type, |
| 293 | model_config.trust_remote_code, |
| 294 | model_config.model_impl, |
| 295 | tuple(getattr(model_config.hf_config, "architectures", None) or []), |
| 296 | ) |
| 297 | ) |
| 298 | if key in _MODEL_ARCH_BY_HASH: |
| 299 | return _MODEL_ARCH_BY_HASH[key] |
| 300 | |
| 301 | model_cls_and_arch = _get_model_architecture(model_config) |
| 302 | _MODEL_ARCH_BY_HASH[key] = model_cls_and_arch |
| 303 | return model_cls_and_arch |
| 304 | |
| 305 | |
| 306 | def get_model_cls(model_config: ModelConfig) -> type[nn.Module]: |
| 307 | return get_model_architecture(model_config)[0] |
| 308 | |
| 309 | |
| 310 | def get_architecture_class_name(model_config: ModelConfig) -> str: |
| 311 | return get_model_architecture(model_config)[1] |
| 312 | |
| 313 | |
| 314 | def configure_quant_config( |
| 315 | quant_config: QuantizationConfig, model_class: type[nn.Module] |
| 316 | ): |
| 317 | """Pass packed_modules_mapping by reference to quant_config so that |
| 318 | quant_config can properly match fused modules |
| 319 | |
| 320 | Note that model attributes are passed by reference to quant_config, |
| 321 | enabling them to be updated by model_class.__new__ (ex. chatglm, qwen) |
| 322 | |
| 323 | Once the `SupportsQuant` mixin has been added to all models, this |
| 324 | function can be removed |
| 325 | """ |
| 326 | if not issubclass(model_class, SupportsQuant): |
| 327 | hf_to_vllm_mapper = getattr(model_class, "hf_to_vllm_mapper", None) |
| 328 | packed_mapping = getattr(model_class, "packed_modules_mapping", None) |
| 329 | |
| 330 | # pass mappings by reference to quant_config |
| 331 | if hf_to_vllm_mapper is not None: |
| 332 | quant_config.apply_vllm_mapper(hf_to_vllm_mapper.get_rename_mapper()) |
| 333 | if packed_mapping is not None: |
| 334 | quant_config.packed_modules_mapping = packed_mapping |