A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
Move `LoadConfig` from `config/__init__.py` to `config/load.py` (#24566)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
Sep 10, 2025 at 14:14 UTC
f36355abfd1d87d9e89fcc881fc99f3df72eeb9b
17 files changed
+137
-107
tests/lora/test_worker.py
+3
-3
@@ -6,9 +6,9 @@ import random
6
import tempfile
7
from unittest.mock import patch
8
9
-from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10
- ModelConfig, ParallelConfig, SchedulerConfig,
11
- VllmConfig)
9
+from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
10
+ ParallelConfig, SchedulerConfig, VllmConfig)
11
+from vllm.config.load import LoadConfig
12
from vllm.lora.models import LoRAMapping
13
from vllm.lora.request import LoRARequest
14
from vllm.v1.worker.gpu_worker import Worker
tests/model_executor/model_loader/test_registry.py
+2
-1
@@ -4,7 +4,8 @@
4
import pytest
5
from torch import nn
6
7
-from vllm.config import LoadConfig, ModelConfig
7
+from vllm.config import ModelConfig
8
+from vllm.config.load import LoadConfig
9
from vllm.model_executor.model_loader import (get_model_loader,
10
register_model_loader)
11
from vllm.model_executor.model_loader.base_loader import BaseModelLoader
tests/runai_model_streamer_test/test_runai_model_streamer_loader.py
+1
-1
@@ -2,7 +2,7 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
from vllm import SamplingParams
5
-from vllm.config import LoadConfig
5
+from vllm.config.load import LoadConfig
6
from vllm.model_executor.model_loader import get_model_loader
7
8
load_format = "runai_streamer"
tests/test_config.py
+3
-2
@@ -6,8 +6,9 @@ from dataclasses import MISSING, Field, asdict, dataclass, field
6
import pytest
7
8
from vllm.compilation.backends import VllmBackend
9
-from vllm.config import (LoadConfig, ModelConfig, PoolerConfig, VllmConfig,
10
- get_field, update_config)
9
+from vllm.config import (ModelConfig, PoolerConfig, VllmConfig, get_field,
10
+ update_config)
11
+from vllm.config.load import LoadConfig
12
from vllm.model_executor.layers.pooler import PoolingType
13
from vllm.platforms import current_platform
14
tests/v1/spec_decode/test_eagle.py
+2
-1
@@ -12,9 +12,10 @@ from tests.v1.attention.utils import (BatchSpec, _Backend,
12
create_common_attn_metadata,
13
create_standard_kv_cache_spec,
14
get_attention_backend)
15
-from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, ModelConfig,
15
+from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
16
ParallelConfig, SchedulerConfig, SpeculativeConfig,
17
VllmConfig)
18
+from vllm.config.load import LoadConfig
19
from vllm.model_executor.models.llama import LlamaForCausalLM
20
from vllm.platforms import current_platform
21
from vllm.v1.spec_decode.eagle import EagleProposer
vllm/config/__init__.py
+1
-88
@@ -34,6 +34,7 @@ from vllm.config.compilation import (CompilationConfig, CompilationLevel,
34
CUDAGraphMode, PassConfig)
35
from vllm.config.kv_events import KVEventsConfig
36
from vllm.config.kv_transfer import KVTransferConfig
37
+from vllm.config.load import LoadConfig
38
from vllm.config.parallel import (DistributedExecutorBackend, EPLBConfig,
39
ParallelConfig)
40
from vllm.config.scheduler import SchedulerConfig, SchedulerPolicy
@@ -64,8 +65,6 @@ if TYPE_CHECKING:
65
from vllm.model_executor.layers.quantization import QuantizationMethods
66
from vllm.model_executor.layers.quantization.base_config import (
67
QuantizationConfig)
67
- from vllm.model_executor.model_loader import LoadFormats
68
- from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
68
from vllm.v1.sample.logits_processor import LogitsProcessor
69
70
HfOverrides = Union[dict, Callable[[type], type]]
@@ -75,8 +74,6 @@ else:
74
QuantizationConfig = Any
75
QuantizationMethods = Any
76
BaseModelLoader = Any
78
- LoadFormats = Any
79
- TensorizerConfig = Any
77
LogitsProcessor = Any
78
HfOverrides = Union[dict[str, Any], Callable[[type], type]]
79
@@ -1801,90 +1798,6 @@ class ModelConfig:
1798
return max_model_len
1799
1800
1804
-@config
1805
-@dataclass
1806
-class LoadConfig:
1807
- """Configuration for loading the model weights."""
1808
-
1809
- load_format: Union[str, LoadFormats] = "auto"
1810
- """The format of the model weights to load:\n
1811
- - "auto" will try to load the weights in the safetensors format and fall
1812
- back to the pytorch bin format if safetensors format is not available.\n
1813
- - "pt" will load the weights in the pytorch bin format.\n
1814
- - "safetensors" will load the weights in the safetensors format.\n
1815
- - "npcache" will load the weights in pytorch format and store a numpy cache
1816
- to speed up the loading.\n
1817
- - "dummy" will initialize the weights with random values, which is mainly
1818
- for profiling.\n
1819
- - "tensorizer" will use CoreWeave's tensorizer library for fast weight
1820
- loading. See the Tensorize vLLM Model script in the Examples section for
1821
- more information.\n
1822
- - "runai_streamer" will load the Safetensors weights using Run:ai Model
1823
- Streamer.\n
1824
- - "bitsandbytes" will load the weights using bitsandbytes quantization.\n
1825
- - "sharded_state" will load weights from pre-sharded checkpoint files,
1826
- supporting efficient loading of tensor-parallel models.\n
1827
- - "gguf" will load weights from GGUF format files (details specified in
1828
- https://github.com/ggml-org/ggml/blob/master/docs/gguf.md).\n
1829
- - "mistral" will load weights from consolidated safetensors files used by
1830
- Mistral models.
1831
- - Other custom values can be supported via plugins."""
1832
- download_dir: Optional[str] = None
1833
- """Directory to download and load the weights, default to the default
1834
- cache directory of Hugging Face."""
1835
- model_loader_extra_config: Union[dict, TensorizerConfig] = field(
1836
- default_factory=dict)
1837
- """Extra config for model loader. This will be passed to the model loader
1838
- corresponding to the chosen load_format."""
1839
- device: Optional[str] = None
1840
- """Device to which model weights will be loaded, default to
1841
- device_config.device"""
1842
- ignore_patterns: Optional[Union[list[str], str]] = None
1843
- """The list of patterns to ignore when loading the model. Default to
1844
- "original/**/*" to avoid repeated loading of llama's checkpoints."""
1845
- use_tqdm_on_load: bool = True
1846
- """Whether to enable tqdm for showing progress bar when loading model
1847
- weights."""
1848
- pt_load_map_location: Union[str, dict[str, str]] = "cpu"
1849
- """
1850
- pt_load_map_location: the map location for loading pytorch checkpoint, to
1851
- support loading checkpoints can only be loaded on certain devices like
1852
- "cuda", this is equivalent to {"": "cuda"}. Another supported format is
1853
- mapping from different devices like from GPU 1 to GPU 0:
1854
- {"cuda:1": "cuda:0"}. Note that when passed from command line, the strings
1855
- in dictionary needs to be double quoted for json parsing. For more details,
1856
- see original doc for `map_location` in https://pytorch.org/docs/stable/generated/torch.load.html
1857
- """
1858
-
1859
- def compute_hash(self) -> str:
1860
- """
1861
- WARNING: Whenever a new field is added to this config,
1862
- ensure that it is included in the factors list if
1863
- it affects the computation graph.
1864
-
1865
- Provide a hash that uniquely identifies all the configs
1866
- that affect the structure of the computation
1867
- graph from input ids/embeddings to the final hidden states,
1868
- excluding anything before input ids/embeddings and after
1869
- the final hidden states.
1870
- """
1871
- # no factors to consider.
1872
- # this config will not affect the computation graph.
1873
- factors: list[Any] = []
1874
- hash_str = hashlib.md5(str(factors).encode(),
1875
- usedforsecurity=False).hexdigest()
1876
- return hash_str
1877
-
1878
- def __post_init__(self):
1879
- self.load_format = self.load_format.lower()
1880
- if self.ignore_patterns is not None and len(self.ignore_patterns) > 0:
1881
- logger.info(
1882
- "Ignoring the following patterns when downloading weights: %s",
1883
- self.ignore_patterns)
1884
- else:
1885
- self.ignore_patterns = ["original/**/*"]
1886
-
1887
-
1801
Device = Literal["auto", "cuda", "cpu", "tpu", "xpu"]
1802
1803
vllm/config/load.py
new
+104
@@ -0,0 +1,104 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+import hashlib
5
+from dataclasses import field
6
+from typing import TYPE_CHECKING, Any, Optional, Union
7
+
8
+from pydantic.dataclasses import dataclass
9
+
10
+from vllm.config.utils import config
11
+from vllm.logger import init_logger
12
+
13
+if TYPE_CHECKING:
14
+ from vllm.model_executor.model_loader import LoadFormats
15
+ from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
16
+else:
17
+ LoadFormats = Any
18
+ TensorizerConfig = Any
19
+
20
+logger = init_logger(__name__)
21
+
22
+
23
+@config
24
+@dataclass
25
+class LoadConfig:
26
+ """Configuration for loading the model weights."""
27
+
28
+ load_format: Union[str, LoadFormats] = "auto"
29
+ """The format of the model weights to load:\n
30
+ - "auto" will try to load the weights in the safetensors format and fall
31
+ back to the pytorch bin format if safetensors format is not available.\n
32
+ - "pt" will load the weights in the pytorch bin format.\n
33
+ - "safetensors" will load the weights in the safetensors format.\n
34
+ - "npcache" will load the weights in pytorch format and store a numpy cache
35
+ to speed up the loading.\n
36
+ - "dummy" will initialize the weights with random values, which is mainly
37
+ for profiling.\n
38
+ - "tensorizer" will use CoreWeave's tensorizer library for fast weight
39
+ loading. See the Tensorize vLLM Model script in the Examples section for
40
+ more information.\n
41
+ - "runai_streamer" will load the Safetensors weights using Run:ai Model
42
+ Streamer.\n
43
+ - "bitsandbytes" will load the weights using bitsandbytes quantization.\n
44
+ - "sharded_state" will load weights from pre-sharded checkpoint files,
45
+ supporting efficient loading of tensor-parallel models.\n
46
+ - "gguf" will load weights from GGUF format files (details specified in
47
+ https://github.com/ggml-org/ggml/blob/master/docs/gguf.md).\n
48
+ - "mistral" will load weights from consolidated safetensors files used by
49
+ Mistral models.
50
+ - Other custom values can be supported via plugins."""
51
+ download_dir: Optional[str] = None
52
+ """Directory to download and load the weights, default to the default
53
+ cache directory of Hugging Face."""
54
+ model_loader_extra_config: Union[dict, TensorizerConfig] = field(
55
+ default_factory=dict)
56
+ """Extra config for model loader. This will be passed to the model loader
57
+ corresponding to the chosen load_format."""
58
+ device: Optional[str] = None
59
+ """Device to which model weights will be loaded, default to
60
+ device_config.device"""
61
+ ignore_patterns: Optional[Union[list[str], str]] = None
62
+ """The list of patterns to ignore when loading the model. Default to
63
+ "original/**/*" to avoid repeated loading of llama's checkpoints."""
64
+ use_tqdm_on_load: bool = True
65
+ """Whether to enable tqdm for showing progress bar when loading model
66
+ weights."""
67
+ pt_load_map_location: Union[str, dict[str, str]] = "cpu"
68
+ """
69
+ pt_load_map_location: the map location for loading pytorch checkpoint, to
70
+ support loading checkpoints can only be loaded on certain devices like
71
+ "cuda", this is equivalent to {"": "cuda"}. Another supported format is
72
+ mapping from different devices like from GPU 1 to GPU 0:
73
+ {"cuda:1": "cuda:0"}. Note that when passed from command line, the strings
74
+ in dictionary needs to be double quoted for json parsing. For more details,
75
+ see original doc for `map_location` in https://pytorch.org/docs/stable/generated/torch.load.html
76
+ """
77
+
78
+ def compute_hash(self) -> str:
79
+ """
80
+ WARNING: Whenever a new field is added to this config,
81
+ ensure that it is included in the factors list if
82
+ it affects the computation graph.
83
+
84
+ Provide a hash that uniquely identifies all the configs
85
+ that affect the structure of the computation
86
+ graph from input ids/embeddings to the final hidden states,
87
+ excluding anything before input ids/embeddings and after
88
+ the final hidden states.
89
+ """
90
+ # no factors to consider.
91
+ # this config will not affect the computation graph.
92
+ factors: list[Any] = []
93
+ hash_str = hashlib.md5(str(factors).encode(),
94
+ usedforsecurity=False).hexdigest()
95
+ return hash_str
96
+
97
+ def __post_init__(self):
98
+ self.load_format = self.load_format.lower()
99
+ if self.ignore_patterns is not None and len(self.ignore_patterns) > 0:
100
+ logger.info(
101
+ "Ignoring the following patterns when downloading weights: %s",
102
+ self.ignore_patterns)
103
+ else:
104
+ self.ignore_patterns = ["original/**/*"]
vllm/model_executor/model_loader/__init__.py
+3
-2
@@ -5,7 +5,8 @@ from typing import Literal, Optional
5
6
from torch import nn
7
8
-from vllm.config import LoadConfig, ModelConfig, VllmConfig
8
+from vllm.config import ModelConfig, VllmConfig
9
+from vllm.config.load import LoadConfig
10
from vllm.logger import init_logger
11
from vllm.model_executor.model_loader.base_loader import BaseModelLoader
12
from vllm.model_executor.model_loader.bitsandbytes_loader import (
@@ -67,7 +68,7 @@ def register_model_loader(load_format: str):
68
load_format (str): The model loader format name.
69
70
Examples:
70
- >>> from vllm.config import LoadConfig
71
+ >>> from vllm.config.load import LoadConfig
72
>>> from vllm.model_executor.model_loader import get_model_loader, register_model_loader
73
>>> from vllm.model_executor.model_loader.base_loader import BaseModelLoader
74
>>>
vllm/model_executor/model_loader/base_loader.py
+2
-1
@@ -5,7 +5,8 @@ from abc import ABC, abstractmethod
5
import torch
6
import torch.nn as nn
7
8
-from vllm.config import LoadConfig, ModelConfig, VllmConfig
8
+from vllm.config import ModelConfig, VllmConfig
9
+from vllm.config.load import LoadConfig
10
from vllm.logger import init_logger
11
from vllm.model_executor.model_loader.utils import (
12
initialize_model, process_weights_after_loading, set_default_torch_dtype)
vllm/model_executor/model_loader/bitsandbytes_loader.py
+2
-1
@@ -16,7 +16,8 @@ from packaging import version
16
from torch import nn
17
from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
18
19
-from vllm.config import LoadConfig, ModelConfig
19
+from vllm.config import ModelConfig
20
+from vllm.config.load import LoadConfig
21
from vllm.distributed import (get_tensor_model_parallel_rank,
22
get_tensor_model_parallel_world_size)
23
# yapf: enable
vllm/model_executor/model_loader/default_loader.py
+2
-1
@@ -11,7 +11,8 @@ import torch
11
from torch import nn
12
from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
13
14
-from vllm.config import LoadConfig, ModelConfig
14
+from vllm.config import ModelConfig
15
+from vllm.config.load import LoadConfig
16
from vllm.logger import init_logger
17
from vllm.model_executor.model_loader.base_loader import BaseModelLoader
18
from vllm.model_executor.model_loader.weight_utils import (
vllm/model_executor/model_loader/dummy_loader.py
+2
-1
@@ -2,7 +2,8 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import torch.nn as nn
4
5
-from vllm.config import LoadConfig, ModelConfig
5
+from vllm.config import ModelConfig
6
+from vllm.config.load import LoadConfig
7
from vllm.model_executor.model_loader.base_loader import BaseModelLoader
8
from vllm.model_executor.model_loader.weight_utils import (
9
initialize_dummy_weights)
vllm/model_executor/model_loader/gguf_loader.py
+2
-1
@@ -9,7 +9,8 @@ import torch.nn as nn
9
from huggingface_hub import hf_hub_download
10
from transformers import AutoModelForCausalLM
11
12
-from vllm.config import LoadConfig, ModelConfig, VllmConfig
12
+from vllm.config import ModelConfig, VllmConfig
13
+from vllm.config.load import LoadConfig
14
from vllm.model_executor.model_loader.base_loader import BaseModelLoader
15
from vllm.model_executor.model_loader.utils import (
16
initialize_model, process_weights_after_loading, set_default_torch_dtype)
vllm/model_executor/model_loader/runai_streamer_loader.py
+2
-1
@@ -9,7 +9,8 @@ import torch
9
from torch import nn
10
from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
11
12
-from vllm.config import LoadConfig, ModelConfig
12
+from vllm.config import ModelConfig
13
+from vllm.config.load import LoadConfig
14
from vllm.model_executor.model_loader.base_loader import BaseModelLoader
15
from vllm.model_executor.model_loader.weight_utils import (
16
download_safetensors_index_file_from_hf, download_weights_from_hf,
vllm/model_executor/model_loader/sharded_state_loader.py
+2
-1
@@ -10,7 +10,8 @@ from typing import Any, Optional
10
import torch
11
from torch import nn
12
13
-from vllm.config import LoadConfig, ModelConfig
13
+from vllm.config import ModelConfig
14
+from vllm.config.load import LoadConfig
15
from vllm.logger import init_logger
16
from vllm.model_executor.model_loader.base_loader import BaseModelLoader
17
from vllm.model_executor.model_loader.weight_utils import (
vllm/model_executor/model_loader/tensorizer_loader.py
+2
-1
@@ -8,7 +8,8 @@ from typing import Union
8
import torch
9
from torch import nn
10
11
-from vllm.config import LoadConfig, ModelConfig, ParallelConfig, VllmConfig
11
+from vllm.config import ModelConfig, ParallelConfig, VllmConfig
12
+from vllm.config.load import LoadConfig
13
from vllm.logger import init_logger
14
from vllm.model_executor.model_loader.base_loader import BaseModelLoader
15
from vllm.model_executor.model_loader.tensorizer import (
vllm/model_executor/model_loader/weight_utils.py
+2
-1
@@ -23,7 +23,8 @@ from safetensors.torch import load_file, safe_open, save_file
23
from tqdm.auto import tqdm
24
25
from vllm import envs
26
-from vllm.config import LoadConfig, ModelConfig
26
+from vllm.config import ModelConfig
27
+from vllm.config.load import LoadConfig
28
from vllm.distributed import get_tensor_model_parallel_rank
29
from vllm.logger import init_logger
30
from vllm.model_executor.layers.quantization import (QuantizationConfig,