A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 23:30 UTC
DOC: Documentation pages fixes (#38125)
Signed-off-by: Mateusz Sokół <mat646@gmail.com>
Mateusz Sokół committed
Mar 26, 2026 at 09:55 UTC
b1cb1d3d2c2332e2bd30d63ba18ff6a4a046c122
1 file changed
+31
-26
vllm/config/load.py
+31
-26
@@ -24,38 +24,43 @@ class LoadConfig:
24
"""Configuration for loading the model weights."""
25
26
load_format: str | LoadFormats = "auto"
27
- """The format of the model weights to load:\n
27
+ """
28
+ The format of the model weights to load.
29
+
30
- "auto" will try to load the weights in the safetensors format and fall
29
- back to the pytorch bin format if safetensors format is not available.\n
30
- - "pt" will load the weights in the pytorch bin format.\n
31
- - "safetensors" will load the weights in the safetensors format.\n
31
+ back to the pytorch bin format if safetensors format is not available.
32
+ - "pt" will load the weights in the pytorch bin format.
33
+ - "safetensors" will load the weights in the safetensors format.
34
- "instanttensor" will load the Safetensors weights on CUDA devices using
33
- InstantTensor, which enables distributed loading with pipelined prefetching
34
- and fast direct I/O.\n
35
+ InstantTensor, which enables distributed loading with pipelined prefetching
36
+ and fast direct I/O.
37
- "npcache" will load the weights in pytorch format and store a numpy cache
36
- to speed up the loading.\n
38
+ to speed up the loading.
39
- "dummy" will initialize the weights with random values, which is mainly
38
- for profiling.\n
40
+ for profiling.
41
- "tensorizer" will use CoreWeave's tensorizer library for fast weight
40
- loading. See the Tensorize vLLM Model script in the Examples section for
41
- more information.\n
42
+ loading. See the Tensorize vLLM Model script in the Examples section for
43
+ more information.
44
- "runai_streamer" will load the Safetensors weights using Run:ai Model
43
- Streamer.\n
45
+ Streamer.
46
- "runai_streamer_sharded" will load weights from pre-sharded checkpoint
45
- files using Run:ai Model Streamer.\n
46
- - "bitsandbytes" will load the weights using bitsandbytes quantization.\n
47
+ files using Run:ai Model Streamer.
48
+ - "bitsandbytes" will load the weights using bitsandbytes quantization.
49
- "sharded_state" will load weights from pre-sharded checkpoint files,
48
- supporting efficient loading of tensor-parallel models.\n
50
+ supporting efficient loading of tensor-parallel models.
51
- "gguf" will load weights from GGUF format files (details specified in
50
- https://github.com/ggml-org/ggml/blob/master/docs/gguf.md).\n
52
+ https://github.com/ggml-org/ggml/blob/master/docs/gguf.md).
53
- "mistral" will load weights from consolidated safetensors files used by
52
- Mistral models.\n
53
- - Other custom values can be supported via plugins."""
54
+ Mistral models.\n
55
+ - Other custom values can be supported via plugins.
56
+ """
57
download_dir: str | None = None
58
"""Directory to download and load the weights, default to the default
59
cache directory of Hugging Face."""
60
safetensors_load_strategy: str | None = None
58
- """Specifies the loading strategy for safetensors weights.
61
+ """
62
+ Specifies the loading strategy for safetensors weights.
63
+
64
- None (default): Uses memory-mapped (lazy) loading. When an NFS
65
filesystem is detected and the total checkpoint size fits within 90%%
66
of available RAM, prefetching is enabled automatically.
@@ -72,7 +77,7 @@ class LoadConfig:
77
- "torchao": Weights are loaded in upfront and then reconstructed
78
into torchao tensor subclasses. This is used when the checkpoint
79
was quantized using torchao and saved using safetensors.
75
- Needs torchao >= 0.14.0
80
+ Needs `torchao >= 0.14.0`.
81
"""
82
model_loader_extra_config: dict | TensorizerConfig = Field(default_factory=dict)
83
"""Extra config for model loader. This will be passed to the model loader
@@ -88,13 +93,13 @@ class LoadConfig:
93
weights."""
94
pt_load_map_location: str | dict[str, str] = "cpu"
95
"""
91
- pt_load_map_location: the map location for loading pytorch checkpoint, to
92
- support loading checkpoints can only be loaded on certain devices like
93
- "cuda", this is equivalent to {"": "cuda"}. Another supported format is
94
- mapping from different devices like from GPU 1 to GPU 0:
95
- {"cuda:1": "cuda:0"}. Note that when passed from command line, the strings
96
- in dictionary needs to be double quoted for json parsing. For more details,
97
- see original doc for `map_location` in https://pytorch.org/docs/stable/generated/torch.load.html
96
+ The map location for loading pytorch checkpoint, to support loading
97
+ checkpoints can only be loaded on certain devices like "cuda", this
98
+ is equivalent to `{"": "cuda"}`. Another supported format is mapping
99
+ from different devices like from GPU 1 to GPU 0: `{"cuda:1": "cuda:0"}`.
100
+ Note that when passed from command line, the strings in dictionary
101
+ need to be double quoted for json parsing. For more details, see
102
+ the original doc for `map_location` parameter in [`torch.load`][] parameter.
103
"""
104
105
def compute_hash(self) -> str: