A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 18:30 UTC
[Doc] add load_format items in docs (#14804)
Signed-off-by: wwl2755 <wangwenlong2755@gmail.com>
wwl2755 committed
Mar 22, 2025 at 00:36 UTC
1c2bec0f8215f57ae3fd40286d944b7201948430
2 files changed
+14
-2
vllm/config.py
+6
@@ -1294,6 +1294,12 @@ class LoadConfig:
1294
"tensorizer" will use CoreWeave's tensorizer library for
1295
fast weight loading.
1296
"bitsandbytes" will load nf4 type weights.
1297
+ "sharded_state" will load weights from pre-sharded checkpoint files,
1298
+ supporting efficient loading of tensor-parallel models.
1299
+ "gguf" will load weights from GGUF format files.
1300
+ "mistral" will load weights from consolidated safetensors files used
1301
+ by Mistral models.
1302
+ "runai_streamer" will load weights from RunAI streamer format files.
1303
model_loader_extra_config: The extra config for the model loader.
1304
ignore_patterns: The list of patterns to ignore when loading the model.
1305
Default to "original/**/*" to avoid repeated loading of llama's
vllm/engine/arg_utils.py
+8
-2
@@ -339,9 +339,15 @@ class EngineArgs:
339
'CoreWeave. See the Tensorize vLLM Model script in the Examples '
340
'section for more information.\n'
341
'* "runai_streamer" will load the Safetensors weights using Run:ai'
342
- 'Model Streamer \n'
342
+ 'Model Streamer.\n'
343
'* "bitsandbytes" will load the weights using bitsandbytes '
344
- 'quantization.\n')
344
+ 'quantization.\n'
345
+ '* "sharded_state" will load weights from pre-sharded checkpoint '
346
+ 'files, supporting efficient loading of tensor-parallel models\n'
347
+ '* "gguf" will load weights from GGUF format files (details '
348
+ 'specified in https://github.com/ggml-org/ggml/blob/master/docs/gguf.md).\n'
349
+ '* "mistral" will load weights from consolidated safetensors files '
350
+ 'used by Mistral models.\n')
351
parser.add_argument(
352
'--config-format',
353
default=EngineArgs.config_format,