A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 08:30 UTC
config.py: Clarify that only local GGUF checkpoints are supported. (#18623)
Signed-off-by: Mathieu Bordere <mathieu@letmetweakit.com>
Mathieu Borderé committed
May 24, 2025 at 02:46 UTC
45ab403a1f29f661262ebe651dde62cb8ed6c98b
1 file changed
+4
-1
vllm/transformers_utils/config.py
+4
-1
@@ -300,7 +300,10 @@ def get_config(
300
" - For Hugging Face models: ensure the presence of a "
301
"'config.json'.\n"
302
" - For Mistral models: ensure the presence of a "
303
- "'params.json'.\n").format(model=model)
303
+ "'params.json'.\n"
304
+ "3. For GGUF: pass the local path of the GGUF checkpoint.\n"
305
+ " Loading GGUF from a remote repo directly is not yet "
306
+ "supported.\n").format(model=model)
307
308
raise ValueError(error_message) from e
309