A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 23:45 UTC
feat(gguf_loader): accept HF repo paths & URLs for GGUF (#20793)
Signed-off-by: Hardik <hardikgupta1999@gmail.com> Signed-off-by: Isotr0py <2037008807@qq.com> Co-authored-by: Isotr0py <2037008807@qq.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Hardik Gupta committed
Jul 23, 2025 at 20:21 UTC
11599b0e1ffdbe7f7e5f7d222dfbef69b41b3ad2
1 file changed
+12
-1
vllm/model_executor/model_loader/gguf_loader.py
+12
-1
@@ -6,6 +6,7 @@ from collections.abc import Generator
6
import gguf
7
import torch
8
import torch.nn as nn
9
+from huggingface_hub import hf_hub_download
10
from transformers import AutoModelForCausalLM
11
12
from vllm.config import LoadConfig, ModelConfig, VllmConfig
@@ -32,8 +33,18 @@ class GGUFModelLoader(BaseModelLoader):
33
def _prepare_weights(self, model_name_or_path: str):
34
if os.path.isfile(model_name_or_path):
35
return model_name_or_path
36
+ # for raw HTTPS link
37
+ if model_name_or_path.startswith(
38
+ ("http://", "https://")) and model_name_or_path.endswith(".gguf"):
39
+ return hf_hub_download(url=model_name_or_path)
40
+ # repo id/filename.gguf
41
+ if "/" in model_name_or_path and model_name_or_path.endswith(".gguf"):
42
+ repo_id, filename = model_name_or_path.rsplit("/", 1)
43
+ return hf_hub_download(repo_id=repo_id, filename=filename)
44
else:
36
- raise ValueError(f"{model_name_or_path} is not a file.")
45
+ raise ValueError(
46
+ f"Unrecognised GGUF reference: {model_name_or_path} "
47
+ "(expected local file, raw URL, or <repo_id>/<filename>.gguf)")
48
49
def _get_gguf_weights_map(self, model_config: ModelConfig):
50
"""