A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 13:30 UTC
[Feature]: Improve GGUF loading from HuggingFace user experience like repo_id:quant_type (#29137)
Signed-off-by: Injae Ryou <injaeryou@gmail.com> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Injae Ryou committed
Nov 25, 2025 at 23:28 UTC
794029f012066e6039b63566ef7d845af1ade831
10 files changed
+579
-36
tests/models/test_gguf_download.py
new
+240
@@ -0,0 +1,240 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+from unittest.mock import MagicMock, patch
5
+
6
+import pytest
7
+
8
+from vllm.config import ModelConfig
9
+from vllm.config.load import LoadConfig
10
+from vllm.model_executor.model_loader.gguf_loader import GGUFModelLoader
11
+from vllm.model_executor.model_loader.weight_utils import download_gguf
12
+
13
+
14
+class TestGGUFDownload:
15
+ """Test GGUF model downloading functionality."""
16
+
17
+ @patch("vllm.model_executor.model_loader.weight_utils.download_weights_from_hf")
18
+ def test_download_gguf_single_file(self, mock_download):
19
+ """Test downloading a single GGUF file."""
20
+ # Setup mock
21
+ mock_folder = "/tmp/mock_cache"
22
+ mock_download.return_value = mock_folder
23
+
24
+ # Mock glob to return a single file
25
+ with patch("glob.glob") as mock_glob:
26
+ mock_glob.side_effect = lambda pattern, **kwargs: (
27
+ [f"{mock_folder}/model-IQ1_S.gguf"] if "IQ1_S" in pattern else []
28
+ )
29
+
30
+ result = download_gguf("unsloth/Qwen3-0.6B-GGUF", "IQ1_S")
31
+
32
+ # Verify download_weights_from_hf was called with correct patterns
33
+ mock_download.assert_called_once_with(
34
+ model_name_or_path="unsloth/Qwen3-0.6B-GGUF",
35
+ cache_dir=None,
36
+ allow_patterns=[
37
+ "*-IQ1_S.gguf",
38
+ "*-IQ1_S-*.gguf",
39
+ "*/*-IQ1_S.gguf",
40
+ "*/*-IQ1_S-*.gguf",
41
+ ],
42
+ revision=None,
43
+ ignore_patterns=None,
44
+ )
45
+
46
+ # Verify result is the file path, not folder
47
+ assert result == f"{mock_folder}/model-IQ1_S.gguf"
48
+
49
+ @patch("vllm.model_executor.model_loader.weight_utils.download_weights_from_hf")
50
+ def test_download_gguf_sharded_files(self, mock_download):
51
+ """Test downloading sharded GGUF files."""
52
+ mock_folder = "/tmp/mock_cache"
53
+ mock_download.return_value = mock_folder
54
+
55
+ # Mock glob to return sharded files
56
+ with patch("glob.glob") as mock_glob:
57
+ mock_glob.side_effect = lambda pattern, **kwargs: (
58
+ [
59
+ f"{mock_folder}/model-Q2_K-00001-of-00002.gguf",
60
+ f"{mock_folder}/model-Q2_K-00002-of-00002.gguf",
61
+ ]
62
+ if "Q2_K" in pattern
63
+ else []
64
+ )
65
+
66
+ result = download_gguf("unsloth/gpt-oss-120b-GGUF", "Q2_K")
67
+
68
+ # Should return the first file after sorting
69
+ assert result == f"{mock_folder}/model-Q2_K-00001-of-00002.gguf"
70
+
71
+ @patch("vllm.model_executor.model_loader.weight_utils.download_weights_from_hf")
72
+ def test_download_gguf_subdir(self, mock_download):
73
+ """Test downloading GGUF files from subdirectory."""
74
+ mock_folder = "/tmp/mock_cache"
75
+ mock_download.return_value = mock_folder
76
+
77
+ with patch("glob.glob") as mock_glob:
78
+ mock_glob.side_effect = lambda pattern, **kwargs: (
79
+ [f"{mock_folder}/Q2_K/model-Q2_K.gguf"]
80
+ if "Q2_K" in pattern or "**/*.gguf" in pattern
81
+ else []
82
+ )
83
+
84
+ result = download_gguf("unsloth/gpt-oss-120b-GGUF", "Q2_K")
85
+
86
+ assert result == f"{mock_folder}/Q2_K/model-Q2_K.gguf"
87
+
88
+ @patch("vllm.model_executor.model_loader.weight_utils.download_weights_from_hf")
89
+ @patch("glob.glob", return_value=[])
90
+ def test_download_gguf_no_files_found(self, mock_glob, mock_download):
91
+ """Test error when no GGUF files are found."""
92
+ mock_folder = "/tmp/mock_cache"
93
+ mock_download.return_value = mock_folder
94
+
95
+ with pytest.raises(ValueError, match="Downloaded GGUF files not found"):
96
+ download_gguf("unsloth/Qwen3-0.6B-GGUF", "IQ1_S")
97
+
98
+
99
+class TestGGUFModelLoader:
100
+ """Test GGUFModelLoader class methods."""
101
+
102
+ @patch("os.path.isfile", return_value=True)
103
+ def test_prepare_weights_local_file(self, mock_isfile):
104
+ """Test _prepare_weights with local file."""
105
+ load_config = LoadConfig(load_format="gguf")
106
+ loader = GGUFModelLoader(load_config)
107
+
108
+ # Create a simple mock ModelConfig with only the model attribute
109
+ model_config = MagicMock()
110
+ model_config.model = "/path/to/model.gguf"
111
+
112
+ result = loader._prepare_weights(model_config)
113
+ assert result == "/path/to/model.gguf"
114
+ mock_isfile.assert_called_once_with("/path/to/model.gguf")
115
+
116
+ @patch("vllm.model_executor.model_loader.gguf_loader.hf_hub_download")
117
+ @patch("os.path.isfile", return_value=False)
118
+ def test_prepare_weights_https_url(self, mock_isfile, mock_hf_download):
119
+ """Test _prepare_weights with HTTPS URL."""
120
+ load_config = LoadConfig(load_format="gguf")
121
+ loader = GGUFModelLoader(load_config)
122
+
123
+ mock_hf_download.return_value = "/downloaded/model.gguf"
124
+
125
+ # Create a simple mock ModelConfig with only the model attribute
126
+ model_config = MagicMock()
127
+ model_config.model = "https://huggingface.co/model.gguf"
128
+
129
+ result = loader._prepare_weights(model_config)
130
+ assert result == "/downloaded/model.gguf"
131
+ mock_hf_download.assert_called_once_with(
132
+ url="https://huggingface.co/model.gguf"
133
+ )
134
+
135
+ @patch("vllm.model_executor.model_loader.gguf_loader.hf_hub_download")
136
+ @patch("os.path.isfile", return_value=False)
137
+ def test_prepare_weights_repo_filename(self, mock_isfile, mock_hf_download):
138
+ """Test _prepare_weights with repo_id/filename.gguf format."""
139
+ load_config = LoadConfig(load_format="gguf")
140
+ loader = GGUFModelLoader(load_config)
141
+
142
+ mock_hf_download.return_value = "/downloaded/model.gguf"
143
+
144
+ # Create a simple mock ModelConfig with only the model attribute
145
+ model_config = MagicMock()
146
+ model_config.model = "unsloth/Qwen3-0.6B-GGUF/model.gguf"
147
+
148
+ result = loader._prepare_weights(model_config)
149
+ assert result == "/downloaded/model.gguf"
150
+ mock_hf_download.assert_called_once_with(
151
+ repo_id="unsloth/Qwen3-0.6B-GGUF", filename="model.gguf"
152
+ )
153
+
154
+ @patch("vllm.config.model.get_hf_image_processor_config", return_value=None)
155
+ @patch("vllm.transformers_utils.config.file_or_path_exists", return_value=True)
156
+ @patch("vllm.config.model.get_config")
157
+ @patch("vllm.config.model.is_gguf", return_value=True)
158
+ @patch("vllm.model_executor.model_loader.gguf_loader.download_gguf")
159
+ @patch("os.path.isfile", return_value=False)
160
+ def test_prepare_weights_repo_quant_type(
161
+ self,
162
+ mock_isfile,
163
+ mock_download_gguf,
164
+ mock_is_gguf,
165
+ mock_get_config,
166
+ mock_file_exists,
167
+ mock_get_image_config,
168
+ ):
169
+ """Test _prepare_weights with repo_id:quant_type format."""
170
+ mock_hf_config = MagicMock()
171
+ mock_hf_config.architectures = ["Qwen3ForCausalLM"]
172
+
173
+ class MockTextConfig:
174
+ max_position_embeddings = 4096
175
+ sliding_window = None
176
+ model_type = "qwen3"
177
+ num_attention_heads = 32
178
+
179
+ mock_text_config = MockTextConfig()
180
+ mock_hf_config.get_text_config.return_value = mock_text_config
181
+ mock_hf_config.dtype = "bfloat16"
182
+ mock_get_config.return_value = mock_hf_config
183
+
184
+ load_config = LoadConfig(load_format="gguf")
185
+ loader = GGUFModelLoader(load_config)
186
+
187
+ mock_download_gguf.return_value = "/downloaded/model-IQ1_S.gguf"
188
+
189
+ model_config = ModelConfig(
190
+ model="unsloth/Qwen3-0.6B-GGUF:IQ1_S", tokenizer="Qwen/Qwen3-0.6B"
191
+ )
192
+ result = loader._prepare_weights(model_config)
193
+ # The actual result will be the downloaded file path from mock
194
+ assert result == "/downloaded/model-IQ1_S.gguf"
195
+ mock_download_gguf.assert_called_once_with(
196
+ "unsloth/Qwen3-0.6B-GGUF",
197
+ "IQ1_S",
198
+ cache_dir=None,
199
+ revision=None,
200
+ ignore_patterns=["original/**/*"],
201
+ )
202
+
203
+ @patch("vllm.config.model.get_hf_image_processor_config", return_value=None)
204
+ @patch("vllm.config.model.get_config")
205
+ @patch("vllm.config.model.is_gguf", return_value=False)
206
+ @patch("vllm.transformers_utils.utils.check_gguf_file", return_value=False)
207
+ @patch("os.path.isfile", return_value=False)
208
+ def test_prepare_weights_invalid_format(
209
+ self,
210
+ mock_isfile,
211
+ mock_check_gguf,
212
+ mock_is_gguf,
213
+ mock_get_config,
214
+ mock_get_image_config,
215
+ ):
216
+ """Test _prepare_weights with invalid format."""
217
+ mock_hf_config = MagicMock()
218
+ mock_hf_config.architectures = ["Qwen3ForCausalLM"]
219
+
220
+ class MockTextConfig:
221
+ max_position_embeddings = 4096
222
+ sliding_window = None
223
+ model_type = "qwen3"
224
+ num_attention_heads = 32
225
+
226
+ mock_text_config = MockTextConfig()
227
+ mock_hf_config.get_text_config.return_value = mock_text_config
228
+ mock_hf_config.dtype = "bfloat16"
229
+ mock_get_config.return_value = mock_hf_config
230
+
231
+ load_config = LoadConfig(load_format="gguf")
232
+ loader = GGUFModelLoader(load_config)
233
+
234
+ # Create ModelConfig with a valid repo_id to avoid validation errors
235
+ # Then test _prepare_weights with invalid format
236
+ model_config = ModelConfig(model="unsloth/Qwen3-0.6B")
237
+ # Manually set model to invalid format after creation
238
+ model_config.model = "invalid-format"
239
+ with pytest.raises(ValueError, match="Unrecognised GGUF reference"):
240
+ loader._prepare_weights(model_config)
tests/transformers_utils/test_utils.py
+146
@@ -1,11 +1,17 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+from pathlib import Path
4
+from unittest.mock import patch
5
6
+import pytest
7
8
from vllm.transformers_utils.utils import (
9
is_cloud_storage,
10
is_gcs,
11
+ is_gguf,
12
+ is_remote_gguf,
13
is_s3,
14
+ split_remote_gguf,
15
)
16
17
@@ -28,3 +34,143 @@ def test_is_cloud_storage():
34
assert is_cloud_storage("s3://model-path/path-to-model")
35
assert not is_cloud_storage("/unix/local/path")
36
assert not is_cloud_storage("nfs://nfs-fqdn.local")
37
+
38
+
39
+class TestIsRemoteGGUF:
40
+ """Test is_remote_gguf utility function."""
41
+
42
+ def test_is_remote_gguf_with_colon_and_slash(self):
43
+ """Test is_remote_gguf with repo_id:quant_type format."""
44
+ # Valid quant types
45
+ assert is_remote_gguf("unsloth/Qwen3-0.6B-GGUF:IQ1_S")
46
+ assert is_remote_gguf("user/repo:Q2_K")
47
+ assert is_remote_gguf("repo/model:Q4_K")
48
+ assert is_remote_gguf("repo/model:Q8_0")
49
+
50
+ # Invalid quant types should return False
51
+ assert not is_remote_gguf("repo/model:quant")
52
+ assert not is_remote_gguf("repo/model:INVALID")
53
+ assert not is_remote_gguf("repo/model:invalid_type")
54
+
55
+ def test_is_remote_gguf_without_colon(self):
56
+ """Test is_remote_gguf without colon."""
57
+ assert not is_remote_gguf("repo/model")
58
+ assert not is_remote_gguf("unsloth/Qwen3-0.6B-GGUF")
59
+
60
+ def test_is_remote_gguf_without_slash(self):
61
+ """Test is_remote_gguf without slash."""
62
+ assert not is_remote_gguf("model.gguf")
63
+ # Even with valid quant_type, no slash means not remote GGUF
64
+ assert not is_remote_gguf("model:IQ1_S")
65
+ assert not is_remote_gguf("model:quant")
66
+
67
+ def test_is_remote_gguf_local_path(self):
68
+ """Test is_remote_gguf with local file path."""
69
+ assert not is_remote_gguf("/path/to/model.gguf")
70
+ assert not is_remote_gguf("./model.gguf")
71
+
72
+ def test_is_remote_gguf_with_path_object(self):
73
+ """Test is_remote_gguf with Path object."""
74
+ assert is_remote_gguf(Path("unsloth/Qwen3-0.6B-GGUF:IQ1_S"))
75
+ assert not is_remote_gguf(Path("repo/model"))
76
+
77
+ def test_is_remote_gguf_with_http_https(self):
78
+ """Test is_remote_gguf with HTTP/HTTPS URLs."""
79
+ # HTTP/HTTPS URLs should return False even with valid quant_type
80
+ assert not is_remote_gguf("http://example.com/repo/model:IQ1_S")
81
+ assert not is_remote_gguf("https://huggingface.co/repo/model:Q2_K")
82
+ assert not is_remote_gguf("http://repo/model:Q4_K")
83
+ assert not is_remote_gguf("https://repo/model:Q8_0")
84
+
85
+ def test_is_remote_gguf_with_cloud_storage(self):
86
+ """Test is_remote_gguf with cloud storage paths."""
87
+ # Cloud storage paths should return False even with valid quant_type
88
+ assert not is_remote_gguf("s3://bucket/repo/model:IQ1_S")
89
+ assert not is_remote_gguf("gs://bucket/repo/model:Q2_K")
90
+ assert not is_remote_gguf("s3://repo/model:Q4_K")
91
+ assert not is_remote_gguf("gs://repo/model:Q8_0")
92
+
93
+
94
+class TestSplitRemoteGGUF:
95
+ """Test split_remote_gguf utility function."""
96
+
97
+ def test_split_remote_gguf_valid(self):
98
+ """Test split_remote_gguf with valid repo_id:quant_type format."""
99
+ repo_id, quant_type = split_remote_gguf("unsloth/Qwen3-0.6B-GGUF:IQ1_S")
100
+ assert repo_id == "unsloth/Qwen3-0.6B-GGUF"
101
+ assert quant_type == "IQ1_S"
102
+
103
+ repo_id, quant_type = split_remote_gguf("repo/model:Q2_K")
104
+ assert repo_id == "repo/model"
105
+ assert quant_type == "Q2_K"
106
+
107
+ def test_split_remote_gguf_with_path_object(self):
108
+ """Test split_remote_gguf with Path object."""
109
+ repo_id, quant_type = split_remote_gguf(Path("unsloth/Qwen3-0.6B-GGUF:IQ1_S"))
110
+ assert repo_id == "unsloth/Qwen3-0.6B-GGUF"
111
+ assert quant_type == "IQ1_S"
112
+
113
+ def test_split_remote_gguf_invalid(self):
114
+ """Test split_remote_gguf with invalid format."""
115
+ # Invalid format (no colon) - is_remote_gguf returns False
116
+ with pytest.raises(ValueError, match="Wrong GGUF model"):
117
+ split_remote_gguf("repo/model")
118
+
119
+ # Invalid quant type - is_remote_gguf returns False
120
+ with pytest.raises(ValueError, match="Wrong GGUF model"):
121
+ split_remote_gguf("repo/model:INVALID_TYPE")
122
+
123
+ # HTTP URL - is_remote_gguf returns False
124
+ with pytest.raises(ValueError, match="Wrong GGUF model"):
125
+ split_remote_gguf("http://repo/model:IQ1_S")
126
+
127
+ # Cloud storage - is_remote_gguf returns False
128
+ with pytest.raises(ValueError, match="Wrong GGUF model"):
129
+ split_remote_gguf("s3://bucket/repo/model:Q2_K")
130
+
131
+
132
+class TestIsGGUF:
133
+ """Test is_gguf utility function."""
134
+
135
+ @patch("vllm.transformers_utils.utils.check_gguf_file", return_value=True)
136
+ def test_is_gguf_with_local_file(self, mock_check_gguf):
137
+ """Test is_gguf with local GGUF file."""
138
+ assert is_gguf("/path/to/model.gguf")
139
+ assert is_gguf("./model.gguf")
140
+
141
+ def test_is_gguf_with_remote_gguf(self):
142
+ """Test is_gguf with remote GGUF format."""
143
+ # Valid remote GGUF format (repo_id:quant_type with valid quant_type)
144
+ assert is_gguf("unsloth/Qwen3-0.6B-GGUF:IQ1_S")
145
+ assert is_gguf("repo/model:Q2_K")
146
+ assert is_gguf("repo/model:Q4_K")
147
+
148
+ # Invalid quant_type should return False
149
+ assert not is_gguf("repo/model:quant")
150
+ assert not is_gguf("repo/model:INVALID")
151
+
152
+ @patch("vllm.transformers_utils.utils.check_gguf_file", return_value=False)
153
+ def test_is_gguf_false(self, mock_check_gguf):
154
+ """Test is_gguf returns False for non-GGUF models."""
155
+ assert not is_gguf("unsloth/Qwen3-0.6B")
156
+ assert not is_gguf("repo/model")
157
+ assert not is_gguf("model")
158
+
159
+ def test_is_gguf_edge_cases(self):
160
+ """Test is_gguf with edge cases."""
161
+ # Empty string
162
+ assert not is_gguf("")
163
+
164
+ # Only colon, no slash (even with valid quant_type)
165
+ assert not is_gguf("model:IQ1_S")
166
+
167
+ # Only slash, no colon
168
+ assert not is_gguf("repo/model")
169
+
170
+ # HTTP/HTTPS URLs
171
+ assert not is_gguf("http://repo/model:IQ1_S")
172
+ assert not is_gguf("https://repo/model:Q2_K")
173
+
174
+ # Cloud storage
175
+ assert not is_gguf("s3://bucket/repo/model:IQ1_S")
176
+ assert not is_gguf("gs://bucket/repo/model:Q2_K")
vllm/config/model.py
+12
-3
@@ -39,7 +39,12 @@ from vllm.transformers_utils.gguf_utils import (
39
maybe_patch_hf_config_from_gguf,
40
)
41
from vllm.transformers_utils.runai_utils import ObjectStorageModel, is_runai_obj_uri
42
-from vllm.transformers_utils.utils import check_gguf_file, maybe_model_redirect
42
+from vllm.transformers_utils.utils import (
43
+ is_gguf,
44
+ is_remote_gguf,
45
+ maybe_model_redirect,
46
+ split_remote_gguf,
47
+)
48
from vllm.utils.import_utils import LazyLoader
49
from vllm.utils.torch_utils import common_broadcastable_dtype
50
@@ -440,7 +445,8 @@ class ModelConfig:
445
self.model = maybe_model_redirect(self.model)
446
# The tokenizer is consistent with the model by default.
447
if self.tokenizer is None:
443
- if check_gguf_file(self.model):
448
+ # Check if this is a GGUF model (either local file or remote GGUF)
449
+ if is_gguf(self.model):
450
raise ValueError(
451
"Using a tokenizer is mandatory when loading a GGUF model. "
452
"Please specify the tokenizer path or name using the "
@@ -832,7 +838,10 @@ class ModelConfig:
838
self.tokenizer = object_storage_tokenizer.dir
839
840
def _get_encoder_config(self):
835
- return get_sentence_transformer_tokenizer_config(self.model, self.revision)
841
+ model = self.model
842
+ if is_remote_gguf(model):
843
+ model, _ = split_remote_gguf(model)
844
+ return get_sentence_transformer_tokenizer_config(model, self.revision)
845
846
def _verify_tokenizer_mode(self) -> None:
847
tokenizer_mode = cast(TokenizerMode, self.tokenizer_mode.lower())
vllm/engine/arg_utils.py
+3
-3
@@ -86,7 +86,7 @@ from vllm.transformers_utils.config import (
86
is_interleaved,
87
maybe_override_with_speculators,
88
)
89
-from vllm.transformers_utils.utils import check_gguf_file, is_cloud_storage
89
+from vllm.transformers_utils.utils import is_cloud_storage, is_gguf
90
from vllm.utils.argparse_utils import FlexibleArgumentParser
91
from vllm.utils.mem_constants import GiB_bytes
92
from vllm.utils.network_utils import get_ip
@@ -1148,8 +1148,8 @@ class EngineArgs:
1148
return engine_args
1149
1150
def create_model_config(self) -> ModelConfig:
1151
- # gguf file needs a specific model loader and doesn't use hf_repo
1152
- if check_gguf_file(self.model):
1151
+ # gguf file needs a specific model loader
1152
+ if is_gguf(self.model):
1153
self.quantization = self.load_format = "gguf"
1154
1155
# NOTE(woosuk): In V1, we use separate processes for workers (unless
vllm/model_executor/model_loader/gguf_loader.py
+22
-9
@@ -18,6 +18,7 @@ from vllm.model_executor.model_loader.utils import (
18
process_weights_after_loading,
19
)
20
from vllm.model_executor.model_loader.weight_utils import (
21
+ download_gguf,
22
get_gguf_extra_tensor_names,
23
get_gguf_weight_type_map,
24
gguf_quant_weights_iterator,
@@ -43,7 +44,8 @@ class GGUFModelLoader(BaseModelLoader):
44
f"load format {load_config.load_format}"
45
)
46
46
- def _prepare_weights(self, model_name_or_path: str):
47
+ def _prepare_weights(self, model_config: ModelConfig):
48
+ model_name_or_path = model_config.model
49
if os.path.isfile(model_name_or_path):
50
return model_name_or_path
51
# for raw HTTPS link
@@ -55,12 +57,23 @@ class GGUFModelLoader(BaseModelLoader):
57
if "/" in model_name_or_path and model_name_or_path.endswith(".gguf"):
58
repo_id, filename = model_name_or_path.rsplit("/", 1)
59
return hf_hub_download(repo_id=repo_id, filename=filename)
58
- else:
59
- raise ValueError(
60
- f"Unrecognised GGUF reference: {model_name_or_path} "
61
- "(expected local file, raw URL, or <repo_id>/<filename>.gguf)"
60
+ # repo_id:quant_type
61
+ elif "/" in model_name_or_path and ":" in model_name_or_path:
62
+ repo_id, quant_type = model_name_or_path.rsplit(":", 1)
63
+ return download_gguf(
64
+ repo_id,
65
+ quant_type,
66
+ cache_dir=self.load_config.download_dir,
67
+ revision=model_config.revision,
68
+ ignore_patterns=self.load_config.ignore_patterns,
69
)
70
71
+ raise ValueError(
72
+ f"Unrecognised GGUF reference: {model_name_or_path} "
73
+ "(expected local file, raw URL, <repo_id>/<filename>.gguf, "
74
+ "or <repo_id>:<quant_type>)"
75
+ )
76
+
77
def _get_gguf_weights_map(self, model_config: ModelConfig):
78
"""
79
GGUF uses this naming convention for their tensors from HF checkpoint:
@@ -244,7 +257,7 @@ class GGUFModelLoader(BaseModelLoader):
257
gguf_to_hf_name_map: dict[str, str],
258
) -> dict[str, str]:
259
weight_type_map = get_gguf_weight_type_map(
247
- model_config.model, gguf_to_hf_name_map
260
+ model_name_or_path, gguf_to_hf_name_map
261
)
262
is_multimodal = hasattr(model_config.hf_config, "vision_config")
263
if is_multimodal:
@@ -290,10 +303,10 @@ class GGUFModelLoader(BaseModelLoader):
303
yield from gguf_quant_weights_iterator(model_name_or_path, gguf_to_hf_name_map)
304
305
def download_model(self, model_config: ModelConfig) -> None:
293
- self._prepare_weights(model_config.model)
306
+ self._prepare_weights(model_config)
307
308
def load_weights(self, model: nn.Module, model_config: ModelConfig) -> None:
296
- local_model_path = self._prepare_weights(model_config.model)
309
+ local_model_path = self._prepare_weights(model_config)
310
gguf_weights_map = self._get_gguf_weights_map(model_config)
311
model.load_weights(
312
self._get_weights_iterator(model_config, local_model_path, gguf_weights_map)
@@ -303,7 +316,7 @@ class GGUFModelLoader(BaseModelLoader):
316
self, vllm_config: VllmConfig, model_config: ModelConfig
317
) -> nn.Module:
318
device_config = vllm_config.device_config
306
- local_model_path = self._prepare_weights(model_config.model)
319
+ local_model_path = self._prepare_weights(model_config)
320
gguf_weights_map = self._get_gguf_weights_map(model_config)
321
# we can only know if tie word embeddings after mapping weights
322
if "lm_head.weight" in get_gguf_extra_tensor_names(
vllm/model_executor/model_loader/weight_utils.py
+46
@@ -369,6 +369,52 @@ def get_sparse_attention_config(
369
return config
370
371
372
+def download_gguf(
373
+ repo_id: str,
374
+ quant_type: str,
375
+ cache_dir: str | None = None,
376
+ revision: str | None = None,
377
+ ignore_patterns: str | list[str] | None = None,
378
+) -> str:
379
+ # Use patterns that snapshot_download can handle directly
380
+ # Patterns to match:
381
+ # - *-{quant_type}.gguf (root)
382
+ # - *-{quant_type}-*.gguf (root sharded)
383
+ # - */*-{quant_type}.gguf (subdir)
384
+ # - */*-{quant_type}-*.gguf (subdir sharded)
385
+ allow_patterns = [
386
+ f"*-{quant_type}.gguf",
387
+ f"*-{quant_type}-*.gguf",
388
+ f"*/*-{quant_type}.gguf",
389
+ f"*/*-{quant_type}-*.gguf",
390
+ ]
391
+
392
+ # Use download_weights_from_hf which handles caching and downloading
393
+ folder = download_weights_from_hf(
394
+ model_name_or_path=repo_id,
395
+ cache_dir=cache_dir,
396
+ allow_patterns=allow_patterns,
397
+ revision=revision,
398
+ ignore_patterns=ignore_patterns,
399
+ )
400
+
401
+ # Find the downloaded file(s) in the folder
402
+ local_files = []
403
+ for pattern in allow_patterns:
404
+ # Convert pattern to glob pattern for local filesystem
405
+ glob_pattern = os.path.join(folder, pattern)
406
+ local_files.extend(glob.glob(glob_pattern))
407
+
408
+ if not local_files:
409
+ raise ValueError(
410
+ f"Downloaded GGUF files not found in {folder} for quant_type {quant_type}"
411
+ )
412
+
413
+ # Sort to ensure consistent ordering (prefer non-sharded files)
414
+ local_files.sort(key=lambda x: (x.count("-"), x))
415
+ return local_files[0]
416
+
417
+
418
def download_weights_from_hf(
419
model_name_or_path: str,
420
cache_dir: str | None,
vllm/transformers_utils/config.py
+40
-11
@@ -42,7 +42,10 @@ from vllm.logger import init_logger
42
from vllm.transformers_utils.config_parser_base import ConfigParserBase
43
from vllm.transformers_utils.utils import (
44
check_gguf_file,
45
+ is_gguf,
46
+ is_remote_gguf,
47
parse_safetensors_file_metadata,
48
+ split_remote_gguf,
49
)
50
51
if envs.VLLM_USE_MODELSCOPE:
@@ -629,10 +632,12 @@ def maybe_override_with_speculators(
632
Returns:
633
Tuple of (resolved_model, resolved_tokenizer, speculative_config)
634
"""
632
- is_gguf = check_gguf_file(model)
633
- if is_gguf:
635
+ if check_gguf_file(model):
636
kwargs["gguf_file"] = Path(model).name
637
gguf_model_repo = Path(model).parent
638
+ elif is_remote_gguf(model):
639
+ repo_id, _ = split_remote_gguf(model)
640
+ gguf_model_repo = Path(repo_id)
641
else:
642
gguf_model_repo = None
643
kwargs["local_files_only"] = huggingface_hub.constants.HF_HUB_OFFLINE
@@ -678,10 +683,18 @@ def get_config(
683
) -> PretrainedConfig:
684
# Separate model folder from file path for GGUF models
685
681
- is_gguf = check_gguf_file(model)
682
- if is_gguf:
683
- kwargs["gguf_file"] = Path(model).name
684
- model = Path(model).parent
686
+ _is_gguf = is_gguf(model)
687
+ _is_remote_gguf = is_remote_gguf(model)
688
+ if _is_gguf:
689
+ if check_gguf_file(model):
690
+ # Local GGUF file
691
+ kwargs["gguf_file"] = Path(model).name
692
+ model = Path(model).parent
693
+ elif _is_remote_gguf:
694
+ # Remote GGUF - extract repo_id from repo_id:quant_type format
695
+ # The actual GGUF file will be downloaded later by GGUFModelLoader
696
+ # Keep model as repo_id:quant_type for download, but use repo_id for config
697
+ model, _ = split_remote_gguf(model)
698
699
if config_format == "auto":
700
try:
@@ -689,10 +702,25 @@ def get_config(
702
# Transformers implementation.
703
if file_or_path_exists(model, MISTRAL_CONFIG_NAME, revision=revision):
704
config_format = "mistral"
692
- elif is_gguf or file_or_path_exists(
705
+ elif (_is_gguf and not _is_remote_gguf) or file_or_path_exists(
706
model, HF_CONFIG_NAME, revision=revision
707
):
708
config_format = "hf"
709
+ # Remote GGUF models must have config.json in repo,
710
+ # otherwise the config can't be parsed correctly.
711
+ # FIXME(Isotr0py): Support remote GGUF repos without config.json
712
+ elif _is_remote_gguf and not file_or_path_exists(
713
+ model, HF_CONFIG_NAME, revision=revision
714
+ ):
715
+ err_msg = (
716
+ "Could not find config.json for remote GGUF model repo. "
717
+ "To load remote GGUF model through `<repo_id>:<quant_type>`, "
718
+ "ensure your model has config.json (HF format) file. "
719
+ "Otherwise please specify --hf-config-path <original_repo> "
720
+ "in engine args to fetch config from unquantized hf model."
721
+ )
722
+ logger.error(err_msg)
723
+ raise ValueError(err_msg)
724
else:
725
raise ValueError(
726
"Could not detect config format for no config file found. "
@@ -713,9 +741,6 @@ def get_config(
741
"'config.json'.\n"
742
" - For Mistral models: ensure the presence of a "
743
"'params.json'.\n"
716
- "3. For GGUF: pass the local path of the GGUF checkpoint.\n"
717
- " Loading GGUF from a remote repo directly is not yet "
718
- "supported.\n"
744
).format(model=model)
745
746
raise ValueError(error_message) from e
@@ -729,7 +754,7 @@ def get_config(
754
**kwargs,
755
)
756
# Special architecture mapping check for GGUF models
732
- if is_gguf:
757
+ if _is_gguf:
758
if config.model_type not in MODEL_FOR_CAUSAL_LM_MAPPING_NAMES:
759
raise RuntimeError(f"Can't get gguf config for {config.model_type}.")
760
model_type = MODEL_FOR_CAUSAL_LM_MAPPING_NAMES[config.model_type]
@@ -889,6 +914,8 @@ def get_pooling_config(model: str, revision: str | None = "main") -> dict | None
914
A dictionary containing the pooling type and whether
915
normalization is used, or None if no pooling configuration is found.
916
"""
917
+ if is_remote_gguf(model):
918
+ model, _ = split_remote_gguf(model)
919
920
modules_file_name = "modules.json"
921
@@ -1108,6 +1135,8 @@ def get_hf_image_processor_config(
1135
# Separate model folder from file path for GGUF models
1136
if check_gguf_file(model):
1137
model = Path(model).parent
1138
+ elif is_remote_gguf(model):
1139
+ model, _ = split_remote_gguf(model)
1140
return get_image_processor_config(
1141
model, token=hf_token, revision=revision, **kwargs
1142
)
vllm/transformers_utils/processor.py
+5
-5
@@ -18,7 +18,7 @@ from transformers.processing_utils import ProcessorMixin
18
from transformers.video_processing_utils import BaseVideoProcessor
19
from typing_extensions import TypeVar
20
21
-from vllm.transformers_utils.utils import check_gguf_file, convert_model_repo_to_path
21
+from vllm.transformers_utils.utils import convert_model_repo_to_path, is_gguf
22
from vllm.utils.func_utils import get_allowed_kwarg_only_overrides
23
24
if TYPE_CHECKING:
@@ -236,8 +236,8 @@ def cached_processor_from_config(
236
processor_cls: type[_P] | tuple[type[_P], ...] = ProcessorMixin,
237
**kwargs: Any,
238
) -> _P:
239
- if check_gguf_file(model_config.model):
240
- assert not check_gguf_file(model_config.tokenizer), (
239
+ if is_gguf(model_config.model):
240
+ assert not is_gguf(model_config.tokenizer), (
241
"For multimodal GGUF models, the original tokenizer "
242
"should be used to correctly load processor."
243
)
@@ -350,8 +350,8 @@ def cached_image_processor_from_config(
350
model_config: "ModelConfig",
351
**kwargs: Any,
352
):
353
- if check_gguf_file(model_config.model):
354
- assert not check_gguf_file(model_config.tokenizer), (
353
+ if is_gguf(model_config.model):
354
+ assert not is_gguf(model_config.tokenizer), (
355
"For multimodal GGUF models, the original tokenizer "
356
"should be used to correctly load image processor."
357
)
vllm/transformers_utils/tokenizer.py
+12
-5
@@ -20,7 +20,12 @@ from vllm.transformers_utils.config import (
20
list_filtered_repo_files,
21
)
22
from vllm.transformers_utils.tokenizers import MistralTokenizer
23
-from vllm.transformers_utils.utils import check_gguf_file
23
+from vllm.transformers_utils.utils import (
24
+ check_gguf_file,
25
+ is_gguf,
26
+ is_remote_gguf,
27
+ split_remote_gguf,
28
+)
29
30
if TYPE_CHECKING:
31
from vllm.config import ModelConfig
@@ -180,10 +185,12 @@ def get_tokenizer(
185
kwargs["truncation_side"] = "left"
186
187
# Separate model folder from file path for GGUF models
183
- is_gguf = check_gguf_file(tokenizer_name)
184
- if is_gguf:
185
- kwargs["gguf_file"] = Path(tokenizer_name).name
186
- tokenizer_name = Path(tokenizer_name).parent
188
+ if is_gguf(tokenizer_name):
189
+ if check_gguf_file(tokenizer_name):
190
+ kwargs["gguf_file"] = Path(tokenizer_name).name
191
+ tokenizer_name = Path(tokenizer_name).parent
192
+ elif is_remote_gguf(tokenizer_name):
193
+ tokenizer_name, _ = split_remote_gguf(tokenizer_name)
194
195
# if `tokenizer_mode` == "auto", check if tokenizer can be loaded via Mistral format
196
# first to use official Mistral tokenizer if possible.
vllm/transformers_utils/utils.py
+53
@@ -9,6 +9,8 @@ from os import PathLike
9
from pathlib import Path
10
from typing import Any
11
12
+from gguf import GGMLQuantizationType
13
+
14
import vllm.envs as envs
15
from vllm.logger import init_logger
16
@@ -46,6 +48,57 @@ def check_gguf_file(model: str | PathLike) -> bool:
48
return False
49
50
51
+@cache
52
+def is_remote_gguf(model: str | Path) -> bool:
53
+ """Check if the model is a remote GGUF model."""
54
+ model = str(model)
55
+ return (
56
+ (not is_cloud_storage(model))
57
+ and (not model.startswith(("http://", "https://")))
58
+ and ("/" in model and ":" in model)
59
+ and is_valid_gguf_quant_type(model.rsplit(":", 1)[1])
60
+ )
61
+
62
+
63
+def is_valid_gguf_quant_type(gguf_quant_type: str) -> bool:
64
+ """Check if the quant type is a valid GGUF quant type."""
65
+ return getattr(GGMLQuantizationType, gguf_quant_type, None) is not None
66
+
67
+
68
+def split_remote_gguf(model: str | Path) -> tuple[str, str]:
69
+ """Split the model into repo_id and quant type."""
70
+ model = str(model)
71
+ if is_remote_gguf(model):
72
+ parts = model.rsplit(":", 1)
73
+ return (parts[0], parts[1])
74
+ raise ValueError(
75
+ "Wrong GGUF model or invalid GGUF quant type: %s.\n"
76
+ "- It should be in repo_id:quant_type format.\n"
77
+ "- Valid GGMLQuantizationType values: %s",
78
+ model,
79
+ GGMLQuantizationType._member_names_,
80
+ )
81
+
82
+
83
+def is_gguf(model: str | Path) -> bool:
84
+ """Check if the model is a GGUF model.
85
+
86
+ Args:
87
+ model: Model name, path, or Path object to check.
88
+
89
+ Returns:
90
+ True if the model is a GGUF model, False otherwise.
91
+ """
92
+ model = str(model)
93
+
94
+ # Check if it's a local GGUF file
95
+ if check_gguf_file(model):
96
+ return True
97
+
98
+ # Check if it's a remote GGUF model (repo_id:quant_type format)
99
+ return is_remote_gguf(model)
100
+
101
+
102
def modelscope_list_repo_files(
103
repo_id: str,
104
revision: str | None = None,