A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 04:45 UTC
[Chore]: Reorganize gguf utils funtions under `transformers_utils` (#29891)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Isotr0py committed
Dec 3, 2025 at 01:33 UTC
63b1da76ba35cd8cb220c79c44556e07fa4fb0c6
9 files changed
+96
-95
tests/models/test_gguf_download.py
+1
-1
@@ -203,7 +203,7 @@ class TestGGUFModelLoader:
203
@patch("vllm.config.model.get_hf_image_processor_config", return_value=None)
204
@patch("vllm.config.model.get_config")
205
@patch("vllm.config.model.is_gguf", return_value=False)
206
- @patch("vllm.transformers_utils.utils.check_gguf_file", return_value=False)
206
+ @patch("vllm.transformers_utils.gguf_utils.check_gguf_file", return_value=False)
207
@patch("os.path.isfile", return_value=False)
208
def test_prepare_weights_invalid_format(
209
self,
tests/transformers_utils/test_utils.py
+7
-5
@@ -5,13 +5,15 @@ from unittest.mock import patch
5
6
import pytest
7
8
+from vllm.transformers_utils.gguf_utils import (
9
+ is_gguf,
10
+ is_remote_gguf,
11
+ split_remote_gguf,
12
+)
13
from vllm.transformers_utils.utils import (
14
is_cloud_storage,
15
is_gcs,
11
- is_gguf,
12
- is_remote_gguf,
16
is_s3,
14
- split_remote_gguf,
17
)
18
19
@@ -132,7 +134,7 @@ class TestSplitRemoteGGUF:
134
class TestIsGGUF:
135
"""Test is_gguf utility function."""
136
135
- @patch("vllm.transformers_utils.utils.check_gguf_file", return_value=True)
137
+ @patch("vllm.transformers_utils.gguf_utils.check_gguf_file", return_value=True)
138
def test_is_gguf_with_local_file(self, mock_check_gguf):
139
"""Test is_gguf with local GGUF file."""
140
assert is_gguf("/path/to/model.gguf")
@@ -149,7 +151,7 @@ class TestIsGGUF:
151
assert not is_gguf("repo/model:quant")
152
assert not is_gguf("repo/model:INVALID")
153
152
- @patch("vllm.transformers_utils.utils.check_gguf_file", return_value=False)
154
+ @patch("vllm.transformers_utils.gguf_utils.check_gguf_file", return_value=False)
155
def test_is_gguf_false(self, mock_check_gguf):
156
"""Test is_gguf returns False for non-GGUF models."""
157
assert not is_gguf("unsloth/Qwen3-0.6B")
vllm/config/model.py
+3
-5
@@ -37,15 +37,13 @@ from vllm.transformers_utils.config import (
37
uses_xdrope_dim,
38
)
39
from vllm.transformers_utils.gguf_utils import (
40
- maybe_patch_hf_config_from_gguf,
41
-)
42
-from vllm.transformers_utils.runai_utils import ObjectStorageModel, is_runai_obj_uri
43
-from vllm.transformers_utils.utils import (
40
is_gguf,
41
is_remote_gguf,
46
- maybe_model_redirect,
42
+ maybe_patch_hf_config_from_gguf,
43
split_remote_gguf,
44
)
45
+from vllm.transformers_utils.runai_utils import ObjectStorageModel, is_runai_obj_uri
46
+from vllm.transformers_utils.utils import maybe_model_redirect
47
from vllm.utils.import_utils import LazyLoader
48
from vllm.utils.torch_utils import common_broadcastable_dtype
49
vllm/engine/arg_utils.py
+2
-1
@@ -86,8 +86,9 @@ from vllm.transformers_utils.config import (
86
is_interleaved,
87
maybe_override_with_speculators,
88
)
89
+from vllm.transformers_utils.gguf_utils import is_gguf
90
from vllm.transformers_utils.repo_utils import get_model_path
90
-from vllm.transformers_utils.utils import is_cloud_storage, is_gguf
91
+from vllm.transformers_utils.utils import is_cloud_storage
92
from vllm.utils.argparse_utils import FlexibleArgumentParser
93
from vllm.utils.mem_constants import GiB_bytes
94
from vllm.utils.network_utils import get_ip
vllm/tokenizers/registry.py
+3
-3
@@ -11,14 +11,14 @@ from typing_extensions import assert_never
11
12
import vllm.envs as envs
13
from vllm.logger import init_logger
14
-from vllm.transformers_utils.gguf_utils import get_gguf_file_path_from_hf
15
-from vllm.transformers_utils.repo_utils import list_filtered_repo_files
16
-from vllm.transformers_utils.utils import (
14
+from vllm.transformers_utils.gguf_utils import (
15
check_gguf_file,
16
+ get_gguf_file_path_from_hf,
17
is_gguf,
18
is_remote_gguf,
19
split_remote_gguf,
20
)
21
+from vllm.transformers_utils.repo_utils import list_filtered_repo_files
22
from vllm.utils.import_utils import resolve_obj_by_qualname
23
24
from .protocol import TokenizerLike
vllm/transformers_utils/config.py
+7
-7
@@ -26,8 +26,15 @@ from transformers.utils import CONFIG_NAME as HF_CONFIG_NAME
26
27
from vllm import envs
28
from vllm.logger import init_logger
29
+from vllm.transformers_utils.utils import parse_safetensors_file_metadata
30
31
from .config_parser_base import ConfigParserBase
32
+from .gguf_utils import (
33
+ check_gguf_file,
34
+ is_gguf,
35
+ is_remote_gguf,
36
+ split_remote_gguf,
37
+)
38
from .repo_utils import (
39
_get_hf_token,
40
file_or_path_exists,
@@ -36,13 +43,6 @@ from .repo_utils import (
43
try_get_local_file,
44
with_retry,
45
)
39
-from .utils import (
40
- check_gguf_file,
41
- is_gguf,
42
- is_remote_gguf,
43
- parse_safetensors_file_metadata,
44
- split_remote_gguf,
45
-)
46
47
if envs.VLLM_USE_MODELSCOPE:
48
from modelscope import AutoConfig
vllm/transformers_utils/gguf_utils.py
+71
@@ -2,10 +2,14 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
"""GGUF utility functions."""
4
5
+from functools import cache
6
+from os import PathLike
7
from pathlib import Path
8
9
import gguf
10
+import regex as re
11
from gguf.constants import Keys, VisionProjectorType
12
+from gguf.quants import GGMLQuantizationType
13
from transformers import Gemma3Config, PretrainedConfig, SiglipVisionConfig
14
15
from vllm.logger import init_logger
@@ -15,6 +19,73 @@ from .repo_utils import list_filtered_repo_files
19
logger = init_logger(__name__)
20
21
22
+@cache
23
+def check_gguf_file(model: str | PathLike) -> bool:
24
+ """Check if the file is a GGUF model."""
25
+ model = Path(model)
26
+ if not model.is_file():
27
+ return False
28
+ elif model.suffix == ".gguf":
29
+ return True
30
+
31
+ try:
32
+ with model.open("rb") as f:
33
+ header = f.read(4)
34
+
35
+ return header == b"GGUF"
36
+ except Exception as e:
37
+ logger.debug("Error reading file %s: %s", model, e)
38
+ return False
39
+
40
+
41
+@cache
42
+def is_remote_gguf(model: str | Path) -> bool:
43
+ """Check if the model is a remote GGUF model."""
44
+ pattern = r"^[a-zA-Z0-9][a-zA-Z0-9._-]*/[a-zA-Z0-9][a-zA-Z0-9._-]*:[A-Za-z0-9_+-]+$"
45
+ model = str(model)
46
+ if re.fullmatch(pattern, model):
47
+ _, quant_type = model.rsplit(":", 1)
48
+ return is_valid_gguf_quant_type(quant_type)
49
+ return False
50
+
51
+
52
+def is_valid_gguf_quant_type(gguf_quant_type: str) -> bool:
53
+ """Check if the quant type is a valid GGUF quant type."""
54
+ return getattr(GGMLQuantizationType, gguf_quant_type, None) is not None
55
+
56
+
57
+def split_remote_gguf(model: str | Path) -> tuple[str, str]:
58
+ """Split the model into repo_id and quant type."""
59
+ model = str(model)
60
+ if is_remote_gguf(model):
61
+ parts = model.rsplit(":", 1)
62
+ return (parts[0], parts[1])
63
+ raise ValueError(
64
+ f"Wrong GGUF model or invalid GGUF quant type: {model}.\n"
65
+ "- It should be in repo_id:quant_type format.\n"
66
+ f"- Valid GGMLQuantizationType values: {GGMLQuantizationType._member_names_}",
67
+ )
68
+
69
+
70
+def is_gguf(model: str | Path) -> bool:
71
+ """Check if the model is a GGUF model.
72
+
73
+ Args:
74
+ model: Model name, path, or Path object to check.
75
+
76
+ Returns:
77
+ True if the model is a GGUF model, False otherwise.
78
+ """
79
+ model = str(model)
80
+
81
+ # Check if it's a local GGUF file
82
+ if check_gguf_file(model):
83
+ return True
84
+
85
+ # Check if it's a remote GGUF model (repo_id:quant_type format)
86
+ return is_remote_gguf(model)
87
+
88
+
89
def detect_gguf_multimodal(model: str) -> Path | None:
90
"""Check if GGUF model has multimodal projector file.
91
vllm/transformers_utils/processor.py
+2
-1
@@ -18,7 +18,8 @@ from transformers.processing_utils import ProcessorMixin
18
from transformers.video_processing_utils import BaseVideoProcessor
19
from typing_extensions import TypeVar
20
21
-from vllm.transformers_utils.utils import convert_model_repo_to_path, is_gguf
21
+from vllm.transformers_utils.gguf_utils import is_gguf
22
+from vllm.transformers_utils.utils import convert_model_repo_to_path
23
from vllm.utils.func_utils import get_allowed_kwarg_only_overrides
24
25
if TYPE_CHECKING:
vllm/transformers_utils/utils.py
-72
@@ -9,8 +9,6 @@ from os import PathLike
9
from pathlib import Path
10
from typing import Any
11
12
-from gguf import GGMLQuantizationType
13
-
12
import vllm.envs as envs
13
from vllm.logger import init_logger
14
@@ -29,76 +27,6 @@ def is_cloud_storage(model_or_path: str) -> bool:
27
return is_s3(model_or_path) or is_gcs(model_or_path)
28
29
32
-@cache
33
-def check_gguf_file(model: str | PathLike) -> bool:
34
- """Check if the file is a GGUF model."""
35
- model = Path(model)
36
- if not model.is_file():
37
- return False
38
- elif model.suffix == ".gguf":
39
- return True
40
-
41
- try:
42
- with model.open("rb") as f:
43
- header = f.read(4)
44
-
45
- return header == b"GGUF"
46
- except Exception as e:
47
- logger.debug("Error reading file %s: %s", model, e)
48
- return False
49
-
50
-
51
-@cache
52
-def is_remote_gguf(model: str | Path) -> bool:
53
- """Check if the model is a remote GGUF model."""
54
- model = str(model)
55
- return (
56
- (not is_cloud_storage(model))
57
- and (not model.startswith(("http://", "https://")))
58
- and ("/" in model and ":" in model)
59
- and is_valid_gguf_quant_type(model.rsplit(":", 1)[1])
60
- )
61
-
62
-
63
-def is_valid_gguf_quant_type(gguf_quant_type: str) -> bool:
64
- """Check if the quant type is a valid GGUF quant type."""
65
- return getattr(GGMLQuantizationType, gguf_quant_type, None) is not None
66
-
67
-
68
-def split_remote_gguf(model: str | Path) -> tuple[str, str]:
69
- """Split the model into repo_id and quant type."""
70
- model = str(model)
71
- if is_remote_gguf(model):
72
- parts = model.rsplit(":", 1)
73
- return (parts[0], parts[1])
74
- raise ValueError(
75
- "Wrong GGUF model or invalid GGUF quant type: %s.\n"
76
- "- It should be in repo_id:quant_type format.\n"
77
- "- Valid GGMLQuantizationType values: %s",
78
- model,
79
- GGMLQuantizationType._member_names_,
80
- )
81
-
82
-
83
-def is_gguf(model: str | Path) -> bool:
84
- """Check if the model is a GGUF model.
85
-
86
- Args:
87
- model: Model name, path, or Path object to check.
88
-
89
- Returns:
90
- True if the model is a GGUF model, False otherwise.
91
- """
92
- model = str(model)
93
-
94
- # Check if it's a local GGUF file
95
- if check_gguf_file(model):
96
- return True
97
-
98
- # Check if it's a remote GGUF model (repo_id:quant_type format)
99
- return is_remote_gguf(model)
100
-
101
-
30
def modelscope_list_repo_files(
31
repo_id: str,
32
revision: str | None = None,