A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 07:15 UTC
[GGUF] Support non-standard quant types with prefix (e.g. UD-IQ1_S) (#39471)
Signed-off-by: Injae Ryou <injaeryou@gmail.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Injae Ryou committed
Apr 10, 2026 at 16:22 UTC
447ce22212e55e145a324488f076cb4e28e7d3e8
2 files changed
+65
-3
tests/transformers_utils/test_utils.py
+27
@@ -81,6 +81,25 @@ class TestIsRemoteGGUF:
81
assert not is_remote_gguf("repo/model:INVALID_M")
82
assert not is_remote_gguf("repo/model:Q9_K_M")
83
84
+ def test_is_remote_gguf_nonstandard_quant_type(self):
85
+ """Test is_remote_gguf with non-standard quant types containing
86
+ a known GGML type."""
87
+ # Non-standard quant types with known GGML type after prefix
88
+ assert is_remote_gguf("unsloth/Qwen3.5-35B-A3B-GGUF:UD-Q4_K_XL")
89
+ assert is_remote_gguf("user/Model:UD-Q4_K_M")
90
+ assert is_remote_gguf("user/SomeModel:Custom-Q8_0")
91
+
92
+ # Exact GGML type after prefix (no suffix stripping needed)
93
+ assert is_remote_gguf("user/Model-GGUF:UD-IQ4_NL")
94
+ assert is_remote_gguf("user/Model-GGUF:UD-Q8_0")
95
+
96
+ # Completely unknown quant types should still fail
97
+ assert not is_remote_gguf("repo/model:TOTALLY-RANDOM")
98
+ assert not is_remote_gguf("user/Model:UD-INVALID")
99
+
100
+ # No dash separator → not recognized as prefixed
101
+ assert not is_remote_gguf("repo/model:UDIQ4NL")
102
+
103
def test_is_remote_gguf_without_colon(self):
104
"""Test is_remote_gguf without colon."""
105
assert not is_remote_gguf("repo/model")
@@ -143,6 +162,14 @@ class TestSplitRemoteGGUF:
162
assert repo_id == "repo/model"
163
assert quant_type == "Q3_K_S"
164
165
+ def test_split_remote_gguf_nonstandard_quant_type(self):
166
+ """Test split_remote_gguf with non-standard quant types in GGUF repos."""
167
+ repo_id, quant_type = split_remote_gguf(
168
+ "unsloth/Qwen3.5-35B-A3B-GGUF:UD-Q4_K_XL"
169
+ )
170
+ assert repo_id == "unsloth/Qwen3.5-35B-A3B-GGUF"
171
+ assert quant_type == "UD-Q4_K_XL"
172
+
173
def test_split_remote_gguf_with_path_object(self):
174
"""Test split_remote_gguf with Path object."""
175
repo_id, quant_type = split_remote_gguf(Path("unsloth/Qwen3-0.6B-GGUF:IQ1_S"))
vllm/transformers_utils/gguf_utils.py
+38
-3
@@ -40,15 +40,48 @@ def check_gguf_file(model: str | PathLike) -> bool:
40
41
@cache
42
def is_remote_gguf(model: str | Path) -> bool:
43
- """Check if the model is a remote GGUF model."""
43
+ """Check if the model is a remote GGUF model.
44
+
45
+ Recognizes two forms:
46
+ 1. Standard: ``repo_id:quant_type`` where *quant_type* is a known
47
+ GGML quantization type (e.g. ``Q4_K_M``).
48
+ 2. Non-standard: ``repo_id:quant_type`` where *quant_type* contains
49
+ a known GGML type with extra prefixes (e.g. ``UD-Q4_K_XL``).
50
+ A warning is logged and actual file existence is validated later
51
+ during download.
52
+ """
53
pattern = r"^[a-zA-Z0-9][a-zA-Z0-9._-]*/[a-zA-Z0-9][a-zA-Z0-9._-]*:[A-Za-z0-9_+-]+$"
54
model = str(model)
55
if re.fullmatch(pattern, model):
56
_, quant_type = model.rsplit(":", 1)
48
- return is_valid_gguf_quant_type(quant_type)
57
+ if is_valid_gguf_quant_type(quant_type):
58
+ return True
59
+ if is_nonstandard_gguf_quant_type(quant_type):
60
+ logger.warning(
61
+ "Non-standard GGUF quant type '%s' detected.",
62
+ quant_type,
63
+ )
64
+ return True
65
return False
66
67
68
+def is_nonstandard_gguf_quant_type(quant_type: str) -> bool:
69
+ """Check if a non-standard quant type contains a known GGML type.
70
+
71
+ Splits the quant type by the last ``-`` and checks whether the
72
+ trailing part is a standard GGML type. For example::
73
+
74
+ UD-Q4_K_XL → rsplit → ["UD", "Q4_K_XL"] → Q4_K_XL valid ✓
75
+ UD-IQ4_NL → rsplit → ["UD", "IQ4_NL"] → IQ4_NL valid ✓
76
+ Custom-UD-Q4_K → rsplit → ["Custom-UD", "Q4_K"] → Q4_K valid ✓
77
+ RANDOM → no "-" → False
78
+ """
79
+ if "-" not in quant_type:
80
+ return False
81
+ _, remainder = quant_type.rsplit("-", 1)
82
+ return is_valid_gguf_quant_type(remainder)
83
+
84
+
85
# Common suffixes used in GGUF file naming conventions
86
# e.g., Q4_K_M, Q3_K_S, Q5_K_L, Q2_K_XL
87
_GGUF_QUANT_SUFFIXES = ("_M", "_S", "_L", "_XL", "_XS", "_XXS")
@@ -84,7 +117,9 @@ def split_remote_gguf(model: str | Path) -> tuple[str, str]:
117
f"Wrong GGUF model or invalid GGUF quant type: {model}.\n"
118
"- It should be in repo_id:quant_type format.\n"
119
f"- Valid base quant types: {GGMLQuantizationType._member_names_}\n"
87
- f"- Extended suffixes also supported: {_GGUF_QUANT_SUFFIXES}",
120
+ f"- Extended suffixes also supported: {_GGUF_QUANT_SUFFIXES}\n"
121
+ "- Non-standard GGUF quant types also supported: "
122
+ "dash-separated prefixes (e.g. UD-Q4_K_XL, Custom-Q8_0)",
123
)
124
125