A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 06:15 UTC
[UX] Use gguf `repo_id:quant_type` syntax for examples and docs (#33371)
Signed-off-by: mgoin <mgoin64@gmail.com>
Michael Goin committed
Jan 30, 2026 at 23:14 UTC
29fba76781441ff38796eb0c189bbdd0497dc58c
4 files changed
+79
-28
docs/features/quantization/gguf.md
+19
-15
@@ -6,34 +6,38 @@
6
!!! warning
7
Currently, vllm only supports loading single-file GGUF models. If you have a multi-files GGUF model, you can use [gguf-split](https://github.com/ggerganov/llama.cpp/pull/6135) tool to merge them to a single-file model.
8
9
-To run a GGUF model with vLLM, you can download and use the local GGUF model from [TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF](https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF) with the following command:
9
+To run a GGUF model with vLLM, you can use the `repo_id:quant_type` format to load directly from HuggingFace. For example, to load a Q4_K_M quantized model from [unsloth/Qwen3-0.6B-GGUF](https://huggingface.co/unsloth/Qwen3-0.6B-GGUF):
10
11
```bash
12
-wget https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF/resolve/main/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf
12
# We recommend using the tokenizer from base model to avoid long-time and buggy tokenizer conversion.
14
-vllm serve ./tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf \
15
- --tokenizer TinyLlama/TinyLlama-1.1B-Chat-v1.0
13
+vllm serve unsloth/Qwen3-0.6B-GGUF:Q4_K_M --tokenizer Qwen/Qwen3-0.6B
14
```
15
16
You can also add `--tensor-parallel-size 2` to enable tensor parallelism inference with 2 GPUs:
17
18
```bash
21
-# We recommend using the tokenizer from base model to avoid long-time and buggy tokenizer conversion.
22
-vllm serve ./tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf \
23
- --tokenizer TinyLlama/TinyLlama-1.1B-Chat-v1.0 \
19
+vllm serve unsloth/Qwen3-0.6B-GGUF:Q4_K_M \
20
+ --tokenizer Qwen/Qwen3-0.6B \
21
--tensor-parallel-size 2
22
```
23
24
+Alternatively, you can download and use a local GGUF file:
25
+
26
+```bash
27
+wget https://huggingface.co/unsloth/Qwen3-0.6B-GGUF/resolve/main/Qwen3-0.6B-Q4_K_M.gguf
28
+vllm serve ./Qwen3-0.6B-Q4_K_M.gguf --tokenizer Qwen/Qwen3-0.6B
29
+```
30
+
31
!!! warning
32
We recommend using the tokenizer from base model instead of GGUF model. Because the tokenizer conversion from GGUF is time-consuming and unstable, especially for some models with large vocab size.
33
30
-GGUF assumes that huggingface can convert the metadata to a config file. In case huggingface doesn't support your model you can manually create a config and pass it as hf-config-path
34
+GGUF assumes that HuggingFace can convert the metadata to a config file. In case HuggingFace doesn't support your model you can manually create a config and pass it as hf-config-path
35
36
```bash
33
-# If you model is not supported by huggingface you can manually provide a huggingface compatible config path
34
-vllm serve ./tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf \
35
- --tokenizer TinyLlama/TinyLlama-1.1B-Chat-v1.0 \
36
- --hf-config-path Tinyllama/TInyLlama-1.1B-Chat-v1.0
37
+# If your model is not supported by HuggingFace you can manually provide a HuggingFace compatible config path
38
+vllm serve unsloth/Qwen3-0.6B-GGUF:Q4_K_M \
39
+ --tokenizer Qwen/Qwen3-0.6B \
40
+ --hf-config-path Qwen/Qwen3-0.6B
41
```
42
43
You can also use the GGUF model directly through the LLM entrypoint:
@@ -66,10 +70,10 @@ You can also use the GGUF model directly through the LLM entrypoint:
70
# Create a sampling params object.
71
sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
72
69
- # Create an LLM.
73
+ # Create an LLM using repo_id:quant_type format.
74
llm = LLM(
71
- model="./tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf",
72
- tokenizer="TinyLlama/TinyLlama-1.1B-Chat-v1.0",
75
+ model="unsloth/Qwen3-0.6B-GGUF:Q4_K_M",
76
+ tokenizer="Qwen/Qwen3-0.6B",
77
)
78
# Generate texts from the prompts. The output is a list of RequestOutput objects
79
# that contain the prompt, generated text, and other information.
examples/offline_inference/basic/README.md
+2
-9
@@ -56,17 +56,10 @@ Try it yourself with the following argument:
56
57
vLLM supports models that are quantized using GGUF.
58
59
-Try one yourself by downloading a quantized GGUF model and using the following arguments:
60
-
61
-```python
62
-from huggingface_hub import hf_hub_download
63
-repo_id = "bartowski/Phi-3-medium-4k-instruct-GGUF"
64
-filename = "Phi-3-medium-4k-instruct-IQ2_M.gguf"
65
-print(hf_hub_download(repo_id, filename=filename))
66
-```
59
+Try one yourself using the `repo_id:quant_type` format to load directly from HuggingFace:
60
61
```bash
69
---model {local-path-printed-above} --tokenizer microsoft/Phi-3-medium-4k-instruct
62
+--model unsloth/Qwen3-0.6B-GGUF:Q4_K_M --tokenizer Qwen/Qwen3-0.6B
63
```
64
65
### CPU offload
tests/transformers_utils/test_utils.py
+34
-1
@@ -43,7 +43,7 @@ class TestIsRemoteGGUF:
43
44
def test_is_remote_gguf_with_colon_and_slash(self):
45
"""Test is_remote_gguf with repo_id:quant_type format."""
46
- # Valid quant types
46
+ # Valid quant types (exact GGML types)
47
assert is_remote_gguf("unsloth/Qwen3-0.6B-GGUF:IQ1_S")
48
assert is_remote_gguf("user/repo:Q2_K")
49
assert is_remote_gguf("repo/model:Q4_K")
@@ -54,6 +54,24 @@ class TestIsRemoteGGUF:
54
assert not is_remote_gguf("repo/model:INVALID")
55
assert not is_remote_gguf("repo/model:invalid_type")
56
57
+ def test_is_remote_gguf_extended_quant_types(self):
58
+ """Test is_remote_gguf with extended quant type naming conventions."""
59
+ # Extended quant types with _M, _S, _L suffixes
60
+ assert is_remote_gguf("repo/model:Q4_K_M")
61
+ assert is_remote_gguf("repo/model:Q4_K_S")
62
+ assert is_remote_gguf("repo/model:Q3_K_L")
63
+ assert is_remote_gguf("repo/model:Q5_K_M")
64
+ assert is_remote_gguf("repo/model:Q3_K_S")
65
+
66
+ # Extended quant types with _XL, _XS, _XXS suffixes
67
+ assert is_remote_gguf("repo/model:Q5_K_XL")
68
+ assert is_remote_gguf("repo/model:IQ4_XS")
69
+ assert is_remote_gguf("repo/model:IQ3_XXS")
70
+
71
+ # Invalid extended types (base type doesn't exist)
72
+ assert not is_remote_gguf("repo/model:INVALID_M")
73
+ assert not is_remote_gguf("repo/model:Q9_K_M")
74
+
75
def test_is_remote_gguf_without_colon(self):
76
"""Test is_remote_gguf without colon."""
77
assert not is_remote_gguf("repo/model")
@@ -106,6 +124,16 @@ class TestSplitRemoteGGUF:
124
assert repo_id == "repo/model"
125
assert quant_type == "Q2_K"
126
127
+ def test_split_remote_gguf_extended_quant_types(self):
128
+ """Test split_remote_gguf with extended quant type naming conventions."""
129
+ repo_id, quant_type = split_remote_gguf("unsloth/Qwen3-0.6B-GGUF:Q4_K_M")
130
+ assert repo_id == "unsloth/Qwen3-0.6B-GGUF"
131
+ assert quant_type == "Q4_K_M"
132
+
133
+ repo_id, quant_type = split_remote_gguf("repo/model:Q3_K_S")
134
+ assert repo_id == "repo/model"
135
+ assert quant_type == "Q3_K_S"
136
+
137
def test_split_remote_gguf_with_path_object(self):
138
"""Test split_remote_gguf with Path object."""
139
repo_id, quant_type = split_remote_gguf(Path("unsloth/Qwen3-0.6B-GGUF:IQ1_S"))
@@ -147,6 +175,11 @@ class TestIsGGUF:
175
assert is_gguf("repo/model:Q2_K")
176
assert is_gguf("repo/model:Q4_K")
177
178
+ # Extended quant types with suffixes
179
+ assert is_gguf("repo/model:Q4_K_M")
180
+ assert is_gguf("repo/model:Q3_K_S")
181
+ assert is_gguf("repo/model:Q5_K_L")
182
+
183
# Invalid quant_type should return False
184
assert not is_gguf("repo/model:quant")
185
assert not is_gguf("repo/model:INVALID")
vllm/transformers_utils/gguf_utils.py
+24
-3
@@ -49,9 +49,29 @@ def is_remote_gguf(model: str | Path) -> bool:
49
return False
50
51
52
+# Common suffixes used in GGUF file naming conventions
53
+# e.g., Q4_K_M, Q3_K_S, Q5_K_L, Q2_K_XL
54
+_GGUF_QUANT_SUFFIXES = ("_M", "_S", "_L", "_XL", "_XS", "_XXS")
55
+
56
+
57
def is_valid_gguf_quant_type(gguf_quant_type: str) -> bool:
53
- """Check if the quant type is a valid GGUF quant type."""
54
- return getattr(GGMLQuantizationType, gguf_quant_type, None) is not None
58
+ """Check if the quant type is a valid GGUF quant type.
59
+
60
+ Supports both exact GGML quant types (e.g., Q4_K, IQ1_S) and
61
+ extended naming conventions (e.g., Q4_K_M, Q3_K_S, Q5_K_L).
62
+ """
63
+ # Check for exact match first
64
+ if getattr(GGMLQuantizationType, gguf_quant_type, None) is not None:
65
+ return True
66
+
67
+ # Check for extended naming conventions (e.g., Q4_K_M -> Q4_K)
68
+ for suffix in _GGUF_QUANT_SUFFIXES:
69
+ if gguf_quant_type.endswith(suffix):
70
+ base_type = gguf_quant_type[: -len(suffix)]
71
+ if getattr(GGMLQuantizationType, base_type, None) is not None:
72
+ return True
73
+
74
+ return False
75
76
77
def split_remote_gguf(model: str | Path) -> tuple[str, str]:
@@ -63,7 +83,8 @@ def split_remote_gguf(model: str | Path) -> tuple[str, str]:
83
raise ValueError(
84
f"Wrong GGUF model or invalid GGUF quant type: {model}.\n"
85
"- It should be in repo_id:quant_type format.\n"
66
- f"- Valid GGMLQuantizationType values: {GGMLQuantizationType._member_names_}",
86
+ f"- Valid base quant types: {GGMLQuantizationType._member_names_}\n"
87
+ f"- Extended suffixes also supported: {_GGUF_QUANT_SUFFIXES}",
88
)
89
90