A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 11:00 UTC
[BugFix] Optional tokenizer argument when loading GGUF models (#29582)
Signed-off-by: Injae Ryou <injaeryou@gmail.com> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Injae Ryou committed
Nov 28, 2025 at 01:53 UTC
0840abdd242bbc7d0c42f0bfa73fec94a44e921b
3 files changed
+59
-8
vllm/config/model.py
+8
-7
@@ -439,13 +439,6 @@ class ModelConfig:
439
self.model = maybe_model_redirect(self.model)
440
# The tokenizer is consistent with the model by default.
441
if self.tokenizer is None:
442
- # Check if this is a GGUF model (either local file or remote GGUF)
443
- if is_gguf(self.model):
444
- raise ValueError(
445
- "Using a tokenizer is mandatory when loading a GGUF model. "
446
- "Please specify the tokenizer path or name using the "
447
- "--tokenizer argument."
448
- )
442
self.tokenizer = self.model
443
if self.tokenizer_revision is None:
444
self.tokenizer_revision = self.revision
@@ -699,6 +692,14 @@ class ModelConfig:
692
693
self.multimodal_config = MultiModalConfig(**mm_config_kwargs)
694
695
+ # Multimodal GGUF models must use original repo for mm processing
696
+ if is_gguf(self.tokenizer) and self.is_multimodal_model:
697
+ raise ValueError(
698
+ "Loading a multimodal GGUF model needs to use original "
699
+ "tokenizer. Please specify the unquantized hf model's "
700
+ "repo name or path using the --tokenizer argument."
701
+ )
702
+
703
if self.disable_sliding_window:
704
# Set after get_and_verify_max_len to ensure that max_model_len
705
# can be correctly capped to sliding window size
vllm/transformers_utils/gguf_utils.py
+42
@@ -9,6 +9,7 @@ from gguf.constants import Keys, VisionProjectorType
9
from transformers import Gemma3Config, PretrainedConfig, SiglipVisionConfig
10
11
from vllm.logger import init_logger
12
+from vllm.transformers_utils.config import list_filtered_repo_files
13
14
logger = init_logger(__name__)
15
@@ -164,3 +165,44 @@ def maybe_patch_hf_config_from_gguf(
165
hf_config = new_hf_config
166
167
return hf_config
168
+
169
+
170
+def get_gguf_file_path_from_hf(
171
+ repo_id: str | Path,
172
+ quant_type: str,
173
+ revision: str | None = None,
174
+) -> str:
175
+ """Get the GGUF file path from HuggingFace Hub based on repo_id and quant_type.
176
+
177
+ Args:
178
+ repo_id: The HuggingFace repository ID (e.g., "Qwen/Qwen3-0.6B")
179
+ quant_type: The quantization type (e.g., "Q4_K_M", "F16")
180
+ revision: Optional revision/branch name
181
+
182
+ Returns:
183
+ The path to the GGUF file on HuggingFace Hub (e.g., "filename.gguf"),
184
+ """
185
+ repo_id = str(repo_id)
186
+ gguf_patterns = [
187
+ f"*-{quant_type}.gguf",
188
+ f"*-{quant_type}-*.gguf",
189
+ f"*/*-{quant_type}.gguf",
190
+ f"*/*-{quant_type}-*.gguf",
191
+ ]
192
+ matching_files = list_filtered_repo_files(
193
+ repo_id,
194
+ allow_patterns=gguf_patterns,
195
+ revision=revision,
196
+ )
197
+
198
+ if len(matching_files) == 0:
199
+ raise ValueError(
200
+ "Could not find GGUF file for repo %s with quantization %s.",
201
+ repo_id,
202
+ quant_type,
203
+ )
204
+
205
+ # Sort to ensure consistent ordering (prefer non-sharded files)
206
+ matching_files.sort(key=lambda x: (x.count("-"), x))
207
+ gguf_filename = matching_files[0]
208
+ return gguf_filename
vllm/transformers_utils/tokenizer.py
+9
-1
@@ -19,6 +19,7 @@ from vllm.transformers_utils.config import (
19
get_sentence_transformer_tokenizer_config,
20
list_filtered_repo_files,
21
)
22
+from vllm.transformers_utils.gguf_utils import get_gguf_file_path_from_hf
23
from vllm.transformers_utils.tokenizers import MistralTokenizer
24
from vllm.transformers_utils.utils import (
25
check_gguf_file,
@@ -190,7 +191,14 @@ def get_tokenizer(
191
kwargs["gguf_file"] = Path(tokenizer_name).name
192
tokenizer_name = Path(tokenizer_name).parent
193
elif is_remote_gguf(tokenizer_name):
193
- tokenizer_name, _ = split_remote_gguf(tokenizer_name)
194
+ tokenizer_name, quant_type = split_remote_gguf(tokenizer_name)
195
+ # Get the HuggingFace Hub path for the GGUF file
196
+ gguf_file = get_gguf_file_path_from_hf(
197
+ tokenizer_name,
198
+ quant_type,
199
+ revision=revision,
200
+ )
201
+ kwargs["gguf_file"] = gguf_file
202
203
# if `tokenizer_mode` == "auto", check if tokenizer can be loaded via Mistral format
204
# first to use official Mistral tokenizer if possible.