A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Core][Bugfix] Accept GGUF model without .gguf extension (#8056)
Isotr0py committed
Sep 2, 2024 at 20:43 UTC
4ca65a97638054ed04b37c2bf3e868d4c1209e9c
4 files changed
+23
-5
vllm/engine/arg_utils.py
+2
-1
@@ -16,6 +16,7 @@ from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
16
from vllm.executor.executor_base import ExecutorBase
17
from vllm.logger import init_logger
18
from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
19
+from vllm.transformers_utils.utils import check_gguf_file
20
from vllm.utils import FlexibleArgumentParser
21
22
if TYPE_CHECKING:
@@ -753,7 +754,7 @@ class EngineArgs:
754
755
def create_engine_config(self) -> EngineConfig:
756
# gguf file needs a specific model loader and doesn't use hf_repo
756
- if self.model.endswith(".gguf"):
757
+ if check_gguf_file(self.model):
758
self.quantization = self.load_format = "gguf"
759
760
# bitsandbytes quantization needs a specific model loader
vllm/transformers_utils/config.py
+3
-2
@@ -16,6 +16,7 @@ from vllm.transformers_utils.configs import (ChatGLMConfig, DbrxConfig,
16
MedusaConfig, MLPSpeculatorConfig,
17
MPTConfig, NemotronConfig,
18
RWConfig, UltravoxConfig)
19
+from vllm.transformers_utils.utils import check_gguf_file
20
21
if VLLM_USE_MODELSCOPE:
22
from modelscope import AutoConfig
@@ -56,7 +57,7 @@ def get_config(
57
) -> PretrainedConfig:
58
59
# Separate model folder from file path for GGUF models
59
- is_gguf = Path(model).is_file() and Path(model).suffix == ".gguf"
60
+ is_gguf = check_gguf_file(model)
61
if is_gguf:
62
kwargs["gguf_file"] = Path(model).name
63
model = Path(model).parent
@@ -112,7 +113,7 @@ def get_hf_image_processor_config(
113
if VLLM_USE_MODELSCOPE:
114
return dict()
115
# Separate model folder from file path for GGUF models
115
- if Path(model).is_file() and Path(model).suffix == ".gguf":
116
+ if check_gguf_file(model):
117
model = Path(model).parent
118
return get_image_processor_config(model, revision=revision, **kwargs)
119
vllm/transformers_utils/tokenizer.py
+2
-2
@@ -12,6 +12,7 @@ from vllm.logger import init_logger
12
from vllm.lora.request import LoRARequest
13
from vllm.transformers_utils.tokenizers import (BaichuanTokenizer,
14
MistralTokenizer)
15
+from vllm.transformers_utils.utils import check_gguf_file
16
from vllm.utils import make_async
17
18
logger = init_logger(__name__)
@@ -96,8 +97,7 @@ def get_tokenizer(
97
kwargs["truncation_side"] = "left"
98
99
# Separate model folder from file path for GGUF models
99
- is_gguf = Path(tokenizer_name).is_file() and Path(
100
- tokenizer_name).suffix == ".gguf"
100
+ is_gguf = check_gguf_file(tokenizer_name)
101
if is_gguf:
102
kwargs["gguf_file"] = Path(tokenizer_name).name
103
tokenizer_name = Path(tokenizer_name).parent
vllm/transformers_utils/utils.py
new
+16
@@ -0,0 +1,16 @@
1
+from os import PathLike
2
+from pathlib import Path
3
+from typing import Union
4
+
5
+
6
+def check_gguf_file(model: Union[str, PathLike]) -> bool:
7
+ """Check if the file is a GGUF model."""
8
+ model = Path(model)
9
+ if not model.is_file():
10
+ return False
11
+ elif model.suffix == ".gguf":
12
+ return True
13
+
14
+ with open(model, "rb") as f:
15
+ header = f.read(4)
16
+ return header == b"GGUF"