@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4ca65a9763
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Core][Bugfix] Accept GGUF model without .gguf extension (#8056)

Isotr0py committed Sep 2, 2024 at 20:43 UTC 4ca65a97638054ed04b37c2bf3e868d4c1209e9c
4 files changed +23 -5
vllm/engine/arg_utils.py
+2 -1
@@ -16,6 +16,7 @@ from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
16 from vllm.executor.executor_base import ExecutorBase
17 from vllm.logger import init_logger
18 from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
19 +from vllm.transformers_utils.utils import check_gguf_file
20 from vllm.utils import FlexibleArgumentParser
21
22 if TYPE_CHECKING:
@@ -753,7 +754,7 @@ class EngineArgs:
754
755 def create_engine_config(self) -> EngineConfig:
756 # gguf file needs a specific model loader and doesn't use hf_repo
756 - if self.model.endswith(".gguf"):
757 + if check_gguf_file(self.model):
758 self.quantization = self.load_format = "gguf"
759
760 # bitsandbytes quantization needs a specific model loader
vllm/transformers_utils/config.py
+3 -2
@@ -16,6 +16,7 @@ from vllm.transformers_utils.configs import (ChatGLMConfig, DbrxConfig,
16 MedusaConfig, MLPSpeculatorConfig,
17 MPTConfig, NemotronConfig,
18 RWConfig, UltravoxConfig)
19 +from vllm.transformers_utils.utils import check_gguf_file
20
21 if VLLM_USE_MODELSCOPE:
22 from modelscope import AutoConfig
@@ -56,7 +57,7 @@ def get_config(
57 ) -> PretrainedConfig:
58
59 # Separate model folder from file path for GGUF models
59 - is_gguf = Path(model).is_file() and Path(model).suffix == ".gguf"
60 + is_gguf = check_gguf_file(model)
61 if is_gguf:
62 kwargs["gguf_file"] = Path(model).name
63 model = Path(model).parent
@@ -112,7 +113,7 @@ def get_hf_image_processor_config(
113 if VLLM_USE_MODELSCOPE:
114 return dict()
115 # Separate model folder from file path for GGUF models
115 - if Path(model).is_file() and Path(model).suffix == ".gguf":
116 + if check_gguf_file(model):
117 model = Path(model).parent
118 return get_image_processor_config(model, revision=revision, **kwargs)
119
vllm/transformers_utils/tokenizer.py
+2 -2
@@ -12,6 +12,7 @@ from vllm.logger import init_logger
12 from vllm.lora.request import LoRARequest
13 from vllm.transformers_utils.tokenizers import (BaichuanTokenizer,
14 MistralTokenizer)
15 +from vllm.transformers_utils.utils import check_gguf_file
16 from vllm.utils import make_async
17
18 logger = init_logger(__name__)
@@ -96,8 +97,7 @@ def get_tokenizer(
97 kwargs["truncation_side"] = "left"
98
99 # Separate model folder from file path for GGUF models
99 - is_gguf = Path(tokenizer_name).is_file() and Path(
100 - tokenizer_name).suffix == ".gguf"
100 + is_gguf = check_gguf_file(tokenizer_name)
101 if is_gguf:
102 kwargs["gguf_file"] = Path(tokenizer_name).name
103 tokenizer_name = Path(tokenizer_name).parent
vllm/transformers_utils/utils.py new
+16
@@ -0,0 +1,16 @@
1 +from os import PathLike
2 +from pathlib import Path
3 +from typing import Union
4 +
5 +
6 +def check_gguf_file(model: Union[str, PathLike]) -> bool:
7 + """Check if the file is a GGUF model."""
8 + model = Path(model)
9 + if not model.is_file():
10 + return False
11 + elif model.suffix == ".gguf":
12 + return True
13 +
14 + with open(model, "rb") as f:
15 + header = f.read(4)
16 + return header == b"GGUF"