A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[BugFix] Fix Granite model configuration (#8216)
Nick Hill committed
Sep 5, 2024 at 20:39 UTC
baa5467547a758af35f442af6edfbc0fb73c83ce
2 files changed
+42
-24
vllm/transformers_utils/config.py
+38
-24
@@ -10,12 +10,16 @@ from transformers.models.auto.modeling_auto import (
10
11
from vllm.envs import VLLM_USE_MODELSCOPE
12
from vllm.logger import init_logger
13
+# yapf conflicts with isort for this block
14
+# yapf: disable
15
from vllm.transformers_utils.configs import (ChatGLMConfig, DbrxConfig,
16
EAGLEConfig, ExaoneConfig,
15
- InternVLChatConfig, JAISConfig,
16
- MedusaConfig, MLPSpeculatorConfig,
17
- MPTConfig, NemotronConfig,
18
- RWConfig, UltravoxConfig)
17
+ GraniteConfig, InternVLChatConfig,
18
+ JAISConfig, MedusaConfig,
19
+ MLPSpeculatorConfig, MPTConfig,
20
+ NemotronConfig, RWConfig,
21
+ UltravoxConfig)
22
+# yapf: enable
23
from vllm.transformers_utils.utils import check_gguf_file
24
25
if VLLM_USE_MODELSCOPE:
@@ -39,6 +43,9 @@ _CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
43
"internvl_chat": InternVLChatConfig,
44
"nemotron": NemotronConfig,
45
"ultravox": UltravoxConfig,
46
+ # Granite can be removed from here once we have upgraded to
47
+ # transformers 4.45+
48
+ "granite": GraniteConfig,
49
}
50
51
for name, cls in _CONFIG_REGISTRY.items():
@@ -62,29 +69,36 @@ def get_config(
69
kwargs["gguf_file"] = Path(model).name
70
model = Path(model).parent
71
65
- try:
66
- config = AutoConfig.from_pretrained(
67
- model,
68
- trust_remote_code=trust_remote_code,
69
- revision=revision,
70
- code_revision=code_revision,
71
- **kwargs)
72
- except ValueError as e:
73
- if (not trust_remote_code and
74
- "requires you to execute the configuration file" in str(e)):
75
- err_msg = (
76
- "Failed to load the model config. If the model is a custom "
77
- "model not yet available in the HuggingFace transformers "
78
- "library, consider setting `trust_remote_code=True` in LLM "
79
- "or using the `--trust-remote-code` flag in the CLI.")
80
- raise RuntimeError(err_msg) from e
81
- else:
82
- raise e
83
- if config.model_type in _CONFIG_REGISTRY:
84
- config_class = _CONFIG_REGISTRY[config.model_type]
72
+ config_dict, _ = PretrainedConfig.get_config_dict(
73
+ model, revision=revision, code_revision=code_revision, **kwargs)
74
+
75
+ # Use custom model class if it's in our registry
76
+ model_type = config_dict.get("model_type")
77
+ if model_type in _CONFIG_REGISTRY:
78
+ config_class = _CONFIG_REGISTRY[model_type]
79
config = config_class.from_pretrained(model,
80
revision=revision,
81
code_revision=code_revision)
82
+ else:
83
+ try:
84
+ config = AutoConfig.from_pretrained(
85
+ model,
86
+ trust_remote_code=trust_remote_code,
87
+ revision=revision,
88
+ code_revision=code_revision,
89
+ **kwargs)
90
+ except ValueError as e:
91
+ if (not trust_remote_code
92
+ and "requires you to execute the configuration file"
93
+ in str(e)):
94
+ err_msg = (
95
+ "Failed to load the model config. If the model is a custom "
96
+ "model not yet available in the HuggingFace transformers "
97
+ "library, consider setting `trust_remote_code=True` in LLM "
98
+ "or using the `--trust-remote-code` flag in the CLI.")
99
+ raise RuntimeError(err_msg) from e
100
+ else:
101
+ raise e
102
103
# Special architecture mapping check for GGUF models
104
if is_gguf:
vllm/transformers_utils/configs/__init__.py
+4
@@ -6,6 +6,7 @@ from vllm.transformers_utils.configs.exaone import ExaoneConfig
6
# tiiuae/falcon-7b(-instruct) models. Newer Falcon models will use the
7
# `FalconConfig` class from the official HuggingFace transformers library.
8
from vllm.transformers_utils.configs.falcon import RWConfig
9
+from vllm.transformers_utils.configs.granite import GraniteConfig
10
from vllm.transformers_utils.configs.internvl import InternVLChatConfig
11
from vllm.transformers_utils.configs.jais import JAISConfig
12
from vllm.transformers_utils.configs.medusa import MedusaConfig
@@ -27,4 +28,7 @@ __all__ = [
28
"MLPSpeculatorConfig",
29
"NemotronConfig",
30
"UltravoxConfig",
31
+ # Granite can be removed from here once we have upgraded to
32
+ # transformers 4.45+
33
+ "GraniteConfig",
34
]