@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / baa5467547
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[BugFix] Fix Granite model configuration (#8216)

Nick Hill committed Sep 5, 2024 at 20:39 UTC baa5467547a758af35f442af6edfbc0fb73c83ce
2 files changed +42 -24
vllm/transformers_utils/config.py
+38 -24
@@ -10,12 +10,16 @@ from transformers.models.auto.modeling_auto import (
10
11 from vllm.envs import VLLM_USE_MODELSCOPE
12 from vllm.logger import init_logger
13 +# yapf conflicts with isort for this block
14 +# yapf: disable
15 from vllm.transformers_utils.configs import (ChatGLMConfig, DbrxConfig,
16 EAGLEConfig, ExaoneConfig,
15 - InternVLChatConfig, JAISConfig,
16 - MedusaConfig, MLPSpeculatorConfig,
17 - MPTConfig, NemotronConfig,
18 - RWConfig, UltravoxConfig)
17 + GraniteConfig, InternVLChatConfig,
18 + JAISConfig, MedusaConfig,
19 + MLPSpeculatorConfig, MPTConfig,
20 + NemotronConfig, RWConfig,
21 + UltravoxConfig)
22 +# yapf: enable
23 from vllm.transformers_utils.utils import check_gguf_file
24
25 if VLLM_USE_MODELSCOPE:
@@ -39,6 +43,9 @@ _CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
43 "internvl_chat": InternVLChatConfig,
44 "nemotron": NemotronConfig,
45 "ultravox": UltravoxConfig,
46 + # Granite can be removed from here once we have upgraded to
47 + # transformers 4.45+
48 + "granite": GraniteConfig,
49 }
50
51 for name, cls in _CONFIG_REGISTRY.items():
@@ -62,29 +69,36 @@ def get_config(
69 kwargs["gguf_file"] = Path(model).name
70 model = Path(model).parent
71
65 - try:
66 - config = AutoConfig.from_pretrained(
67 - model,
68 - trust_remote_code=trust_remote_code,
69 - revision=revision,
70 - code_revision=code_revision,
71 - **kwargs)
72 - except ValueError as e:
73 - if (not trust_remote_code and
74 - "requires you to execute the configuration file" in str(e)):
75 - err_msg = (
76 - "Failed to load the model config. If the model is a custom "
77 - "model not yet available in the HuggingFace transformers "
78 - "library, consider setting `trust_remote_code=True` in LLM "
79 - "or using the `--trust-remote-code` flag in the CLI.")
80 - raise RuntimeError(err_msg) from e
81 - else:
82 - raise e
83 - if config.model_type in _CONFIG_REGISTRY:
84 - config_class = _CONFIG_REGISTRY[config.model_type]
72 + config_dict, _ = PretrainedConfig.get_config_dict(
73 + model, revision=revision, code_revision=code_revision, **kwargs)
74 +
75 + # Use custom model class if it's in our registry
76 + model_type = config_dict.get("model_type")
77 + if model_type in _CONFIG_REGISTRY:
78 + config_class = _CONFIG_REGISTRY[model_type]
79 config = config_class.from_pretrained(model,
80 revision=revision,
81 code_revision=code_revision)
82 + else:
83 + try:
84 + config = AutoConfig.from_pretrained(
85 + model,
86 + trust_remote_code=trust_remote_code,
87 + revision=revision,
88 + code_revision=code_revision,
89 + **kwargs)
90 + except ValueError as e:
91 + if (not trust_remote_code
92 + and "requires you to execute the configuration file"
93 + in str(e)):
94 + err_msg = (
95 + "Failed to load the model config. If the model is a custom "
96 + "model not yet available in the HuggingFace transformers "
97 + "library, consider setting `trust_remote_code=True` in LLM "
98 + "or using the `--trust-remote-code` flag in the CLI.")
99 + raise RuntimeError(err_msg) from e
100 + else:
101 + raise e
102
103 # Special architecture mapping check for GGUF models
104 if is_gguf:
vllm/transformers_utils/configs/__init__.py
+4
@@ -6,6 +6,7 @@ from vllm.transformers_utils.configs.exaone import ExaoneConfig
6 # tiiuae/falcon-7b(-instruct) models. Newer Falcon models will use the
7 # `FalconConfig` class from the official HuggingFace transformers library.
8 from vllm.transformers_utils.configs.falcon import RWConfig
9 +from vllm.transformers_utils.configs.granite import GraniteConfig
10 from vllm.transformers_utils.configs.internvl import InternVLChatConfig
11 from vllm.transformers_utils.configs.jais import JAISConfig
12 from vllm.transformers_utils.configs.medusa import MedusaConfig
@@ -27,4 +28,7 @@ __all__ = [
28 "MLPSpeculatorConfig",
29 "NemotronConfig",
30 "UltravoxConfig",
31 + # Granite can be removed from here once we have upgraded to
32 + # transformers 4.45+
33 + "GraniteConfig",
34 ]