A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 03:30 UTC
[Misc] Clean up InternVL family config registration (#19992)
Signed-off-by: Isotr0py <2037008807@qq.com>
Isotr0py committed
Jul 4, 2025 at 04:01 UTC
71d6de3a26347333288ef92bf18d285f25b80ead
5 files changed
+40
-82
vllm/transformers_utils/config.py
+22
-6
@@ -33,10 +33,8 @@ from vllm.logger import init_logger
33
from vllm.transformers_utils.configs import (ChatGLMConfig, Cohere2Config,
34
DbrxConfig, DeepseekVLV2Config,
35
EAGLEConfig, ExaoneConfig,
36
- H2OVLChatConfig,
37
- InternVLChatConfig, JAISConfig,
38
- KimiVLConfig, MedusaConfig,
39
- MiniMaxText01Config,
36
+ JAISConfig, KimiVLConfig,
37
+ MedusaConfig, MiniMaxText01Config,
38
MiniMaxVL01Config, MllamaConfig,
39
MLPSpeculatorConfig, MPTConfig,
40
NemotronConfig, NVLM_D_Config,
@@ -90,8 +88,6 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = {
88
"medusa": MedusaConfig,
89
"eagle": EAGLEConfig,
90
"exaone": ExaoneConfig,
93
- "h2ovl_chat": H2OVLChatConfig,
94
- "internvl_chat": InternVLChatConfig,
91
"minimax_text_01": MiniMaxText01Config,
92
"minimax_vl_01": MiniMaxVL01Config,
93
"nemotron": NemotronConfig,
@@ -104,6 +100,10 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = {
100
**_CONFIG_REGISTRY_OVERRIDE_HF
101
}
102
103
+_CONFIG_ATTRS_MAPPING: dict[str, str] = {
104
+ "llm_config": "text_config",
105
+}
106
+
107
108
class ConfigFormat(str, enum.Enum):
109
AUTO = "auto"
@@ -286,6 +286,18 @@ def is_encoder_decoder(config: PretrainedConfig) -> bool:
286
return getattr(config, "is_encoder_decoder", False)
287
288
289
+def _maybe_remap_hf_config_attrs(config: PretrainedConfig) -> PretrainedConfig:
290
+ """Remap config attributes to match the expected names."""
291
+ for old_attr, new_attr in _CONFIG_ATTRS_MAPPING.items():
292
+ if hasattr(config, old_attr):
293
+ if not hasattr(config, new_attr):
294
+ config.update({new_attr: getattr(config, old_attr)})
295
+ delattr(config, old_attr)
296
+ logger.debug("Remapped config attribute '%s' to '%s'", old_attr,
297
+ new_attr)
298
+ return config
299
+
300
+
301
def get_config(
302
model: Union[str, Path],
303
trust_remote_code: bool,
@@ -361,6 +373,9 @@ def get_config(
373
revision=revision,
374
code_revision=code_revision,
375
token=_get_hf_token(),
376
+ # some old custom model's config needs
377
+ # `has_no_defaults_at_init=True` to work.
378
+ has_no_defaults_at_init=trust_remote_code,
379
**kwargs,
380
)
381
except ValueError as e:
@@ -376,6 +391,7 @@ def get_config(
391
raise RuntimeError(err_msg) from e
392
else:
393
raise e
394
+ config = _maybe_remap_hf_config_attrs(config)
395
396
elif config_format == ConfigFormat.MISTRAL:
397
config = load_params_config(model, revision, **kwargs)
vllm/transformers_utils/configs/__init__.py
-4
@@ -11,8 +11,6 @@ from vllm.transformers_utils.configs.exaone import ExaoneConfig
11
# tiiuae/falcon-7b(-instruct) models. Newer Falcon models will use the
12
# `FalconConfig` class from the official HuggingFace transformers library.
13
from vllm.transformers_utils.configs.falcon import RWConfig
14
-from vllm.transformers_utils.configs.h2ovl import H2OVLChatConfig
15
-from vllm.transformers_utils.configs.internvl import InternVLChatConfig
14
from vllm.transformers_utils.configs.jais import JAISConfig
15
from vllm.transformers_utils.configs.kimi_vl import KimiVLConfig
16
from vllm.transformers_utils.configs.medusa import MedusaConfig
@@ -38,8 +36,6 @@ __all__ = [
36
"DeepseekVLV2Config",
37
"MPTConfig",
38
"RWConfig",
41
- "H2OVLChatConfig",
42
- "InternVLChatConfig",
39
"JAISConfig",
40
"MedusaConfig",
41
"EAGLEConfig",
vllm/transformers_utils/configs/h2ovl.py
deleted
-16
@@ -1,16 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-# Adapted from
5
-# https://huggingface.co/h2oai/h2ovl-mississippi-2b/blob/main/configuration_h2ovl_chat.py
6
-# --------------------------------------------------------
7
-# H2OVL-Mississippi
8
-# Copyright (c) 2024 H2O.AI
9
-# Licensed under Apache 2.0 License [see LICENSE for details]
10
-# --------------------------------------------------------
11
-
12
-from .internvl import InternVLChatConfig
13
-
14
-
15
-class H2OVLChatConfig(InternVLChatConfig):
16
- model_type = "h2ovl_chat"
vllm/transformers_utils/configs/internvl.py
deleted
-54
@@ -1,54 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-# Adapted from
5
-# https://huggingface.co/OpenGVLab/InternVL2-1B/blob/main/configuration_internvl_chat.py
6
-# --------------------------------------------------------
7
-# InternVL
8
-# Copyright (c) 2024 OpenGVLab
9
-# Licensed under The MIT License [see LICENSE for details]
10
-# --------------------------------------------------------
11
-from transformers.configuration_utils import PretrainedConfig
12
-
13
-
14
-class InternVLChatConfig(PretrainedConfig):
15
- model_type = 'internvl_chat'
16
- is_composition = True
17
-
18
- def __init__(self,
19
- vision_config=None,
20
- llm_config=None,
21
- use_backbone_lora=0,
22
- use_llm_lora=0,
23
- select_layer=-1,
24
- force_image_size=None,
25
- downsample_ratio=0.5,
26
- template=None,
27
- dynamic_image_size=False,
28
- use_thumbnail=False,
29
- ps_version='v1',
30
- min_dynamic_patch=1,
31
- max_dynamic_patch=6,
32
- **kwargs):
33
- super().__init__(**kwargs)
34
-
35
- if vision_config is None:
36
- vision_config = {}
37
-
38
- if llm_config is None:
39
- llm_config = {}
40
-
41
- self.vision_config = PretrainedConfig(**vision_config)
42
- self.text_config = PretrainedConfig(**llm_config)
43
-
44
- self.use_backbone_lora = use_backbone_lora
45
- self.use_llm_lora = use_llm_lora
46
- self.select_layer = select_layer
47
- self.force_image_size = force_image_size
48
- self.downsample_ratio = downsample_ratio
49
- self.template = template
50
- self.dynamic_image_size = dynamic_image_size
51
- self.use_thumbnail = use_thumbnail
52
- self.ps_version = ps_version # pixel shuffle version
53
- self.min_dynamic_patch = min_dynamic_patch
54
- self.max_dynamic_patch = max_dynamic_patch
vllm/transformers_utils/configs/nvlm_d.py
+18
-2
@@ -8,8 +8,24 @@
8
# Copyright (c) 2024 NVIDIA
9
# Licensed under Apache 2.0 License [see LICENSE for details]
10
# --------------------------------------------------------
11
-from .internvl import InternVLChatConfig
11
+from transformers import Qwen2Config
12
+from transformers.configuration_utils import PretrainedConfig
13
14
14
-class NVLM_D_Config(InternVLChatConfig):
15
+class NVLM_D_Config(PretrainedConfig):
16
model_type = 'NVLM_D'
17
+ is_composition = True
18
+
19
+ def __init__(self, vision_config=None, llm_config=None, **kwargs):
20
+ super().__init__(**kwargs)
21
+
22
+ # Handle vision_config initialization
23
+ if vision_config is None:
24
+ vision_config = {}
25
+
26
+ # Handle llm_config initialization
27
+ if llm_config is None:
28
+ llm_config = {}
29
+
30
+ self.vision_config = PretrainedConfig(**vision_config)
31
+ self.text_config = Qwen2Config(**llm_config)