A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 22:35 UTC
[CI/Build] Fix registry tests (#21934)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Jul 31, 2025 at 00:10 UTC
004203e95330ac9a878df8192619570b0770667e
6 files changed
+70
-19
tests/models/registry.py
+11
-5
@@ -170,8 +170,10 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
170
min_transformers_version="4.54"),
171
"Ernie4_5_MoeForCausalLM": _HfExamplesInfo("baidu/ERNIE-4.5-21B-A3B-PT",
172
min_transformers_version="4.54"),
173
- "ExaoneForCausalLM": _HfExamplesInfo("LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct"), # noqa: E501
174
- "Exaone4ForCausalLM": _HfExamplesInfo("LGAI-EXAONE/EXAONE-4.0-32B"), # noqa: E501
173
+ "ExaoneForCausalLM": _HfExamplesInfo("LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct",
174
+ trust_remote_code=True),
175
+ "Exaone4ForCausalLM": _HfExamplesInfo("LGAI-EXAONE/EXAONE-4.0-32B",
176
+ min_transformers_version="4.54"),
177
"Fairseq2LlamaForCausalLM": _HfExamplesInfo("mgleize/fairseq2-dummy-Llama-3.2-1B"), # noqa: E501
178
"FalconForCausalLM": _HfExamplesInfo("tiiuae/falcon-7b"),
179
"FalconH1ForCausalLM":_HfExamplesInfo("tiiuae/Falcon-H1-0.5B-Base",
@@ -199,8 +201,10 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
201
trust_remote_code=True),
202
"HunYuanMoEV1ForCausalLM": _HfExamplesInfo("tencent/Hunyuan-A13B-Instruct",
203
trust_remote_code=True),
204
+ # TODO: Remove is_available_online once their config.json is fixed
205
"HunYuanDenseV1ForCausalLM":_HfExamplesInfo("tencent/Hunyuan-7B-Instruct-0124",
203
- trust_remote_code=True),
206
+ trust_remote_code=True,
207
+ is_available_online=False),
208
"HCXVisionForCausalLM": _HfExamplesInfo(
209
"naver-hyperclovax/HyperCLOVAX-SEED-Vision-Instruct-3B",
210
trust_remote_code=True),
@@ -275,7 +279,8 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
279
"StableLMEpochForCausalLM": _HfExamplesInfo("stabilityai/stablelm-zephyr-3b"), # noqa: E501
280
"StableLmForCausalLM": _HfExamplesInfo("stabilityai/stablelm-3b-4e1t"),
281
"Starcoder2ForCausalLM": _HfExamplesInfo("bigcode/starcoder2-3b"),
278
- "SolarForCausalLM": _HfExamplesInfo("upstage/solar-pro-preview-instruct"),
282
+ "SolarForCausalLM": _HfExamplesInfo("upstage/solar-pro-preview-instruct",
283
+ trust_remote_code=True),
284
"TeleChat2ForCausalLM": _HfExamplesInfo("Tele-AI/TeleChat2-3B",
285
trust_remote_code=True),
286
"TeleFLMForCausalLM": _HfExamplesInfo("CofeAI/FLM-2-52B-Instruct-2407",
@@ -449,7 +454,8 @@ _MULTIMODAL_EXAMPLE_MODELS = {
454
max_model_len=4096),
455
"Qwen2_5OmniModel": _HfExamplesInfo("Qwen/Qwen2.5-Omni-3B"),
456
"Qwen2_5OmniForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2.5-Omni-7B-AWQ"), # noqa: E501
452
- "SkyworkR1VChatModel": _HfExamplesInfo("Skywork/Skywork-R1V-38B"),
457
+ "SkyworkR1VChatModel": _HfExamplesInfo("Skywork/Skywork-R1V-38B",
458
+ trust_remote_code=True),
459
"SmolVLMForConditionalGeneration": _HfExamplesInfo("HuggingFaceTB/SmolVLM2-2.2B-Instruct"), # noqa: E501
460
"UltravoxModel": _HfExamplesInfo("fixie-ai/ultravox-v0_5-llama-3_2-1b", # noqa: E501
461
trust_remote_code=True),
vllm/model_executor/models/mpt.py
+10
-10
@@ -8,7 +8,7 @@ from typing import Optional, Union
8
9
import torch
10
import torch.nn as nn
11
-from transformers import PretrainedConfig
11
+from transformers import MptConfig
12
13
from vllm.attention import Attention
14
from vllm.compilation.decorators import support_torch_compile
@@ -50,7 +50,7 @@ class MPTAttention(nn.Module):
50
51
def __init__(
52
self,
53
- config: PretrainedConfig,
53
+ config: MptConfig,
54
cache_config: Optional[CacheConfig] = None,
55
quant_config: Optional[QuantizationConfig] = None,
56
prefix: str = "",
@@ -59,15 +59,15 @@ class MPTAttention(nn.Module):
59
self.d_model = config.d_model
60
self.total_num_heads = config.n_heads
61
self.head_dim = self.d_model // self.total_num_heads
62
- self.clip_qkv = config.attn_config["clip_qkv"]
63
- self.qk_ln = config.attn_config["qk_ln"]
64
- self.alibi_bias_max = config.attn_config["alibi_bias_max"]
62
+ self.clip_qkv = config.attn_config.clip_qkv
63
+ self.qk_ln = config.attn_config.qk_ln
64
+ self.alibi_bias_max = config.attn_config.alibi_bias_max
65
if "kv_n_heads" in config.attn_config:
66
- self.total_num_kv_heads = config.attn_config['kv_n_heads']
66
+ self.total_num_kv_heads = config.attn_config.kv_n_heads
67
else:
68
self.total_num_kv_heads = self.total_num_heads
69
- assert not config.attn_config["prefix_lm"]
70
- assert config.attn_config["alibi"]
69
+ assert not config.attn_config.prefix_lm
70
+ assert config.attn_config.alibi
71
72
# pylint: disable=invalid-name
73
self.Wqkv = QKVParallelLinear(
@@ -144,7 +144,7 @@ class MPTMLP(nn.Module):
144
145
def __init__(
146
self,
147
- config: PretrainedConfig,
147
+ config: MptConfig,
148
quant_config: Optional[QuantizationConfig] = None,
149
):
150
super().__init__()
@@ -176,7 +176,7 @@ class MPTBlock(nn.Module):
176
177
def __init__(
178
self,
179
- config: PretrainedConfig,
179
+ config: MptConfig,
180
cache_config: Optional[CacheConfig] = None,
181
quant_config: Optional[QuantizationConfig] = None,
182
prefix: str = "",
vllm/model_executor/models/telechat2.py
+13
-2
@@ -37,9 +37,20 @@ from .utils import (AutoWeightsLoader, PPMissingLayer, WeightsMapper,
37
class TeleChat2Model(LlamaModel):
38
39
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
40
+ hf_config = vllm_config.model_config.hf_config
41
+
42
+ vllm_config.model_config.hf_config.attribute_map = {
43
+ "num_hidden_layers": "n_layer",
44
+ "num_attention_heads": "n_head",
45
+ "intermediate_size": "ffn_hidden_size",
46
+ "rms_norm_eps": "layer_norm_epsilon"
47
+ }
48
+ vllm_config.model_config.hf_config.hidden_act = "silu"
49
+
50
# 1. Initialize the LlamaModel with bias
41
- vllm_config.model_config.hf_config.bias = True
42
- vllm_config.model_config.hf_config.mlp_bias = True
51
+ hf_config.bias = True
52
+ hf_config.mlp_bias = True
53
+
54
super().__init__(vllm_config=vllm_config, prefix=prefix)
55
# 2. Remove the bias from the qkv_proj and gate_up_proj based on config
56
# Telechat2's gate_up_proj and qkv_proj don't have bias
vllm/transformers_utils/config.py
+3
-2
@@ -34,8 +34,8 @@ from vllm.transformers_utils.configs import (ChatGLMConfig, DeepseekVLV2Config,
34
KimiVLConfig, MedusaConfig,
35
MllamaConfig, MLPSpeculatorConfig,
36
Nemotron_Nano_VL_Config,
37
- NemotronConfig, RWConfig,
38
- UltravoxConfig)
37
+ NemotronConfig, NVLM_D_Config,
38
+ RWConfig, UltravoxConfig)
39
# yapf: enable
40
from vllm.transformers_utils.configs.mistral import adapt_config_dict
41
from vllm.transformers_utils.utils import check_gguf_file
@@ -81,6 +81,7 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = {
81
"medusa": MedusaConfig,
82
"eagle": EAGLEConfig,
83
"nemotron": NemotronConfig,
84
+ "NVLM_D": NVLM_D_Config,
85
"ultravox": UltravoxConfig,
86
**_CONFIG_REGISTRY_OVERRIDE_HF
87
}
vllm/transformers_utils/configs/__init__.py
+2
@@ -23,6 +23,7 @@ from vllm.transformers_utils.configs.moonvit import MoonViTConfig
23
from vllm.transformers_utils.configs.nemotron import NemotronConfig
24
from vllm.transformers_utils.configs.nemotron_h import NemotronHConfig
25
from vllm.transformers_utils.configs.nemotron_vl import Nemotron_Nano_VL_Config
26
+from vllm.transformers_utils.configs.nvlm_d import NVLM_D_Config
27
from vllm.transformers_utils.configs.ultravox import UltravoxConfig
28
29
__all__ = [
@@ -39,5 +40,6 @@ __all__ = [
40
"NemotronConfig",
41
"NemotronHConfig",
42
"Nemotron_Nano_VL_Config",
43
+ "NVLM_D_Config",
44
"UltravoxConfig",
45
]
vllm/transformers_utils/configs/nvlm_d.py
new
+31
@@ -0,0 +1,31 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+# Adapted from
5
+# https://huggingface.co/nvidia/NVLM-D-72B/blob/main/configuration_nvlm_d.py
6
+# --------------------------------------------------------
7
+# NVLM-D
8
+# Copyright (c) 2024 NVIDIA
9
+# Licensed under Apache 2.0 License [see LICENSE for details]
10
+# --------------------------------------------------------
11
+from transformers import Qwen2Config
12
+from transformers.configuration_utils import PretrainedConfig
13
+
14
+
15
+class NVLM_D_Config(PretrainedConfig):
16
+ model_type = 'NVLM_D'
17
+ is_composition = True
18
+
19
+ def __init__(self, vision_config=None, llm_config=None, **kwargs):
20
+ super().__init__(**kwargs)
21
+
22
+ # Handle vision_config initialization
23
+ if vision_config is None:
24
+ vision_config = {}
25
+
26
+ # Handle llm_config initialization
27
+ if llm_config is None:
28
+ llm_config = {}
29
+
30
+ self.vision_config = PretrainedConfig(**vision_config)
31
+ self.text_config = Qwen2Config(**llm_config)