A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 22:15 UTC
Various Transformers v5 config fixes (#38247)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
Mar 26, 2026 at 23:06 UTC
f73bcb1c51cfc764f534fcd109f8437e196be2ec
7 files changed
+45
-25
vllm/transformers_utils/config.py
+11
-1
@@ -119,6 +119,8 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
119
tarsier2="Tarsier2Config",
120
)
121
122
+_SPECULATIVE_DECODING_CONFIGS: set[str] = {"eagle", "speculators"}
123
+
124
_CONFIG_ATTRS_MAPPING: dict[str, str] = {
125
"llm_config": "text_config",
126
}
@@ -190,7 +192,7 @@ class HFConfigParser(ConfigParserBase):
192
dummy_model_type = hf_overrides(dummy_config).model_type
193
model_type = dummy_model_type.removeprefix("dummy_")
194
193
- if model_type in _CONFIG_REGISTRY:
195
+ if model_type in _SPECULATIVE_DECODING_CONFIGS:
196
config_class = _CONFIG_REGISTRY[model_type]
197
config = config_class.from_pretrained(
198
model,
@@ -200,6 +202,14 @@ class HFConfigParser(ConfigParserBase):
202
**kwargs,
203
)
204
else:
205
+ if model_type in _CONFIG_REGISTRY:
206
+ # Register the config class to AutoConfig to ensure it's used in future
207
+ # calls to `from_pretrained`
208
+ config_class = _CONFIG_REGISTRY[model_type]
209
+ config_class.model_type = model_type
210
+ AutoConfig.register(model_type, config_class, exist_ok=True)
211
+ # Now that it is registered, it is not considered remote code anymore
212
+ trust_remote_code = False
213
try:
214
kwargs = _maybe_update_auto_config_kwargs(kwargs, model_type=model_type)
215
config = AutoConfig.from_pretrained(
vllm/transformers_utils/configs/colmodernvbert.py
+1
-1
@@ -20,7 +20,6 @@ class ColModernVBertConfig(PretrainedConfig):
20
vlm_config: dict | None = None,
21
**kwargs,
22
):
23
- super().__init__(**kwargs)
23
self.embedding_dim = embedding_dim
24
25
if vlm_config is None:
@@ -55,6 +54,7 @@ class ColModernVBertConfig(PretrainedConfig):
54
intermediate_size=vis_cfg.get("intermediate_size", 3072),
55
num_attention_heads=vis_cfg.get("num_attention_heads", 12),
56
)
57
+ super().__init__(**kwargs)
58
59
@property
60
def image_seq_len(self) -> int:
vllm/transformers_utils/configs/deepseek_vl2.py
+21
-13
@@ -87,6 +87,18 @@ class MlpProjectorConfig(PretrainedConfig):
87
super().__init__(**kwargs)
88
89
90
+if hasattr(DeepseekV2Config, "validate"):
91
+ # Transformers v5
92
+ from huggingface_hub.dataclasses import strict
93
+
94
+ @strict
95
+ class DeepseekVLV2TextConfig(DeepseekV2Config):
96
+ kv_lora_rank: int | None = None
97
+else:
98
+ # Transformers v4
99
+ DeepseekVLV2TextConfig = DeepseekV2Config # type: ignore[misc]
100
+
101
+
102
class DeepseekVLV2Config(PretrainedConfig):
103
model_type = "deepseek_vl_v2"
104
architectures: list[str] | None = None
@@ -102,22 +114,17 @@ class DeepseekVLV2Config(PretrainedConfig):
114
candidate_resolutions: tuple[tuple[int, int]] = ((384, 384),),
115
**kwargs,
116
):
105
- super().__init__(**kwargs)
106
-
107
- if self.architectures is None:
108
- self.architectures = ["DeepseekVLV2ForCausalLM"]
117
+ if "architectures" not in kwargs:
118
+ kwargs["architectures"] = ["DeepseekVLV2ForCausalLM"]
119
110
- vision_config = kwargs.get("vision_config", {})
120
+ vision_config = kwargs.pop("vision_config", {})
121
self.vision_config = VisionEncoderConfig(**vision_config)
122
113
- projector_config = kwargs.get("projector_config", {})
123
+ projector_config = kwargs.pop("projector_config", {})
124
self.projector_config = MlpProjectorConfig(**projector_config)
125
116
- language_config = kwargs.get("language_config", {})
117
- # remove kv_lora_rank if not specified, passing None is prohibited
118
- if language_config.get("kv_lora_rank") is None:
119
- language_config.pop("kv_lora_rank", None)
120
- self.text_config = DeepseekV2Config(**language_config)
126
+ language_config = kwargs.pop("language_config", {})
127
+ self.text_config = DeepseekVLV2TextConfig(**language_config)
128
129
self.tile_tag = tile_tag
130
self.global_view_pos = global_view_pos
@@ -125,7 +132,8 @@ class DeepseekVLV2Config(PretrainedConfig):
132
self.vocab_size = self.text_config.vocab_size
133
134
# update model_type for OCR models
128
- if "DeepseekOCRForCausalLM" in self.architectures:
135
+ if "DeepseekOCRForCausalLM" in kwargs["architectures"]:
136
self.model_type = "deepseek_ocr"
130
- elif "DeepseekOCR2ForCausalLM" in self.architectures:
137
+ elif "DeepseekOCR2ForCausalLM" in kwargs["architectures"]:
138
self.model_type = "deepseek_ocr2"
139
+ super().__init__(**kwargs)
vllm/transformers_utils/configs/flex_olmo.py
+7
-7
@@ -39,13 +39,6 @@ class FlexOlmoConfig(PretrainedConfig):
39
if "architectures" not in kwargs:
40
kwargs["architectures"] = ["FlexOlmoForCausalLM"]
41
42
- super().__init__(
43
- pad_token_id=pad_token_id,
44
- bos_token_id=bos_token_id,
45
- eos_token_id=eos_token_id,
46
- tie_word_embeddings=tie_word_embeddings,
47
- **kwargs,
48
- )
42
self.vocab_size = vocab_size
43
self.max_position_embeddings = max_position_embeddings
44
self.hidden_size = hidden_size
@@ -80,3 +73,10 @@ class FlexOlmoConfig(PretrainedConfig):
73
# BC: if there is a 'type' field, move it to 'rope_type'.
74
if self.rope_parameters is not None and "type" in self.rope_parameters:
75
self.rope_parameters["rope_type"] = self.rope_parameters["type"]
76
+ super().__init__(
77
+ pad_token_id=pad_token_id,
78
+ bos_token_id=bos_token_id,
79
+ eos_token_id=eos_token_id,
80
+ tie_word_embeddings=tie_word_embeddings,
81
+ **kwargs,
82
+ )
vllm/transformers_utils/configs/isaac.py
+1
-2
@@ -50,8 +50,6 @@ class IsaacConfig(Qwen3Config):
50
vision_attn_implementation: str | None = None,
51
**kwargs,
52
):
53
- super().__init__(**kwargs)
54
-
53
if isinstance(text_config, dict):
54
# from HF config
55
self.text_config = self.sub_configs["text_config"](**text_config)
@@ -92,6 +90,7 @@ class IsaacConfig(Qwen3Config):
90
vision_max_num_patches,
91
)
92
self.vision_attn_implementation = vision_attn_implementation
93
+ super().__init__(**kwargs)
94
95
96
__all__ = [
vllm/transformers_utils/configs/qwen3_next.py
+1
-1
@@ -220,7 +220,6 @@ class Qwen3NextConfig(PretrainedConfig):
220
):
221
if mlp_only_layers is None:
222
mlp_only_layers = []
223
- super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)
223
self.vocab_size = vocab_size
224
self.max_position_embeddings = max_position_embeddings
225
self.hidden_size = hidden_size
@@ -279,6 +278,7 @@ class Qwen3NextConfig(PretrainedConfig):
278
self.output_router_logits = output_router_logits
279
self.router_aux_loss_coef = router_aux_loss_coef
280
self.mlp_only_layers = mlp_only_layers
281
+ super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)
282
283
284
__all__ = ["Qwen3NextConfig"]
vllm/transformers_utils/configs/step3p5.py
+3
@@ -80,6 +80,9 @@ class Step3p5Config(PretrainedConfig):
80
81
self.att_impl_type = att_impl_type
82
self.use_head_wise_attn_gate = use_head_wise_attn_gate
83
+ # For some reason the checkpoint has longer layer_types than num_hidden_layers
84
+ if layer_types is not None:
85
+ layer_types = layer_types[: self.num_hidden_layers]
86
self.layer_types = layer_types
87
self.use_rope_layers = use_rope_layers
88
self.yarn_only_types = yarn_only_types