A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 08:15 UTC
Fix various config related issues for Transformers v5 (#37681)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
Mar 20, 2026 at 16:30 UTC
6ade4bc5a544f6319af968aff4d9b7e71f37434f
8 files changed
+42
-21
tests/entrypoints/openai/chat_completion/test_completion_with_function_calling.py
+4
-3
@@ -231,13 +231,14 @@ def k2_server():
231
"--gpu-memory-utilization",
232
"0.4",
233
] + ROCM_EXTRA_ARGS
234
- # hack to test kimi_k2 tool use tool_id format.
235
- # avoid error in is_deepseek_mla check by setting kv_lora_rank=null
234
+ # Test kimi_k2 tool use tool_id format by overriding model_type.
235
+ # is_deepseek_mla safely returns False via getattr when kv_lora_rank
236
+ # is absent from the underlying config.
237
with RemoteOpenAIServer(
238
MODEL_NAME,
239
args,
240
env_dict=ROCM_ENV_OVERRIDES,
240
- override_hf_configs={"model_type": "kimi_k2", "kv_lora_rank": None},
241
+ override_hf_configs={"model_type": "kimi_k2"},
242
) as remote_server:
243
yield remote_server
244
vllm/transformers_utils/configs/colpali.py
-2
@@ -27,7 +27,6 @@ class ColPaliConfig(PaliGemmaConfig):
27
embedding_dim: int | None = None,
28
embed_dim: int | None = None,
29
dim: int | None = None,
30
- projection_dim: int | None = None,
30
colbert_dim: int | None = None,
31
pooling: str | None = None,
32
vlm_config: dict | None = None,
@@ -37,7 +36,6 @@ class ColPaliConfig(PaliGemmaConfig):
36
self.embedding_dim = embedding_dim
37
self.embed_dim = embed_dim
38
self.dim = dim
40
- self.projection_dim = projection_dim
39
self.colbert_dim = colbert_dim
40
self.pooling = pooling
41
vllm/transformers_utils/configs/deepseek_vl2.py
-2
@@ -90,8 +90,6 @@ class MlpProjectorConfig(PretrainedConfig):
90
class DeepseekVLV2Config(PretrainedConfig):
91
model_type = "deepseek_vl_v2"
92
architectures: list[str] | None = None
93
- vision_config: VisionEncoderConfig
94
- projector_config: MlpProjectorConfig
93
94
tile_tag: str = "2D"
95
global_view_pos: str = "head"
vllm/transformers_utils/configs/mistral.py
+4
-1
@@ -257,7 +257,6 @@ def _remap_mistral_audio_args(config: dict) -> dict:
257
encoder_attention_heads=encoder_args["n_heads"],
258
encoder_head_dim=encoder_args["head_dim"],
259
vocab_size=encoder_args["vocab_size"],
260
- max_source_positions=encoder_args["max_source_positions"],
260
is_encoder_decoder=False, # Override WhisperConfig default
261
is_causal=encoder_args.get("causal", False),
262
sliding_window=encoder_args.get("sliding_window", None),
@@ -270,6 +269,10 @@ def _remap_mistral_audio_args(config: dict) -> dict:
269
max_position_embeddings=block_pool_size * config["max_position_embeddings"],
270
),
271
}
272
+ # Sometimes max_source_positions is explicitly set to None in params.json but this
273
+ # is not a valid value for WhisperConfig (or downstream code that uses it).
274
+ if (max_source_positions := encoder_args.get("max_source_positions")) is not None:
275
+ config["audio_config"].max_source_positions = max_source_positions
276
if quant_config:
277
config["quantization_config"] = quant_config
278
return config
vllm/transformers_utils/configs/parakeet.py
+15
-5
@@ -6,11 +6,21 @@ from transformers import ParakeetEncoderConfig, PretrainedConfig
6
7
8
class ParakeetConfig(ParakeetEncoderConfig):
9
- llm_hidden_size: int
10
- projection_hidden_size: int
11
- projection_bias: bool
12
- projection_eps: float = 1e-5
13
- sampling_rate: int
9
+ def __init__(
10
+ self,
11
+ llm_hidden_size: int,
12
+ projection_hidden_size: int,
13
+ projection_bias: bool,
14
+ sampling_rate: int,
15
+ projection_eps: float = 1e-5,
16
+ **kwargs,
17
+ ):
18
+ super().__init__(**kwargs)
19
+ self.llm_hidden_size = llm_hidden_size
20
+ self.projection_hidden_size = projection_hidden_size
21
+ self.projection_bias = projection_bias
22
+ self.sampling_rate = sampling_rate
23
+ self.projection_eps = projection_eps
24
25
@staticmethod
26
def from_hf_config(
vllm/transformers_utils/configs/qwen3_asr.py
+1
-1
@@ -408,7 +408,6 @@ class Qwen3ASRConfig(PretrainedConfig):
408
support_languages=None,
409
**kwargs,
410
):
411
- super().__init__(**kwargs)
411
if thinker_config is None:
412
thinker_config = {}
413
logger.info(
@@ -417,6 +416,7 @@ class Qwen3ASRConfig(PretrainedConfig):
416
417
self.thinker_config = Qwen3ASRThinkerConfig(**thinker_config)
418
self.support_languages = support_languages
419
+ super().__init__(**kwargs)
420
421
def get_text_config(self, decoder=False) -> "PretrainedConfig":
422
"""
vllm/transformers_utils/configs/speculators/base.py
+16
-5
@@ -1,6 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import os
4
+from dataclasses import fields, is_dataclass
5
from typing import Any
6
7
from transformers import PretrainedConfig
@@ -15,11 +16,21 @@ class SpeculatorsConfig(PretrainedConfig):
16
model_type = "speculators"
17
18
def __init__(self, **kwargs):
18
- """In Transformers v5, `PretrainedConfig` is decorated with `dataclass` and
19
- `huggingface_hub.dataclasses.strict(accept_kwargs=True)`.
20
- Inheriting classes do not inherit the `accept_kwargs=True` behaviour so we must
21
- explicitly pass any kwargs to `PretrainedConfig.__init__`."""
22
- super().__init__(**kwargs)
19
+ # Transformers v4 - super().__init__ which sets all kwargs as attributes
20
+ if not is_dataclass(PretrainedConfig):
21
+ return super().__init__(**kwargs)
22
+ # Transformers v5 - super().__init__ performs some validation before
23
+ # setting all kwargs as attributes, so we set them first to be safe
24
+ pre_trained_config_fields = {f.name for f in fields(PretrainedConfig)}
25
+ super_kwargs = dict()
26
+ for key, value in kwargs.items():
27
+ if key == "model_type":
28
+ continue # model_type is set as a class variable, so skip it here
29
+ elif key in pre_trained_config_fields:
30
+ super_kwargs[key] = value
31
+ else:
32
+ setattr(self, key, value)
33
+ super().__init__(**super_kwargs)
34
35
@classmethod
36
def from_pretrained(
vllm/transformers_utils/model_arch_config_convertor.py
+2
-2
@@ -228,7 +228,7 @@ class ModelArchConfigConvertorBase:
228
"pangu_ultra_moe_mtp",
229
"bailing_hybrid",
230
):
231
- return self.hf_text_config.kv_lora_rank is not None
231
+ return getattr(self.hf_text_config, "kv_lora_rank", None) is not None
232
elif self.hf_text_config.model_type == "eagle":
233
# if the model is an EAGLE module, check for the
234
# underlying architecture
@@ -241,7 +241,7 @@ class ModelArchConfigConvertorBase:
241
"deepseek_v32",
242
"deepseek_mtp",
243
)
244
- and self.hf_text_config.kv_lora_rank is not None
244
+ and getattr(self.hf_text_config, "kv_lora_rank", None) is not None
245
)
246
return False
247