A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 16:00 UTC
[Bugfix] Use HF config fields as fallback when loading Mistral config (#29239)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Nov 23, 2025 at 02:22 UTC
d1cf8214e523ce664797b3f65a26ffdc6e81f032
4 files changed
+25
-4
.buildkite/test-amd.yaml
+1
@@ -754,6 +754,7 @@ steps:
754
torch_nightly: true
755
source_file_dependencies:
756
- vllm/model_executor/models/
757
+ - vllm/transformers_utils/
758
- tests/models/test_initialization.py
759
commands:
760
# Only when vLLM model source is modified - test initialization of a large
.buildkite/test-pipeline.yaml
+1
@@ -691,6 +691,7 @@ steps:
691
torch_nightly: true
692
source_file_dependencies:
693
- vllm/model_executor/models/
694
+ - vllm/transformers_utils/
695
- tests/models/test_initialization.py
696
commands:
697
# Only when vLLM model source is modified - test initialization of a large
vllm/transformers_utils/config.py
+13
-1
@@ -204,7 +204,19 @@ class MistralConfigParser(ConfigParserBase):
204
205
from vllm.transformers_utils.configs.mistral import adapt_config_dict
206
207
- config = adapt_config_dict(config_dict)
207
+ # Get missing fields from HF config if available
208
+ try:
209
+ hf_config_dict, _ = PretrainedConfig.get_config_dict(
210
+ model,
211
+ revision=revision,
212
+ code_revision=code_revision,
213
+ token=_get_hf_token(),
214
+ **kwargs,
215
+ )
216
+ except OSError: # Not found
217
+ hf_config_dict = {}
218
+
219
+ config = adapt_config_dict(config_dict, defaults=hf_config_dict)
220
221
# Mistral configs may define sliding_window as list[int]. Convert it
222
# to int and add the layer_types list[str] to make it HF compatible
vllm/transformers_utils/configs/mistral.py
+10
-3
@@ -9,14 +9,18 @@ from vllm.logger import init_logger
9
logger = init_logger(__name__)
10
11
12
-def adapt_config_dict(config_dict: dict[str, Any], **kwargs) -> PretrainedConfig:
13
- config_dict.update(kwargs)
12
+def adapt_config_dict(
13
+ config_dict: dict[str, Any],
14
+ defaults: dict[str, Any],
15
+) -> PretrainedConfig:
16
config_dict = _remap_general_mistral_args(config_dict)
17
18
if bool(config_dict.get("quantization")):
19
config_dict = _remap_mistral_quantization_args(config_dict)
20
19
- if bool(config_dict.get("moe")):
21
+ if config_dict.get("model_type") == "mamba":
22
+ config_dict["architectures"] = ["Mamba2ForCausalLM"]
23
+ elif bool(config_dict.get("moe")):
24
config_dict["architectures"] = ["MixtralForCausalLM"]
25
else:
26
config_dict["architectures"] = ["MistralForCausalLM"]
@@ -52,6 +56,9 @@ def adapt_config_dict(config_dict: dict[str, Any], **kwargs) -> PretrainedConfig
56
if is_audio:
57
config_dict = _remap_mistral_audio_args(config_dict)
58
59
+ for k, v in defaults.items():
60
+ config_dict.setdefault(k, v)
61
+
62
config = PretrainedConfig.from_dict(config_dict)
63
64
logger.debug("Initialized config %s", config)