A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 09:45 UTC
[BugFix] Correct max_model_len derivation from config.json for Mistral format (#17937)
Signed-off-by: 汪志鹏 <wangzhipeng628@gmail.com> Co-authored-by: tracelogfb <48808670+tracelogfb@users.noreply.github.com> Co-authored-by: Stephen Chen <tracelog@meta.com>
汪志鹏 committed
May 16, 2025 at 21:20 UTC
4ee4826ede9706ca33492ead69fef99d0b0b0667
1 file changed
+18
-3
vllm/transformers_utils/config.py
+18
-3
@@ -686,9 +686,24 @@ def load_params_config(model: Union[str, Path], revision: Optional[str],
686
config_dict["hidden_act"] = config_dict.get("activation", "silu")
687
config_dict["tie_word_embeddings"] = config_dict.get(
688
"tie_embeddings", False)
689
- config_dict["max_seq_len"] = config_dict.get("max_seq_len", 128_000)
690
- config_dict["max_position_embeddings"] = config_dict.get(
691
- "max_position_embeddings", 128_000)
689
+
690
+ if config_dict.get("max_position_embeddings") is None:
691
+ max_position_embeddings = 128_000
692
+ try:
693
+ trust_remote_code_val = kwargs.get("trust_remote_code", False)
694
+ hf_config = get_config(model=model,
695
+ trust_remote_code=trust_remote_code_val,
696
+ revision=revision,
697
+ config_format=ConfigFormat.HF)
698
+ if hf_value := hf_config.get_text_config().max_position_embeddings:
699
+ max_position_embeddings = hf_value
700
+ except Exception as e:
701
+ logger.warning(
702
+ "The params.json file is missing 'max_position_embeddings'"
703
+ " and could not get a value from the HF config."
704
+ " Defaulting to 128000",
705
+ exc_info=e)
706
+ config_dict["max_position_embeddings"] = max_position_embeddings
707
708
if config_dict.get("quantization") is not None:
709
quantization = config_dict.get("quantization", {})