A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Hotfix][VLM] Fixing max position embeddings for Pixtral (#8399)
Roger Wang committed
Sep 12, 2024 at 09:28 UTC
520ca380aef75f34cd2f5a146d30849b483e3be4
1 file changed
+2
vllm/transformers_utils/config.py
+2
@@ -206,6 +206,8 @@ def load_params_config(model, revision) -> PretrainedConfig:
206
config_dict["tie_word_embeddings"] = config_dict.get(
207
"tie_embeddings", False)
208
config_dict["max_seq_len"] = config_dict.get("max_seq_len", 128_000)
209
+ config_dict["max_position_embeddings"] = config_dict.get(
210
+ "max_position_embeddings", 128_000)
211
212
if config_dict.get("moe") is not None:
213
config_dict["architectures"] = ["MixtralForCausalLM"]