A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 14:45 UTC
[Llama4] Enable attention temperature tuning by default for long context (>32k) (#16439)
Signed-off-by: Ye (Charlotte) Qi <yeq@meta.com> Co-authored-by: Ye (Charlotte) Qi <yeq@meta.com>
Yong Hoon Shin committed
Apr 10, 2025 at 21:26 UTC
99ef59cf7f93a6ce7d5ed3fd403db0aa962e5348
1 file changed
+6
-2
vllm/model_executor/models/llama4.py
+6
-2
@@ -467,11 +467,15 @@ class Llama4ForCausalLM(LlamaForCausalLM):
467
}
468
469
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
470
- # Update temperature tuning config from generation config
470
+ # update temperature tuning config from generation config
471
gen_config = vllm_config.model_config.try_get_generation_config()
472
gen_config.update(vllm_config.model_config.override_generation_config)
473
+ # enable temperature tuning by default when max_model_len > 32K
474
+ default_attn_temperature_tuning = \
475
+ vllm_config.model_config.max_model_len > 32768
476
vllm_config.model_config.hf_config.attn_temperature_tuning \
474
- = gen_config.get("attn_temperature_tuning", False)
477
+ = gen_config.get(
478
+ "attn_temperature_tuning", default_attn_temperature_tuning)
479
480
super().__init__(vllm_config=vllm_config,
481
prefix=prefix,