A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 04:45 UTC
[Llama4] Update `attn_temperature_tuning` (#19997)
Signed-off-by: Brayden Zhong <b8zhong@uwaterloo.ca>
Brayden Zhong committed
Jun 24, 2025 at 22:42 UTC
1afa9948f5931109cbe5201fee4fd7614ff7f904
1 file changed
+1
-2
vllm/model_executor/models/llama4.py
+1
-2
@@ -148,9 +148,8 @@ class Llama4Attention(nn.Module):
148
self.q_size = self.num_heads * self.head_dim
149
self.kv_size = self.num_kv_heads * self.head_dim
150
self.scaling = self.head_dim**-0.5
151
- # TODO: attn_temperature_tuning should be a bool in huggingface
151
self.attn_temperature_tuning = self.nope and \
153
- config.attn_temperature_tuning > 0
152
+ config.attn_temperature_tuning
153
154
self.floor_scale = getattr(config, "floor_scale", 8192.0)
155
self.attn_scale = getattr(config, "attn_scale", 0.1)