A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[BugFix] Fix EXAONE4 rotary embeddings (#23918)
Signed-off-by: lkm2835 <lkm2835@gmail.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Kyungmin Lee committed
Sep 2, 2025 at 23:40 UTC
38ba061f6f441ac60fb31c68b12385ae00ff7614
1 file changed
+3
-3
vllm/model_executor/models/exaone4.py
+3
-3
@@ -164,8 +164,8 @@ class Exaone4Attention(nn.Module):
164
is_sliding = config.layer_types[layer_idx] == "sliding_attention"
165
self.sliding_window = config.sliding_window if is_sliding else None
166
167
- # apply rotary embeddings to every layer
168
- self.apply_all_layers = not is_sliding
167
+ # apply rotary embeddings to every layer in full attention models
168
+ self.apply_rope_all_layers = "sliding_attention" not in config.layer_types
169
170
self.rotary_emb = get_rope(
171
self.head_dim,
@@ -201,7 +201,7 @@ class Exaone4Attention(nn.Module):
201
k = self.k_norm(k)
202
k = k.flatten(-2, -1)
203
204
- if self.sliding_window or self.apply_all_layers:
204
+ if self.sliding_window or self.apply_rope_all_layers:
205
q, k = self.rotary_emb(positions, q, k)
206
attn_output = self.attn(q, k, v)
207
output, _ = self.o_proj(attn_output)