Added sliding window attention support for qwen-eagle3 architecture (#47568)
Signed-off-by: shanjiaz <zsjwpianpian@gmail.com>
shanjiaz committed
Jul 13, 2026 at 16:20 UTC
7dc447dda7e6de38f3e1dcf63b0fc3e058a93e8e
2 files changed
+15
vllm/model_executor/models/qwen3.py
+4
@@ -78,6 +78,7 @@ class Qwen3Attention(nn.Module):
78
prefix: str = "",
79
attn_type: str = AttentionType.DECODER,
80
dual_chunk_attention_config: dict[str, Any] | None = None,
81
+ per_layer_sliding_window: int | None = None,
82
) -> None:
83
super().__init__()
84
self.hidden_size = hidden_size
@@ -136,6 +137,7 @@ class Qwen3Attention(nn.Module):
137
num_kv_heads=self.num_kv_heads,
138
cache_config=cache_config,
139
quant_config=quant_config,
140
+ per_layer_sliding_window=per_layer_sliding_window,
141
prefix=f"{prefix}.attn",
142
attn_type=attn_type,
143
**{
@@ -175,6 +177,7 @@ class Qwen3DecoderLayer(nn.Module):
177
cache_config: CacheConfig | None = None,
178
quant_config: QuantizationConfig | None = None,
179
prefix: str = "",
180
+ per_layer_sliding_window: int | None = None,
181
) -> None:
182
super().__init__()
183
self.hidden_size = config.hidden_size
@@ -206,6 +209,7 @@ class Qwen3DecoderLayer(nn.Module):
209
prefix=f"{prefix}.self_attn",
210
attn_type=attn_type,
211
dual_chunk_attention_config=dual_chunk_attention_config,
212
+ per_layer_sliding_window=per_layer_sliding_window,
213
)
214
self.mlp = Qwen3MLP(
215
hidden_size=self.hidden_size,
vllm/model_executor/models/qwen3_eagle3.py
+11
@@ -43,11 +43,22 @@ class Qwen3Eagle3DecoderLayer(Qwen3DecoderLayer):
43
cache_config = vllm_config.cache_config
44
quant_config = get_draft_quant_config(vllm_config)
45
46
+ # Resolve per-layer sliding window from draft config
47
+ sliding_window = None
48
+ layer_types = getattr(config, "layer_types", None)
49
+ if (
50
+ layer_types
51
+ and layer_idx < len(layer_types)
52
+ and layer_types[layer_idx] == "sliding_attention"
53
+ ):
54
+ sliding_window = getattr(config, "sliding_window", None)
55
+
56
super().__init__(
57
config=config,
58
cache_config=cache_config,
59
quant_config=quant_config,
60
prefix=prefix,
61
+ per_layer_sliding_window=sliding_window,
62
)
63
64
# First layer uses 2*hidden_size (embeds + hidden_states concatenated)