A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 23:45 UTC
[BugFix] Fix interleaved sliding window not set for Gemma3n (#21863)
Signed-off-by: Yong Hoon Shin <yhshin@meta.com>
Yong Hoon Shin committed
Jul 29, 2025 at 16:34 UTC
9266d980480c8da52a0c29960e9086128e19d664
2 files changed
+14
-4
vllm/config.py
+7
-2
@@ -723,11 +723,16 @@ class ModelConfig:
723
)
724
725
# Workaround for Gemma 2 which uses interleaved sliding window
726
- # attention, but it's not specified in its config. TODO: remove this
727
- # when Gemma 2 is fixed in Transformers.
726
+ # attention, but it's not specified in its config.
727
+ # TODO: remove this when Gemma 2 config updated in HuggingFace.
728
if self.hf_text_config.model_type == "gemma2":
729
self.hf_text_config.sliding_window_pattern = 2
730
731
+ # TODO: remove this when Gemma 3n config updated in HuggingFace.
732
+ if self.hf_text_config.model_type == "gemma3n_text":
733
+ # 4 sliding window attention followed by 1 full attention
734
+ self.hf_text_config.sliding_window_pattern = "LLLLG"
735
+
736
sliding_window = getattr(self.hf_text_config, "sliding_window", None)
737
sliding_window_pattern = getattr(self.hf_text_config,
738
"sliding_window_pattern", None)
vllm/model_executor/models/gemma3n.py
+7
-2
@@ -297,8 +297,13 @@ class Gemma3nAttention(nn.Module):
297
has_weight=False)
298
299
layer_idx = extract_layer_index(prefix)
300
- if config.layer_types[layer_idx] == "sliding_attention":
301
- self.sliding_window = config.sliding_window
300
+
301
+ is_sliding_window = (
302
+ getattr(config, "interleaved_sliding_window", None) is not None
303
+ and config.layer_types[layer_idx] == "sliding_attention")
304
+
305
+ if is_sliding_window:
306
+ self.sliding_window = config.interleaved_sliding_window
307
rope_theta = config.rope_local_base_freq
308
rope_scaling = {"rope_type": "default"}
309
else: