A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 08:30 UTC
[Bugfix] Fix Mistral-format models with sliding window (#18693)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
May 26, 2025 at 16:44 UTC
61a45e7a7266e29d14349cf67de69f166415e1c8
1 file changed
+10
-4
vllm/config.py
+10
-4
@@ -542,8 +542,10 @@ class ModelConfig:
542
sliding_window = getattr(self.hf_text_config, "sliding_window", None)
543
sliding_window_pattern = getattr(self.hf_text_config,
544
"sliding_window_pattern", None)
545
+ has_interleaved_attention = sliding_window_pattern is not None or (
546
+ isinstance(sliding_window, list))
547
546
- if not (self.disable_sliding_window or sliding_window_pattern is None):
548
+ if not self.disable_sliding_window and has_interleaved_attention:
549
if (backend :=
550
envs.VLLM_ATTENTION_BACKEND) in ("XFORMERS", "FLASHINFER"):
551
sliding_window_len_min = get_min_sliding_window(
@@ -563,7 +565,10 @@ class ModelConfig:
565
# only the attention layer itself is aware of the sliding
566
# window, and use the window size to compute the attention.
567
self.hf_text_config.interleaved_sliding_window = sliding_window
566
- delattr(self.hf_text_config, "sliding_window")
568
+
569
+ if hasattr(self.hf_text_config, "sliding_window"):
570
+ delattr(self.hf_text_config, "sliding_window")
571
+
572
sliding_window = None
573
574
self.max_model_len = _get_and_verify_max_len(
@@ -1041,7 +1046,8 @@ class ModelConfig:
1046
if self.use_async_output_proc:
1047
self.use_async_output_proc = False
1048
1044
- def get_hf_config_sliding_window(self) -> Optional[int]:
1049
+ def get_hf_config_sliding_window(
1050
+ self) -> Union[Optional[int], list[Optional[int]]]:
1051
"""Get the sliding window size, or None if disabled."""
1052
1053
# Some models, like Qwen2 and Qwen1.5, use `use_sliding_window` in
@@ -1052,7 +1058,7 @@ class ModelConfig:
1058
return None
1059
return getattr(self.hf_text_config, "sliding_window", None)
1060
1055
- def get_sliding_window(self) -> Optional[int]:
1061
+ def get_sliding_window(self) -> Optional[Union[int, list[Optional[int]]]]:
1062
"""Get the sliding window size, or None if disabled.
1063
"""
1064
# If user disables sliding window, return None.