A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 18:30 UTC
[V1][BugFix] Detect interleaved sliding window attention (#14896)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Woosuk Kwon committed
Mar 16, 2025 at 14:53 UTC
31060b2757fb19ec67894b7c441383ceec9f1272
1 file changed
+9
-2
vllm/v1/worker/gpu_model_runner.py
+9
-2
@@ -82,8 +82,15 @@ class GPUModelRunner(LoRAModelRunnerMixin):
82
self.kv_cache_dtype = STR_DTYPE_TO_TORCH_DTYPE[
83
cache_config.cache_dtype]
84
85
- self.is_multimodal_model = model_config.is_multimodal_model
85
+ # NOTE(woosuk): sliding_window is None for models with interleaved
86
+ # attention. Use interleaved_sliding_window instead.
87
self.sliding_window = model_config.get_sliding_window()
88
+ self.interleaved_sliding_window = getattr(
89
+ model_config.hf_text_config, "interleaved_sliding_window", None)
90
+ self.window_size = (self.sliding_window
91
+ or self.interleaved_sliding_window)
92
+
93
+ self.is_multimodal_model = model_config.is_multimodal_model
94
self.block_size = cache_config.block_size
95
self.max_model_len = model_config.max_model_len
96
self.max_num_blocks_per_req = cdiv(self.max_model_len, self.block_size)
@@ -674,7 +681,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
681
num_query_heads=self.num_query_heads,
682
num_kv_heads=self.num_kv_heads,
683
use_alibi=False, # FIXME
677
- use_sliding_window=self.sliding_window is not None,
684
+ use_sliding_window=self.window_size is not None,
685
num_sms=self.num_sms,
686
)
687
return common_prefix_len if use_cascade else 0