@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 96d673e0f8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Bugfix] Fix error handling of unsupported sliding window (#11213)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Dec 16, 2024 at 01:59 UTC 96d673e0f897aa8eec234e690c9c5425782d6ffb
1 file changed +8 -6
vllm/model_executor/models/llama.py
+8 -6
@@ -169,13 +169,15 @@ class LlamaAttention(nn.Module):
169 )
170
171 if hasattr(config, "interleaved_sliding_window"):
172 - if isinstance(config.interleaved_sliding_window, int):
173 - sliding_window = config.interleaved_sliding_window
174 - elif isinstance(config.interleaved_sliding_window, list):
175 - sw_idx = layer_idx % len(config.interleaved_sliding_window)
176 - sliding_window = config.interleaved_sliding_window[sw_idx]
172 + interleaved_sliding_window = config.interleaved_sliding_window
173 + if isinstance(interleaved_sliding_window, int):
174 + sliding_window = interleaved_sliding_window
175 + elif isinstance(interleaved_sliding_window, list):
176 + sw_idx = layer_idx % len(interleaved_sliding_window)
177 + sliding_window = interleaved_sliding_window[sw_idx]
178 else:
178 - raise ValueError(f"{type(sliding_window)} is not supported.")
179 + raise ValueError(
180 + f"{type(interleaved_sliding_window)} is not supported.")
181 else:
182 sliding_window = None
183