A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Bugfix] Fix error handling of unsupported sliding window (#11213)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Dec 16, 2024 at 01:59 UTC
96d673e0f897aa8eec234e690c9c5425782d6ffb
1 file changed
+8
-6
vllm/model_executor/models/llama.py
+8
-6
@@ -169,13 +169,15 @@ class LlamaAttention(nn.Module):
169
)
170
171
if hasattr(config, "interleaved_sliding_window"):
172
- if isinstance(config.interleaved_sliding_window, int):
173
- sliding_window = config.interleaved_sliding_window
174
- elif isinstance(config.interleaved_sliding_window, list):
175
- sw_idx = layer_idx % len(config.interleaved_sliding_window)
176
- sliding_window = config.interleaved_sliding_window[sw_idx]
172
+ interleaved_sliding_window = config.interleaved_sliding_window
173
+ if isinstance(interleaved_sliding_window, int):
174
+ sliding_window = interleaved_sliding_window
175
+ elif isinstance(interleaved_sliding_window, list):
176
+ sw_idx = layer_idx % len(interleaved_sliding_window)
177
+ sliding_window = interleaved_sliding_window[sw_idx]
178
else:
178
- raise ValueError(f"{type(sliding_window)} is not supported.")
179
+ raise ValueError(
180
+ f"{type(interleaved_sliding_window)} is not supported.")
181
else:
182
sliding_window = None
183