A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 01:00 UTC
[BugFix][CPU] Fix `TorchSDPABackendImpl` doesn't have `use_irope` (#21200)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Lucas Wilkinson committed
Jul 19, 2025 at 02:18 UTC
468e2400feff561a7e8b5d4c455612662448fe72
1 file changed
+2
-1
vllm/v1/worker/gpu_model_runner.py
+2
-1
@@ -2668,7 +2668,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
2668
# TODO: Support other attention modules, e.g., cross-attention
2669
if attn_module.attn_type == AttentionType.DECODER:
2670
use_local_attention = (self.attention_chunk_size is not None
2671
- and attn_module.impl.use_irope)
2671
+ and getattr(attn_module.impl,
2672
+ "use_irope", False))
2673
if attn_module.sliding_window is not None:
2674
kv_cache_spec[layer_name] = SlidingWindowSpec(
2675
block_size=block_size,