A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 23:45 UTC
[V1] Fix local chunked attention always disabled (#21419)
Signed-off-by: Yong Hoon Shin <yhshin@meta.com>
Yong Hoon Shin committed
Jul 23, 2025 at 15:59 UTC
78c13e30e1641869672b4c5fb7685d04e58ca1df
1 file changed
+2
-1
vllm/attention/layer.py
+2
-1
@@ -143,6 +143,8 @@ class Attention(nn.Module):
143
# the backends)
144
if envs.VLLM_USE_V1:
145
self.use_irope = extra_impl_args.pop("use_irope", False)
146
+ else:
147
+ self.use_irope = extra_impl_args.get("use_irope", False)
148
149
quant_method = quant_config.get_quant_method(
150
self, prefix=prefix) if quant_config else None
@@ -177,7 +179,6 @@ class Attention(nn.Module):
179
kv_sharing_target_layer_name, **extra_impl_args)
180
self.backend = backend_name_to_enum(attn_backend.get_name())
181
self.dtype = dtype
180
- self.use_irope = extra_impl_args.get("use_irope", False)
182
183
# For cuda-alike (CUDA and ROCM) and cpu platforms, we control how
184
# torch.compile works by registering the attention as one giant