A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 23:45 UTC
[Perf] Disable chunked local attention by default with llama4 (#21761)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Lucas Wilkinson committed
Jul 28, 2025 at 18:49 UTC
8aa1485fcff7be3e42300c0615ee0f3f3cbce9a8
2 files changed
+29
-6
vllm/config.py
+17
-6
@@ -4769,12 +4769,23 @@ class VllmConfig:
4769
# Hybrid KV cache manager is not compatible with KV events.
4770
self.scheduler_config.disable_hybrid_kv_cache_manager = True
4771
if self.model_config is not None and \
4772
- self.model_config.attention_chunk_size is not None and \
4773
- self.speculative_config is not None and \
4774
- self.speculative_config.use_eagle():
4775
- # Hybrid KV cache manager is not yet supported with chunked
4776
- # local attention + eagle.
4777
- self.scheduler_config.disable_hybrid_kv_cache_manager = True
4772
+ self.model_config.attention_chunk_size is not None:
4773
+ if self.speculative_config is not None and \
4774
+ self.speculative_config.use_eagle():
4775
+ # Hybrid KV cache manager is not yet supported with chunked
4776
+ # local attention + eagle.
4777
+ self.scheduler_config.disable_hybrid_kv_cache_manager = True
4778
+ elif \
4779
+ not envs.VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHE:
4780
+ logger.warning(
4781
+ "There is a latency regression when using chunked local"
4782
+ " attention with the hybrid KV cache manager. Disabling"
4783
+ " it, by default. To enable it, set the environment "
4784
+ "VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHE=1."
4785
+ )
4786
+ # Hybrid KV cache manager is not yet supported with chunked
4787
+ # local attention.
4788
+ self.scheduler_config.disable_hybrid_kv_cache_manager = True
4789
4790
def update_sizes_for_sequence_parallelism(self,
4791
possible_sizes: list) -> list:
vllm/envs.py
+12
@@ -143,6 +143,7 @@ if TYPE_CHECKING:
143
VLLM_USE_CUDNN_PREFILL: bool = False
144
VLLM_ENABLE_CUDAGRAPH_GC: bool = False
145
VLLM_LOOPBACK_IP: str = ""
146
+ VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHE: bool = False
147
148
149
def get_default_cache_root():
@@ -991,6 +992,17 @@ environment_variables: dict[str, Callable[[], Any]] = {
992
# The default value is "VLLM".
993
"VLLM_PROCESS_NAME_PREFIX":
994
lambda: os.getenv("VLLM_PROCESS_NAME_PREFIX", "VLLM"),
995
+
996
+ # Allow chunked local attention with hybrid kv cache manager.
997
+ # Currently using the Hybrid KV cache manager with chunked local attention
998
+ # in the Llama4 models (the only models currently using chunked local attn)
999
+ # causes a latency regression. For this reason, we disable it by default.
1000
+ # This flag is used to allow users to enable it if they want to (to save on
1001
+ # kv-cache memory usage and enable longer contexts)
1002
+ # TODO(lucas): Remove this flag once latency regression is resolved.
1003
+ "VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHE":
1004
+ lambda: bool(int(os.getenv(\
1005
+ "VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHE", "0"))),
1006
}
1007
1008
# --8<-- [end:env-vars-definition]