A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[MISC] Keep chunked prefill enabled by default with long context when prefix caching is enabled (#8342)
Cody Yu committed
Sep 10, 2024 at 15:28 UTC
b1f3e189586dce42bb3dcda20169a9308c9a25fa
1 file changed
-1
vllm/engine/arg_utils.py
-1
@@ -878,7 +878,6 @@ class EngineArgs:
878
if (is_gpu and not use_sliding_window and not use_spec_decode
879
and not self.enable_lora
880
and not self.enable_prompt_adapter
881
- and not self.enable_prefix_caching
881
and not has_seqlen_agnostic_layers):
882
self.enable_chunked_prefill = True
883
logger.warning(