A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 17:15 UTC
[Misc] Further clean up chunked prefill and prefix caching init (#29186)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Nov 22, 2025 at 19:34 UTC
5a4802588ed8f7918468986fce130c19ee721674
6 files changed
+33
-29
tests/engine/test_arg_utils.py
+1
-1
@@ -279,7 +279,7 @@ def test_prefix_cache_default():
279
args = parser.parse_args([])
280
281
engine_args = EngineArgs.from_cli_args(args=args)
282
- assert not engine_args.enable_prefix_caching, "prefix caching defaults to off."
282
+ assert engine_args.enable_prefix_caching, "prefix caching should default to on."
283
284
# with flag to turn it on.
285
args = parser.parse_args(["--enable-prefix-caching"])
tests/v1/core/test_scheduler.py
+7
-12
@@ -76,11 +76,11 @@ def test_get_num_unfinished_requests():
76
@pytest.mark.parametrize(
77
"enable_prefix_caching, prompt_logprobs",
78
[
79
- (None, None),
79
+ (False, None),
80
(True, 5),
81
],
82
)
83
-def test_schedule(enable_prefix_caching: bool | None, prompt_logprobs: int | None):
83
+def test_schedule(enable_prefix_caching: bool, prompt_logprobs: int | None):
84
"""Test scheduling.
85
Two cases: default APC/no prompt logprobs; APC=True + prompt logprobs
86
"""
@@ -582,12 +582,12 @@ def test_check_stop_min_tokens():
582
@pytest.mark.parametrize(
583
"enable_prefix_caching, prompt_logprobs",
584
[
585
- (None, None),
585
+ (False, None),
586
(True, 5),
587
],
588
)
589
def test_schedule_concurrent_batches(
590
- enable_prefix_caching: bool | None, prompt_logprobs: int | None
590
+ enable_prefix_caching: bool, prompt_logprobs: int | None
591
):
592
scheduler = create_scheduler(
593
max_num_batched_tokens=1024,
@@ -1425,7 +1425,7 @@ def create_scheduler_with_priority(
1425
model: str = "facebook/opt-125m",
1426
max_num_seqs: int = 16,
1427
max_num_batched_tokens: int = 8192,
1428
- enable_prefix_caching: bool | None = None,
1428
+ enable_prefix_caching: bool = False,
1429
long_prefill_token_threshold: int = 0,
1430
disable_chunked_mm_input: bool = False,
1431
use_kv_connector: bool = False,
@@ -1444,7 +1444,7 @@ def create_scheduler_with_priority(
1444
max_num_batch_tokens: max num tokens to batch
1445
enable_prefix_caching: optionally force APC config
1446
(True/False) or use default
1447
- (None)
1447
+ (False)
1448
1449
Returns:
1450
{class}`Scheduler` instance with priority scheduling
@@ -1467,17 +1467,12 @@ def create_scheduler_with_priority(
1467
seed=42,
1468
)
1469
# Cache config, optionally force APC
1470
- kwargs_cache = (
1471
- {}
1472
- if enable_prefix_caching is None
1473
- else {"enable_prefix_caching": enable_prefix_caching}
1474
- )
1470
cache_config = CacheConfig(
1471
block_size=block_size,
1472
gpu_memory_utilization=0.9,
1473
swap_space=0,
1474
cache_dtype="auto",
1480
- **kwargs_cache,
1475
+ enable_prefix_caching=enable_prefix_caching,
1476
)
1477
kv_transfer_config = (
1478
KVTransferConfig(
tests/v1/core/utils.py
+3
-8
@@ -42,7 +42,7 @@ def create_scheduler(
42
model: str = "facebook/opt-125m",
43
max_num_seqs: int = 16,
44
max_num_batched_tokens: int = 8192,
45
- enable_prefix_caching: bool | None = None,
45
+ enable_prefix_caching: bool = False,
46
long_prefill_token_threshold: int = 0,
47
disable_chunked_mm_input: bool = False,
48
use_kv_connector: None | bool | MockKVConfig = None,
@@ -63,7 +63,7 @@ def create_scheduler(
63
max_num_batch_tokens: max num tokens to batch
64
enable_prefix_caching: optionally force APC config
65
(True/False) or use default
66
- (None)
66
+ (False)
67
68
Returns:
69
{class}`Scheduler` instance
@@ -87,17 +87,12 @@ def create_scheduler(
87
skip_tokenizer_init=skip_tokenizer_init,
88
)
89
# Cache config, optionally force APC
90
- kwargs_cache = (
91
- {}
92
- if enable_prefix_caching is None
93
- else {"enable_prefix_caching": enable_prefix_caching}
94
- )
90
cache_config = CacheConfig(
91
block_size=block_size,
92
gpu_memory_utilization=0.9,
93
swap_space=0,
94
cache_dtype="auto",
100
- **kwargs_cache,
95
+ enable_prefix_caching=enable_prefix_caching,
96
)
97
kv_transfer_config = None
98
if isinstance(use_kv_connector, MockKVConfig):
vllm/config/cache.py
+2
-2
@@ -73,8 +73,8 @@ class CacheConfig:
73
sliding_window: int | None = None
74
"""Sliding window size for the KV cache. This is primarily set in
75
`ModelConfig` and that value should be manually duplicated here."""
76
- enable_prefix_caching: bool | None = None
77
- """Whether to enable prefix caching. Enabled by default for V1."""
76
+ enable_prefix_caching: bool = True
77
+ """Whether to enable prefix caching."""
78
prefix_caching_hash_algo: PrefixCachingHashAlgo = "sha256"
79
"""Set the hash algorithm for prefix caching:\n
80
- "sha256" uses Pickle for object serialization before hashing.\n
vllm/engine/arg_utils.py
+19
-5
@@ -425,7 +425,7 @@ class EngineArgs:
425
ParallelConfig.max_parallel_loading_workers
426
)
427
block_size: BlockSize | None = CacheConfig.block_size
428
- enable_prefix_caching: bool | None = CacheConfig.enable_prefix_caching
428
+ enable_prefix_caching: bool | None = None
429
prefix_caching_hash_algo: PrefixCachingHashAlgo = (
430
CacheConfig.prefix_caching_hash_algo
431
)
@@ -1975,10 +1975,11 @@ class EngineArgs:
1975
if self.prefill_context_parallel_size > 1:
1976
default_chunked_prefill = False
1977
default_prefix_caching = False
1978
- logger.warning(
1978
+ logger.warning_once(
1979
"--prefill-context-parallel-size > 1 is not compatible with "
1980
"chunked prefill and prefix caching now. Chunked prefill "
1981
- "and prefix caching have been disabled by default."
1981
+ "and prefix caching have been disabled by default.",
1982
+ scope="local",
1983
)
1984
1985
if self.enable_chunked_prefill is None:
@@ -1988,15 +1989,27 @@ class EngineArgs:
1989
"%s chunked prefill by default",
1990
"Enabling" if default_chunked_prefill else "Disabling",
1991
)
1992
+ elif (
1993
+ model_config.runner_type == "generate"
1994
+ and not self.enable_chunked_prefill
1995
+ and default_chunked_prefill
1996
+ ):
1997
+ logger.warning_once(
1998
+ "This model does not officially support disabling chunked prefill. "
1999
+ "Disabling this manually may cause the engine to crash "
2000
+ "or produce incorrect outputs.",
2001
+ scope="local",
2002
+ )
2003
elif (
2004
model_config.runner_type == "pooling"
2005
and self.enable_chunked_prefill
2006
and not default_chunked_prefill
2007
):
1996
- logger.warning(
2008
+ logger.warning_once(
2009
"This model does not officially support chunked prefill. "
2010
"Enabling this manually may cause the engine to crash "
2011
"or produce incorrect outputs.",
2012
+ scope="local",
2013
)
2014
2015
if self.enable_prefix_caching is None:
@@ -2011,10 +2024,11 @@ class EngineArgs:
2024
and self.enable_prefix_caching
2025
and not default_prefix_caching
2026
):
2014
- logger.warning(
2027
+ logger.warning_once(
2028
"This model does not officially support prefix caching. "
2029
"Enabling this manually may cause the engine to crash "
2030
"or produce incorrect outputs.",
2031
+ scope="local",
2032
)
2033
2034
world_size = self.pipeline_parallel_size * self.tensor_parallel_size
vllm/v1/core/sched/scheduler.py
+1
-1
@@ -180,7 +180,7 @@ class Scheduler(SchedulerInterface):
180
self.kv_cache_manager = KVCacheManager(
181
kv_cache_config=kv_cache_config,
182
max_model_len=self.max_model_len,
183
- enable_caching=bool(self.cache_config.enable_prefix_caching),
183
+ enable_caching=self.cache_config.enable_prefix_caching,
184
use_eagle=self.use_eagle,
185
log_stats=self.log_stats,
186
enable_kv_cache_events=self.enable_kv_cache_events,