@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 017e9f4448

Promote `prefix_cache_retention_interval` to an argument and change the default to 0 (#52216)

Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com>

Tyler Michael Smith committed Aug 17, 2026 at 09:20 UTC 017e9f4448b700e85ee16023287b025693c72b9e

17 files changed +160 -63

tests/config/test_config_utils.py

@@ -216,6 +216,11 @@ def test_cache_config_hash_ignores_kv_cache_sizing_knobs():

216 assert CacheConfig(gpu_memory_utilization=0.5).compute_hash() == base_hash

217

218

219 +def test_cache_config_hash_ignores_prefix_cache_retention_interval():

220 + base_hash = CacheConfig().compute_hash()

221 + assert CacheConfig(prefix_cache_retention_interval=64).compute_hash() == base_hash

222 +

223 +

224 def test_envs_compile_factors_relocation_invariant(tmp_path):

225 """Relocating HOME or the XDG roots must not change the compile-cache

226 env hash.

tests/engine/test_arg_utils.py

+23

@@ -476,6 +476,7 @@ def test_prefix_cache_default():

476 # should be None by default (depends on model).

477 engine_args = EngineArgs.from_cli_args(args=args)

478 assert engine_args.enable_prefix_caching is None

479 + assert engine_args.prefix_cache_retention_interval == 0

480

481 # with flag to turn it on.

482 args = parser.parse_args(["--enable-prefix-caching"])

@@ -487,6 +488,28 @@ def test_prefix_cache_default():

488 engine_args = EngineArgs.from_cli_args(args=args)

489 assert not engine_args.enable_prefix_caching

490

491 + args = parser.parse_args(["--prefix-cache-retention-interval", "64"])

492 + engine_args = EngineArgs.from_cli_args(args=args)

493 + assert engine_args.prefix_cache_retention_interval == 64

494 +

495 +

496 +def test_prefix_cache_retention_interval_from_deprecated_env(

497 + monkeypatch, caplog, disable_log_dedup

498 +):

499 + monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", "64")

500 +

501 + engine_args = EngineArgs()

502 +

503 + assert engine_args.prefix_cache_retention_interval == 64

504 + assert "VLLM_PREFIX_CACHE_RETENTION_INTERVAL" in caplog.text

505 + assert "deprecated" in caplog.text

506 + assert "prefix_cache_retention_interval" in caplog.text

507 +

508 + parser = EngineArgs.add_cli_args(FlexibleArgumentParser())

509 + args = parser.parse_args(["--prefix-cache-retention-interval", "32"])

510 + engine_args = EngineArgs.from_cli_args(args)

511 + assert engine_args.prefix_cache_retention_interval == 32

512 +

513

514 @pytest.mark.parametrize(

515 ("arg", "expected", "option"),

tests/v1/core/test_contiguous_kv_packing.py

@@ -88,6 +88,7 @@ def _make_groups(n_c4, n_c128, n_swa):

88 def _mock_vllm_config(kv_connector_extra_config: dict[str, str] | None = None):

89 config = MagicMock()

90 config.cache_config.num_gpu_blocks_override = None

91 + config.cache_config.prefix_cache_retention_interval = 0

92 config.kv_transfer_config = None

93 if kv_connector_extra_config is not None:

94 config.kv_transfer_config = MagicMock()

@@ -322,6 +323,7 @@ class TestInterleavedPacking:

323 )

324

325 assert config.num_blocks == 32

326 + assert config.prefix_cache_retention_interval == 0

327 assert sum(t.size for t in config.kv_cache_tensors) == page_size * 2 * 32

328 assert config.kv_cache_tensors == [

329 KVCacheTensor(size=page_size * 32, shared_by=["full.0", "sw.0", "sw.1"]),

tests/v1/core/test_kv_cache_utils.py

@@ -816,6 +816,7 @@ def test_metrics_empty_stats():

816 def test_get_kv_cache_configs_multiple_workers():

817 model_config = ModelConfig(max_model_len=16)

818 vllm_config = VllmConfig(model_config=model_config)

819 + vllm_config.cache_config.prefix_cache_retention_interval = None

820

821 ref_kv_cache_spec = new_kv_cache_spec()

822 same_kv_cache_specs = [

@@ -1173,6 +1174,7 @@ def test_get_kv_cache_configs_multiple_workers():

1174 def test_get_kv_cache_configs_pp_sharding(asymmetric_memory):

1175 model_config = ModelConfig(max_model_len=512)

1176 vllm_config = VllmConfig(model_config=model_config)

1177 + vllm_config.cache_config.prefix_cache_retention_interval = None

1178

1179 ref_kv_cache_spec = new_kv_cache_spec()

1180 pp_kv_cache_specs = [

@@ -1702,6 +1704,7 @@ def test_get_kv_cache_config_one_worker():

1704 # pass max_model_len to pass check_enough_kv_cache_memory

1705 model_config = ModelConfig(max_model_len=16)

1706 vllm_config = VllmConfig(model_config=model_config)

1707 + vllm_config.cache_config.prefix_cache_retention_interval = None

1708

1709 mem_per_block_per_layer = 16 * 2 * 64 * 4 * 2

1710 # all layers are full attention -> single group

@@ -2015,6 +2018,7 @@ def test_get_kv_cache_config_one_worker():

2018 def test_get_kv_cache_configs_attention_free():

2019 kv_cache_specs: dict[str, KVCacheSpec] = {}

2020 vllm_config = VllmConfig(model_config=ModelConfig(max_model_len=16))

2021 + vllm_config.cache_config.prefix_cache_retention_interval = None

2022 kv_cache_configs = get_kv_cache_configs(vllm_config, [kv_cache_specs], [0])

2023 assert kv_cache_configs == [

2024 KVCacheConfig(

tests/v1/core/test_prefix_caching.py

+66 -44

@@ -4,6 +4,7 @@

5 import copy

6 from collections.abc import Callable

7 +from dataclasses import replace

8 from math import lcm

9 from types import SimpleNamespace

@@ -110,6 +111,11 @@ def make_kv_cache_manager(kv_cache_config: KVCacheConfig, **kwargs) -> KVCacheMa

111 "scheduler_block_size",

112 lcm(*(g.kv_cache_spec.block_size for g in kv_cache_config.kv_cache_groups)),

113 )

114 + if "retention_interval" in kwargs:

115 + kv_cache_config = replace(

116 + kv_cache_config,

117 + prefix_cache_retention_interval=kwargs.pop("retention_interval"),

118 + )

119 return KVCacheManager(kv_cache_config, **kwargs)

120

121

@@ -3119,9 +3125,8 @@ def test_hybrid_cache_blocks_clamped_to_lcm():

3125 )

3126

3127

3122 -def test_hybrid_local_kv_retention_interval_aligns_in_manager(monkeypatch):

3128 +def test_hybrid_local_kv_retention_interval_aligns_in_manager():

3129 """Verify fixed intervals retain sparse tails plus the latest replay tail."""

3124 - monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", "64")

3130 block_size = 8

3131 kv_cache_config = KVCacheConfig(

3132 num_blocks=100,

@@ -3153,6 +3158,7 @@ def test_hybrid_local_kv_retention_interval_aligns_in_manager(monkeypatch):

3158 max_model_len=8192,

3159 enable_caching=True,

3160 hash_block_size=block_size,

3161 + retention_interval=64,

3162 )

3163

3164 # The SWA manager uses the configured 64-token interval (a multiple of the

@@ -3184,18 +3190,15 @@ def test_hybrid_local_kv_retention_interval_aligns_in_manager(monkeypatch):

3190 "interval, expected_match",

3191 [

3192 # scheduler_block_size is 32 (= lcm(4*8, 8)); 33 is not a multiple of it.

3187 - ("33", "multiple of scheduler_block_size"),

3193 + (33, "multiple of scheduler_block_size"),

3194 # A negative multiple (-32 % 32 == 0) must still be rejected explicitly,

3195 # otherwise it would pass the modulo check and silently degrade to dense.

3190 - ("-32", "non-negative"),

3196 + (-32, "non-negative"),

3197 ],

3198 )

3193 -def test_hybrid_local_kv_retention_interval_rejects_invalid(

3194 - monkeypatch, interval, expected_match

3195 -):

3199 +def test_hybrid_local_kv_retention_interval_rejects_invalid(interval, expected_match):

3200 """A retention interval that is negative or not a multiple of

3201 scheduler_block_size errors out at construction time."""

3198 - monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", interval)

3202 block_size = 8

3203 kv_cache_config = KVCacheConfig(

3204 num_blocks=100,

@@ -3228,12 +3231,36 @@ def test_hybrid_local_kv_retention_interval_rejects_invalid(

3231 max_model_len=8192,

3232 enable_caching=True,

3233 hash_block_size=block_size,

3234 + retention_interval=interval,

3235 )

3236

3237

3234 -def test_hybrid_local_kv_retention_interval_survives_recycling(monkeypatch):

3238 +def test_zero_retention_is_ignored_for_full_attention():

3239 + kv_cache_config = make_kv_cache_config(block_size=16, num_blocks=10)

3240 + manager = make_kv_cache_manager(

3241 + kv_cache_config,

3242 + max_model_len=8192,

3243 + enable_caching=True,

3244 + hash_block_size=16,

3245 + retention_interval=0,

3246 + )

3247 + assert manager.coordinator.retention_interval == 0

3248 +

3249 +

3250 +def test_positive_retention_rejects_full_attention():

3251 + kv_cache_config = make_kv_cache_config(block_size=16, num_blocks=10)

3252 + with pytest.raises(ValueError, match="no sliding-window or Mamba"):

3253 + make_kv_cache_manager(

3254 + kv_cache_config,

3255 + max_model_len=8192,

3256 + enable_caching=True,

3257 + hash_block_size=16,

3258 + retention_interval=16,

3259 + )

3260 +

3261 +

3262 +def test_hybrid_local_kv_retention_interval_survives_recycling():

3263 """Verify retained local checkpoints are reused after block recycling."""

3236 - monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", "1024")

3264 hash_block_size = 4

3265 kv_cache_config = KVCacheConfig(

3266 num_blocks=800,

@@ -3286,6 +3313,7 @@ def test_hybrid_local_kv_retention_interval_survives_recycling(monkeypatch):

3313 max_model_len=4096,

3314 enable_caching=True,

3315 hash_block_size=hash_block_size,

3316 + retention_interval=1024,

3317 )

3318

3319 def fill_request(request_id: str, token_offset: int) -> list[int]:

@@ -3314,9 +3342,8 @@ def test_hybrid_local_kv_retention_interval_survives_recycling(monkeypatch):

3342 assert [len(blocks) for blocks in computed_blocks.blocks] == [4, 16, 128, 256]

3343

3344

3317 -def test_hybrid_local_kv_retention_latest_only_reuses_replay_boundary(monkeypatch):

3345 +def test_hybrid_local_kv_retention_latest_only_reuses_replay_boundary():

3346 """Verify latest-only retention reuses only the replayable prompt boundary."""

3319 - monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", "0")

3347 block_size = 8

3348 kv_cache_config = KVCacheConfig(

3349 num_blocks=100,

@@ -3348,6 +3375,7 @@ def test_hybrid_local_kv_retention_latest_only_reuses_replay_boundary(monkeypatc

3375 max_model_len=8192,

3376 enable_caching=True,

3377 hash_block_size=block_size,

3378 + retention_interval=0,

3379 )

3380

3381 token_ids = [i for i in range(16) for _ in range(block_size)]

@@ -3388,14 +3416,13 @@ def test_hybrid_local_kv_retention_latest_only_reuses_replay_boundary(monkeypatc

3416 assert len(computed_blocks.blocks[1]) == 0

3417

3418

3391 -def test_hybrid_local_kv_retention_mtp_reuses_latest_boundary(monkeypatch):

3419 +def test_hybrid_local_kv_retention_mtp_reuses_latest_boundary():

3420 """Verify MTP/EAGLE SWA retention keeps the extra proof block.

3421

3422 EAGLE/MTP lookup matches one additional local block after the returned

3423 prefix and then drops it. Sparse retention must therefore cache the normal

3424 local tail at the latest replay boundary plus one extra SWA block.

3425 """

3398 - monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", "0")

3426 block_size = 8

3427 kv_cache_config = KVCacheConfig(

3428 num_blocks=100,

@@ -3428,6 +3455,7 @@ def test_hybrid_local_kv_retention_mtp_reuses_latest_boundary(monkeypatch):

3455 max_model_len=8192,

3456 enable_caching=True,

3457 hash_block_size=block_size,

3458 + retention_interval=0,

3459 use_eagle=True,

3460 )

3461

@@ -3824,12 +3852,11 @@ def test_cache_hit_local_and_external_two_groups_preempt_and_reallocate():

3852 assert manager.get_blocks("test").get_block_ids() != ([], [])

3853

3854

3827 -def test_swa_free_split_keeps_cached_tail_ahead_of_scratch(monkeypatch):

3828 - """Default path (no retention): freeing an SWA request must place its

3855 +def test_swa_free_split_keeps_cached_tail_ahead_of_scratch():

3856 + """Dense retention: freeing an SWA request must place its

3857 uncached scratch blocks at the front of the free queue (recycled first)

3858 and keep its cached checkpoint blocks at the back (retained for prefix

3859 hits). This split is always-on, independent of the retention interval."""

3832 - monkeypatch.delenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", raising=False)

3860 block_size = 8

3861 kv_cache_config = KVCacheConfig(

3862 num_blocks=100,

@@ -3937,13 +3964,14 @@ def _make_pure_swa_manager(block_size, sliding_window, num_blocks=100, **kwargs)

3964 )

3965

3966

3940 -def test_pure_swa_retention_interval_caches_sparse_tails(monkeypatch):

3967 +def test_pure_swa_retention_interval_caches_sparse_tails():

3968 """Sparse retention must work for a pure-SWA single-group model, not just

3969 hybrid models: only the per-interval tails plus the latest replay tail are

3970 cached, and a replay still hits the latest replayable boundary."""

3944 - monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", "64")

3971 block_size = 16

3946 - manager = _make_pure_swa_manager(block_size, sliding_window=block_size)

3972 + manager = _make_pure_swa_manager(

3973 + block_size, sliding_window=block_size, retention_interval=64

3974 + )

3975 assert type(manager.coordinator).__name__ == "UnitaryKVCacheCoordinator"

3976

3977 token_ids = [i for i in range(16) for _ in range(block_size)]

@@ -3976,11 +4004,12 @@ def test_pure_swa_retention_interval_caches_sparse_tails(monkeypatch):

4004 assert num_computed == 240

4005

4006

3979 -def test_pure_swa_retention_latest_only(monkeypatch):

4007 +def test_pure_swa_retention_latest_only():

4008 """`=0` on a pure-SWA model keeps only the latest replay tail."""

3981 - monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", "0")

4009 block_size = 16

3983 - manager = _make_pure_swa_manager(block_size, sliding_window=block_size)

4010 + manager = _make_pure_swa_manager(

4011 + block_size, sliding_window=block_size, retention_interval=0

4012 + )

4013

4014 token_ids = [i for i in range(16) for _ in range(block_size)]

4015 req = make_request("0", token_ids, block_size, sha256)

@@ -4008,10 +4037,9 @@ def test_pure_swa_retention_latest_only(monkeypatch):

4037 assert num_computed == 240

4038

4039

4011 -def test_pure_swa_retention_dense_default_caches_all(monkeypatch):

4012 - """With retention unset, a pure-SWA model must keep the dense behavior:

4040 +def test_pure_swa_dense_retention_caches_all():

4041 + """With retention set to ``None``, a pure-SWA model keeps dense behavior:

4042 every block boundary is a potential hit, so all blocks are cached."""

4014 - monkeypatch.delenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", raising=False)

4043 block_size = 16

4044 manager = _make_pure_swa_manager(block_size, sliding_window=block_size)

4045

@@ -4037,7 +4065,7 @@ def test_pure_swa_retention_dense_default_caches_all(monkeypatch):

4065

4066

4067 def test_mamba_reachable_block_mask_sparsifies_retention():

4040 - """Mamba state-snapshot retention: with VLLM_PREFIX_CACHE_RETENTION_INTERVAL

4068 + """Mamba state-snapshot retention: with a configured retention interval,

4069 the manager keeps one cached state per interval-sized segment (plus the

4070 latest replay boundary) instead of a snapshot per block, which is what

4071 lets a small attention block_size avoid Mamba dominating the KV pool."""

@@ -4063,7 +4091,7 @@ def test_mamba_reachable_block_mask_sparsifies_retention():

4091 )

4092 return None if m is None else {i for i, v in enumerate(m) if v}

4093

4066 - # Dense default (None) -> no mask, every block cached (unchanged behavior).

4094 + # Dense retention (None) -> no mask, every block cached.

4095 assert retained(None) is None

4096 # interval == block_size -> every block is a boundary -> stays dense.

4097 assert retained(block_size) is None

@@ -4111,7 +4139,7 @@ def test_mamba_reachable_block_mask_pins_shared_prefix():

4139 assert retained(0, 100) == {5, 14}

4140 # Coexists with segment tails (interval 64 -> {3,7,11,15} + replay 14).

4141 assert retained(64, 96) == {3, 5, 7, 11, 14, 15}

4114 - # Dense default ignores the hint (nothing to sparsify).

4142 + # Dense retention ignores the hint (nothing to sparsify).

4143 assert retained(None, 96) is None

4144 # Out-of-range boundary is a no-op (only replay 14 remains).

4145 assert retained(0, 16 * block_size * 2) == {14}

@@ -4120,14 +4148,13 @@ def test_mamba_reachable_block_mask_pins_shared_prefix():

4148 assert retained(0, None) == {14}

4149

4150

4123 -def test_mamba_shared_prefix_survives_zero_retention(monkeypatch):

4151 +def test_mamba_shared_prefix_survives_zero_retention():

4152 """Manager-level check of the full wiring: a pinned shared-prefix boundary

4153 (``Request.shared_prefix_boundary``, set by the scheduler on Marconi-style

4154 detection) keeps its Mamba state block cached under

4127 - ``VLLM_PREFIX_CACHE_RETENTION_INTERVAL=0``, which otherwise retains only the

4155 + ``prefix_cache_retention_interval=0``, which otherwise retains only the

4156 end-of-prompt replay boundary. Without this, a shared prefix (junction

4157 before ``num_prompt``) would be recomputed by every sharing request."""

4130 - monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", "0")

4158 block_size = 16

4159

4160 # 16-block (256-token) prompt; replay boundary is block 240 // 16 - 1 = 14.

@@ -4140,6 +4167,7 @@ def test_mamba_shared_prefix_survives_zero_retention(monkeypatch):

4167 max_model_len=8192,

4168 enable_caching=True,

4169 hash_block_size=block_size,

4170 + retention_interval=0,

4171 )

4172 req = make_request("r", token_ids, block_size, sha256)

4173 req.shared_prefix_boundary = shared_prefix_boundary

@@ -4163,24 +4191,21 @@ def test_mamba_shared_prefix_survives_zero_retention(monkeypatch):

4191 assert cached_mamba_blocks(96) == {5, 14}

4192

4193

4166 -def test_mamba_shared_prefix_reuse_under_zero_retention(monkeypatch):

4194 +def test_mamba_shared_prefix_reuse_under_zero_retention():

4195 """Full cross-request Marconi flow: a partial shared prefix cached by the

4196 detecting request must stay reusable by a later request under

4169 - ``VLLM_PREFIX_CACHE_RETENTION_INTERVAL=0``. Without the pin the junction is

4197 + ``prefix_cache_retention_interval=0``. Without the pin the junction is

4198 masked out and the later request misses; with it (and under dense) the reuse

4199 is preserved."""

4200 block_size = 16

4201

4202 def last_req_hit(retention, pin):

4175 - if retention is None:

4176 - monkeypatch.delenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", raising=False)

4177 - else:

4178 - monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", str(retention))

4203 manager = make_kv_cache_manager(

4204 _make_hybrid_kv_cache_config(block_size, 200, ["full", "mamba_align"]),

4205 max_model_len=8192,

4206 enable_caching=True,

4207 hash_block_size=block_size,

4208 + retention_interval=retention,

4209 )

4210 shared = [7 for _ in range(2 * block_size)] # 2-block shared prefix

4211

@@ -4261,23 +4286,20 @@ def test_swa_reachable_block_mask_pins_shared_prefix():

4286 assert retained(0, 0, block_size) == {14}

4287

4288

4264 -def test_swa_shared_prefix_reuse_under_zero_retention(monkeypatch):

4289 +def test_swa_shared_prefix_reuse_under_zero_retention():

4290 """SWA cross-request analog: a partial shared prefix's sliding-window tail

4266 - must stay reusable under ``VLLM_PREFIX_CACHE_RETENTION_INTERVAL=0``. Without

4291 + must stay reusable under ``prefix_cache_retention_interval=0``. Without

4292 the pin the junction window is masked out and a later request misses; with

4293 it (and under dense) reuse is preserved."""

4294 block_size = 16

4295

4296 def last_req_hit(retention, pin):

4272 - if retention is None:

4273 - monkeypatch.delenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", raising=False)

4274 - else:

4275 - monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", str(retention))

4297 manager = make_kv_cache_manager(

4298 _make_hybrid_kv_cache_config(block_size, 200, ["full", "sliding_window"]),

4299 max_model_len=8192,

4300 enable_caching=True,

4301 hash_block_size=block_size,

4302 + retention_interval=retention,

4303 )

4304 shared = [7 for _ in range(4 * block_size)] # 4-block shared prefix

4305

tests/v1/engine/test_engine_args.py

@@ -18,6 +18,7 @@ def test_prefix_caching_from_cli():

18 assert vllm_config.cache_config.enable_prefix_caching, (

19 "V1 turns on prefix caching by default."

20 )

21 + assert vllm_config.cache_config.prefix_cache_retention_interval == 0

23 # Turn it off possible with flag.

24 args = parser.parse_args(["--no-enable-prefix-caching"])

@@ -47,6 +48,10 @@ def test_prefix_caching_from_cli():

48 with pytest.raises(ArgumentError):

49 args = parser.parse_args(["--prefix-caching-hash-algo", "invalid"])

51 + args = parser.parse_args(["--prefix-cache-retention-interval", "64"])

52 + vllm_config = EngineArgs.from_cli_args(args=args).create_engine_config()

53 + assert vllm_config.cache_config.prefix_cache_retention_interval == 64

54 +

56 @pytest.mark.skipif(_xxhash is None, reason="xxhash not installed")

57 def test_prefix_caching_xxhash_from_cli():

tests/v1/engine/test_engine_core_client.py

@@ -1066,6 +1066,7 @@ def test_kv_cache_events(

1066 model=model_name,

1067 enforce_eager=True,

1068 enable_prefix_caching=True,

1069 + prefix_cache_retention_interval=None,

1070 block_size=block_size,

1071 )

1072 engine_args.kv_events_config = publisher_config

tests/v1/kv_connector/unit/test_mooncake_store_worker.py

+5 -1

@@ -247,7 +247,9 @@ def _make_vllm_config(

247 )

248

249

250 -def _make_kv_cache_config(*, block_size: int = 16) -> object:

250 +def _make_kv_cache_config(

251 + *, block_size: int = 16, prefix_cache_retention_interval: int | None = 0

252 +) -> object:

253 """Minimal single-group KVCacheConfig for topology tests."""

254 from vllm.v1.kv_cache_interface import (

255 FullAttentionSpec,

@@ -262,6 +264,7 @@ def _make_kv_cache_config(*, block_size: int = 16) -> object:

264 num_blocks=10,

265 kv_cache_tensors=[],

266 kv_cache_groups=[KVCacheGroupSpec(["layer0"], spec)],

267 + prefix_cache_retention_interval=prefix_cache_retention_interval,

268 )

269

270

@@ -1594,6 +1597,7 @@ def test_requester_worker_init_uses_positional_setup(tmp_path, monkeypatch):

1597 "mlx5_0",

1598 "10.0.0.7:50051",

1599 )

1600 + assert w.coord.retention_interval == 0

1601

1602

1603 def test_requester_worker_init_prefers_local_hostname_override(

vllm/config/cache.py

+22 -1

@@ -7,7 +7,7 @@ from typing import Any, ClassVar, Literal

8 from pydantic import Field, field_validator, model_validator

10 -from vllm.config.utils import config

10 +from vllm.config.utils import config, get_from_deprecated_env_if_set

11 from vllm.logger import init_logger

12 from vllm.utils.torch_utils import (

13 is_quantized_kv_cache,

@@ -35,6 +35,17 @@ CacheDType = Literal[

35 "nvfp4",

36 "nvfp4_4over6",

37 ]

38 +

39 +

40 +def _get_prefix_cache_retention_interval() -> int | None:

41 + env_value = get_from_deprecated_env_if_set(

42 + "VLLM_PREFIX_CACHE_RETENTION_INTERVAL",

43 + "v0.29",

44 + "prefix_cache_retention_interval",

45 + )

46 + return 0 if env_value is None else int(env_value)

47 +

48 +

49 MambaDType = Literal["auto", "float32", "float16", "bfloat16"]

50 MambaCacheMode = Literal["all", "align", "none"]

51 PrefixCachingHashAlgo = Literal["sha256", "sha256_cbor", "xxhash", "xxhash_cbor"]

@@ -111,6 +122,15 @@ class CacheConfig:

122 security risk tolerance against the performance benefits before turning this on.

123 - "xxhash_cbor" combines canonical CBOR serialization with xxHash for

124 reproducible hashing. Requires the optional ``xxhash`` package."""

125 + prefix_cache_retention_interval: int | None = Field(

126 + default_factory=_get_prefix_cache_retention_interval, ge=0

127 + )

128 + """Token interval between retained sliding-window and Mamba prefix-cache

129 + checkpoints. ``0`` retains only semantic checkpoints, including the latest

130 + replay boundary and shared-prefix junctions. Positive values additionally

131 + retain periodic checkpoints at the specified interval, which must be a

132 + multiple of the scheduler block size. ``None`` retains checkpoints densely.

133 + Applies only to sliding-window and Mamba cache groups."""

134 kv_cache_dtype_skip_layers: list[str] = field(default_factory=list)

135 """Layer patterns to skip KV cache quantization. Accepts layer indices

136 (e.g., '0', '2', '4') or attention type names (e.g., 'sliding_window')."""

@@ -218,6 +238,7 @@ class CacheConfig:

238 "num_gpu_blocks_override",

239 "enable_prefix_caching",

240 "prefix_caching_hash_algo",

241 + "prefix_cache_retention_interval",

242 # Prefix-caching implementation detail (doesn't affect compiled graph).

243 "prefix_match_unit",

244 "mamba_page_size_padded",

vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py

+1 -1

@@ -1433,7 +1433,7 @@ class MooncakeStoreWorker:

1433 scheduler_block_size=self.block_size,

1434 hash_block_size=self.hash_block_size,

1435 use_eagle=use_eagle,

1436 - retention_interval=envs.VLLM_PREFIX_CACHE_RETENTION_INTERVAL,

1436 + retention_interval=kv_cache_config.prefix_cache_retention_interval,

1437 )

1438 # One ChunkedTokenDatabase per group; addresses populated in

1439 # register_kv_caches once the kv-cache layout is known. Each group's

vllm/engine/arg_utils.py

@@ -522,6 +522,9 @@ class EngineArgs:

522 prefix_caching_hash_algo: PrefixCachingHashAlgo = (

523 CacheConfig.prefix_caching_hash_algo

524 )

525 + prefix_cache_retention_interval: int | None = get_field(

526 + CacheConfig, "prefix_cache_retention_interval"

527 + )

528 disable_sliding_window: bool = ModelConfig.disable_sliding_window

529 disable_cascade_attn: bool = ModelConfig.disable_cascade_attn

530 offload_backend: str = OffloadConfig.offload_backend

@@ -1224,6 +1227,10 @@ class EngineArgs:

1227 cache_group.add_argument(

1228 "--prefix-caching-hash-algo", **cache_kwargs["prefix_caching_hash_algo"]

1229 )

1230 + cache_group.add_argument(

1231 + "--prefix-cache-retention-interval",

1232 + **cache_kwargs["prefix_cache_retention_interval"],

1233 + )

1234 cache_group.add_argument(

1235 "--kv-cache-dtype-skip-layers", **cache_kwargs["kv_cache_dtype_skip_layers"]

1236 )

@@ -2001,6 +2008,7 @@ class EngineArgs:

2008 sliding_window=sliding_window,

2009 enable_prefix_caching=self.enable_prefix_caching,

2010 prefix_caching_hash_algo=self.prefix_caching_hash_algo,

2011 + prefix_cache_retention_interval=self.prefix_cache_retention_interval,

2012 kv_cache_dtype_skip_layers=self.kv_cache_dtype_skip_layers,

2013 kv_sharing_fast_prefill=self.kv_sharing_fast_prefill,

2014 mamba_cache_dtype=self.mamba_cache_dtype,

vllm/envs.py

-6

@@ -1148,12 +1148,6 @@ environment_variables: dict[str, Callable[[], Any]] = {

1148 if "VLLM_PLUGINS" not in os.environ

1149 else os.environ["VLLM_PLUGINS"].split(",")

1150 ),

1151 - # Retain local sliding-window KV checkpoints for prefix caching.

1152 - # Unset (default) preserves the dense local checkpointing behavior. `0`

1153 - # retains only the latest completed prompt boundary. Positive values retain

1154 - # checkpoints at the specified interval boundaries (rounded up to the

1155 - # prefix-cache alignment).

1156 - # Applies to sliding-window attention for now but not yet Mamba/linear attention.

1151 "VLLM_PREFIX_CACHE_RETENTION_INTERVAL": lambda: (

1152 int(os.environ["VLLM_PREFIX_CACHE_RETENTION_INTERVAL"])

1153 if "VLLM_PREFIX_CACHE_RETENTION_INTERVAL" in os.environ

vllm/v1/core/kv_cache_coordinator.py

+6 -5

@@ -4,7 +4,6 @@ from abc import ABC, abstractmethod

4 from collections.abc import Sequence

5 from typing import NamedTuple

7 -from vllm import envs

7 from vllm.logger import init_logger

8 from vllm.utils.math_utils import cdiv, round_down

9 from vllm.v1.core.block_pool import BlockPool

@@ -45,16 +44,18 @@ def _validate_prefix_cache_retention_interval(

44 isinstance(g.kv_cache_spec, (SlidingWindowSpec, MambaSpec))

45 for g in kv_cache_config.kv_cache_groups

46 ):

47 + if retention_interval == 0:

48 + return

49 raise ValueError(

49 - "VLLM_PREFIX_CACHE_RETENTION_INTERVAL is set but this model has "

50 + "prefix_cache_retention_interval is set but this model has "

51 "no sliding-window or Mamba KV cache group, so retention has no "

51 - "effect. Unset it (it only applies to sliding-window and Mamba "

52 + "effect. Set it to 0 (it only applies to sliding-window and Mamba "

53 "attention)."

54 )

56 if retention_interval < 0 or retention_interval % scheduler_block_size != 0:

57 raise ValueError(

57 - f"VLLM_PREFIX_CACHE_RETENTION_INTERVAL ({retention_interval}) "

58 + f"prefix_cache_retention_interval ({retention_interval}) "

59 "must be non-negative and a multiple of scheduler_block_size "

60 f"({scheduler_block_size})."

61 )

@@ -151,7 +152,7 @@ class KVCacheCoordinator(ABC):

152 # A positive retention interval must be a multiple of the base hit granularity

153 # (``scheduler_block_size``) to land on real cache-hit boundaries.

154 # 0 = keep only the latest replay boundary; None = dense;

154 - self.retention_interval = envs.VLLM_PREFIX_CACHE_RETENTION_INTERVAL

155 + self.retention_interval = kv_cache_config.prefix_cache_retention_interval

156 _validate_prefix_cache_retention_interval(

157 self.retention_interval, self.scheduler_block_size, kv_cache_config

158 )

vllm/v1/core/kv_cache_manager.py

+1 -1

@@ -243,7 +243,7 @@ class KVCacheManager:

243 - ``shared_prefix_boundary``: the block-aligned token position of

244 a shared prefix that a sparse-retention group (Mamba / sliding

245 window) has not cached yet (Marconi-style APC), or 0 if none.

246 - Pinned so ``VLLM_PREFIX_CACHE_RETENTION_INTERVAL`` does not drop

246 + Pinned so sparse prefix-cache retention does not drop

247 the junction and defeat cross-request reuse.

248 """

249 # We skip finding the prefix cache hit when prefix caching is

vllm/v1/core/kv_cache_utils.py

@@ -1347,6 +1347,9 @@ def get_kv_cache_config_from_groups(

1347 num_blocks=1,

1348 kv_cache_tensors=[],

1349 kv_cache_groups=kv_cache_groups,

1350 + prefix_cache_retention_interval=(

1351 + vllm_config.cache_config.prefix_cache_retention_interval

1352 + ),

1353 )

1354

1355 # Determine how model runners should initialize the KV cache tensors.

@@ -1406,6 +1409,9 @@ def get_kv_cache_config_from_groups(

1409 num_blocks=num_blocks,

1410 kv_cache_tensors=kv_cache_tensors,

1411 kv_cache_groups=kv_cache_groups,

1412 + prefix_cache_retention_interval=(

1413 + vllm_config.cache_config.prefix_cache_retention_interval

1414 + ),

1415 )

1416

1417

vllm/v1/kv_cache_interface.py

@@ -970,6 +970,8 @@ class KVCacheConfig:

970 For models with multiple types of attention, there will be multiple groups,

971 see `_get_kv_cache_config_uniform_page_size` for more details.

972 """

973 + prefix_cache_retention_interval: int | None = None

974 + """Resolved retention policy for local prefix-cache checkpoints."""

975

976 @property

977 def has_mamba_layers(self) -> bool:

vllm/v1/simple_kv_offload/manager.py

+3 -4

@@ -4,7 +4,7 @@

5 import contextlib

6 from collections.abc import Iterable

7 -from dataclasses import dataclass, field

7 +from dataclasses import dataclass, field, replace

8 from typing import TYPE_CHECKING, Any

10 from vllm.config import VllmConfig

@@ -190,7 +190,6 @@ class SimpleCPUOffloadScheduler:

190 """Derive a CPU KVCacheConfig from the GPU config.

191 Same kv_cache_groups, num_blocks scaled by CPU/GPU memory ratio."""

192 # Import here to avoid potential circular imports

193 - from vllm.v1.kv_cache_interface import KVCacheConfig as KVCacheConfigCls

193 from vllm.v1.kv_cache_interface import KVCacheTensor

194

195 assert len(gpu_config.kv_cache_tensors) > 0

@@ -215,10 +214,10 @@ class SimpleCPUOffloadScheduler:

214 for t in gpu_config.kv_cache_tensors

215 ]

216

218 - return KVCacheConfigCls(

217 + return replace(

218 + gpu_config,

219 num_blocks=num_cpu_blocks,

220 kv_cache_tensors=cpu_tensors,

221 - kv_cache_groups=gpu_config.kv_cache_groups,

221 )

222

223 @staticmethod