[Bugfix] Fall back to T1 when ARC cannot reclaim enough entries from T2 (#55461)
Signed-off-by: Wang Zupeng <zupenwang@gmail.com> Co-authored-by: Codex <noreply@openai.com>
Zupeng Wang committed
Sep 6, 2026 at 18:28 UTC
dd0760165c17fdf750b1aa8e2a26ac885eafd598
2 files changed
+62
-1
tests/v1/kv_offload/cpu/test_manager.py
+55
@@ -782,6 +782,61 @@ class TestARCPolicy:
782
assert counting_t1.items_yielded == len(t1_order)
783
assert counting_t2.items_yielded == len(t2_order)
784
785
+ @pytest.mark.parametrize(
786
+ "pinned, protected, new_keys, expected_evicted",
787
+ [
788
+ ([], [], [7, 8, 9], [6, 5, 3]),
789
+ ([], [], [7, 8, 9, 10], [6, 5, 3, 4]),
790
+ ([5, 6], [], [7, 8], [3, 4]),
791
+ ([], [5, 6], [7, 8], [3, 4]),
792
+ ([5, 6], [3], [7], [4]),
793
+ ([5, 6], [3], [7, 8], None),
794
+ ],
795
+ )
796
+ def test_store_falls_back_to_t1_when_t2_cannot_satisfy_eviction(
797
+ self, pinned, protected, new_keys, expected_evicted
798
+ ):
799
+ """A preference for T2 must not reject stores that can reclaim T1."""
800
+ manager, policy = self._make_manager(enable_events=False)
801
+ for keys in ([1, 2, 3, 4], [5, 6]):
802
+ assert manager.prepare_store(to_keys(keys), _EMPTY_REQ_CTX) is not None
803
+ manager.complete_store(to_keys(keys), _EMPTY_REQ_CTX)
804
+
805
+ # Ghost hits raise the target to 3; promotion leaves only 2 entries in T1.
806
+ manager.touch(to_keys([1, 2]), _EMPTY_REQ_CTX)
807
+ manager.touch(to_keys([1]), _EMPTY_REQ_CTX)
808
+ manager.touch(to_keys([5, 6]), _EMPTY_REQ_CTX)
809
+ assert policy.target_t1_size == 3
810
+ assert list(policy.t1) == to_keys([3, 4])
811
+ assert list(policy.t2) == to_keys([6, 5])
812
+ manager.prepare_load(to_keys(pinned), _EMPTY_REQ_CTX)
813
+
814
+ counting_t1 = _CountingOrderedDict(policy.t1)
815
+ counting_t2 = _CountingOrderedDict(policy.t2)
816
+ policy.t1 = counting_t1
817
+ policy.t2 = counting_t2
818
+ before = [list(q.items()) for q in (policy.t1, policy.t2, policy.b1, policy.b2)]
819
+ counting_t1.items_yielded = counting_t2.items_yielded = 0
820
+ output = manager.prepare_store(to_keys(protected + new_keys), _EMPTY_REQ_CTX)
821
+ assert counting_t1.items_yielded <= len(before[0])
822
+ assert counting_t2.items_yielded <= len(before[1])
823
+
824
+ if expected_evicted is None:
825
+ assert output is None
826
+ assert [
827
+ list(q.items()) for q in (policy.t1, policy.t2, policy.b1, policy.b2)
828
+ ] == before
829
+ else:
830
+ assert output is not None
831
+ assert output.keys_to_store == to_keys(new_keys)
832
+ assert output.evicted_keys == to_keys(expected_evicted)
833
+ manager.complete_store(to_keys(new_keys), _EMPTY_REQ_CTX)
834
+ for key in new_keys:
835
+ assert manager.lookup(to_key(key), _EMPTY_REQ_CTX) is LookupResult.HIT
836
+ for key in pinned + protected:
837
+ assert manager.lookup(to_key(key), _EMPTY_REQ_CTX) is LookupResult.HIT
838
+ manager.complete_load(to_keys(pinned), _EMPTY_REQ_CTX)
839
+
840
def test_batch_eviction_failure_is_atomic(self):
841
"""Finding only some candidates must not partially evict the cache."""
842
cpu_manager, arc_policy = self._make_manager(num_blocks=4, enable_events=False)
vllm/v1/kv_offload/cpu/policies/arc.py
+7
-1
@@ -145,9 +145,15 @@ class ARCCachePolicy(CachePolicy):
145
146
if candidate is None:
147
entry = next_candidate(t2_iter)
148
+ if entry is not None:
149
+ candidate = (*entry, False)
150
+
151
+ if candidate is None:
152
+ entry = next_candidate(t1_iter)
153
if entry is None:
154
return None
150
- candidate = (*entry, False)
155
+ candidate = (*entry, True)
156
+ virtual_t1_size -= 1
157
158
candidates.append(candidate)
159