A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Misc] Move `LRUCache` into its own file (#26342)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Oct 7, 2025 at 23:08 UTC
c0a7b89d8e7e08af8c75139c8af6e105ac20112f
6 files changed
+349
-378
tests/utils_/test_cache.py
new
+125
@@ -0,0 +1,125 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+from vllm.utils.cache import CacheInfo, LRUCache
4
+
5
+
6
+class TestLRUCache(LRUCache):
7
+ def _on_remove(self, key, value):
8
+ if not hasattr(self, "_remove_counter"):
9
+ self._remove_counter = 0
10
+ self._remove_counter += 1
11
+
12
+
13
+def test_lru_cache():
14
+ cache = TestLRUCache(3)
15
+ assert cache.stat() == CacheInfo(hits=0, total=0)
16
+ assert cache.stat(delta=True) == CacheInfo(hits=0, total=0)
17
+
18
+ cache.put(1, 1)
19
+ assert len(cache) == 1
20
+
21
+ cache.put(1, 1)
22
+ assert len(cache) == 1
23
+
24
+ cache.put(2, 2)
25
+ assert len(cache) == 2
26
+
27
+ cache.put(3, 3)
28
+ assert len(cache) == 3
29
+ assert set(cache.cache) == {1, 2, 3}
30
+
31
+ cache.put(4, 4)
32
+ assert len(cache) == 3
33
+ assert set(cache.cache) == {2, 3, 4}
34
+ assert cache._remove_counter == 1
35
+
36
+ assert cache.get(2) == 2
37
+ assert cache.stat() == CacheInfo(hits=1, total=1)
38
+ assert cache.stat(delta=True) == CacheInfo(hits=1, total=1)
39
+
40
+ assert cache[2] == 2
41
+ assert cache.stat() == CacheInfo(hits=2, total=2)
42
+ assert cache.stat(delta=True) == CacheInfo(hits=1, total=1)
43
+
44
+ cache.put(5, 5)
45
+ assert set(cache.cache) == {2, 4, 5}
46
+ assert cache._remove_counter == 2
47
+
48
+ assert cache.pop(5) == 5
49
+ assert len(cache) == 2
50
+ assert set(cache.cache) == {2, 4}
51
+ assert cache._remove_counter == 3
52
+
53
+ assert cache.get(-1) is None
54
+ assert cache.stat() == CacheInfo(hits=2, total=3)
55
+ assert cache.stat(delta=True) == CacheInfo(hits=0, total=1)
56
+
57
+ cache.pop(10)
58
+ assert len(cache) == 2
59
+ assert set(cache.cache) == {2, 4}
60
+ assert cache._remove_counter == 3
61
+
62
+ cache.get(10)
63
+ assert len(cache) == 2
64
+ assert set(cache.cache) == {2, 4}
65
+ assert cache._remove_counter == 3
66
+
67
+ cache.put(6, 6)
68
+ assert len(cache) == 3
69
+ assert set(cache.cache) == {2, 4, 6}
70
+ assert 2 in cache
71
+ assert 4 in cache
72
+ assert 6 in cache
73
+
74
+ cache.remove_oldest()
75
+ assert len(cache) == 2
76
+ assert set(cache.cache) == {2, 6}
77
+ assert cache._remove_counter == 4
78
+
79
+ cache.clear()
80
+ assert len(cache) == 0
81
+ assert cache._remove_counter == 6
82
+ assert cache.stat() == CacheInfo(hits=0, total=0)
83
+ assert cache.stat(delta=True) == CacheInfo(hits=0, total=0)
84
+
85
+ cache._remove_counter = 0
86
+
87
+ cache[1] = 1
88
+ assert len(cache) == 1
89
+
90
+ cache[1] = 1
91
+ assert len(cache) == 1
92
+
93
+ cache[2] = 2
94
+ assert len(cache) == 2
95
+
96
+ cache[3] = 3
97
+ assert len(cache) == 3
98
+ assert set(cache.cache) == {1, 2, 3}
99
+
100
+ cache[4] = 4
101
+ assert len(cache) == 3
102
+ assert set(cache.cache) == {2, 3, 4}
103
+ assert cache._remove_counter == 1
104
+ assert cache[2] == 2
105
+
106
+ cache[5] = 5
107
+ assert set(cache.cache) == {2, 4, 5}
108
+ assert cache._remove_counter == 2
109
+
110
+ del cache[5]
111
+ assert len(cache) == 2
112
+ assert set(cache.cache) == {2, 4}
113
+ assert cache._remove_counter == 3
114
+
115
+ cache.pop(10)
116
+ assert len(cache) == 2
117
+ assert set(cache.cache) == {2, 4}
118
+ assert cache._remove_counter == 3
119
+
120
+ cache[6] = 6
121
+ assert len(cache) == 3
122
+ assert set(cache.cache) == {2, 4, 6}
123
+ assert 2 in cache
124
+ assert 4 in cache
125
+ assert 6 in cache
tests/utils_/test_utils.py
-126
@@ -23,11 +23,8 @@ from vllm_test_utils.monitor import monitor
23
from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
24
from vllm.transformers_utils.detokenizer_utils import convert_ids_list_to_tokens
25
26
-# isort: off
26
from vllm.utils import (
28
- CacheInfo,
27
FlexibleArgumentParser,
30
- LRUCache,
28
MemorySnapshot,
29
PlaceholderModule,
30
bind_kv_cache,
@@ -50,7 +47,6 @@ from vllm.utils import (
47
unique_filepath,
48
)
49
53
-# isort: on
50
from ..utils import create_new_process_for_each_test, error_on_warning
51
52
@@ -557,128 +553,6 @@ def test_bind_kv_cache_pp():
553
assert ctx["layers.0.self_attn"].kv_cache[1] is kv_cache[1][0]
554
555
560
-class TestLRUCache(LRUCache):
561
- def _on_remove(self, key, value):
562
- if not hasattr(self, "_remove_counter"):
563
- self._remove_counter = 0
564
- self._remove_counter += 1
565
-
566
-
567
-def test_lru_cache():
568
- cache = TestLRUCache(3)
569
- assert cache.stat() == CacheInfo(hits=0, total=0)
570
- assert cache.stat(delta=True) == CacheInfo(hits=0, total=0)
571
-
572
- cache.put(1, 1)
573
- assert len(cache) == 1
574
-
575
- cache.put(1, 1)
576
- assert len(cache) == 1
577
-
578
- cache.put(2, 2)
579
- assert len(cache) == 2
580
-
581
- cache.put(3, 3)
582
- assert len(cache) == 3
583
- assert set(cache.cache) == {1, 2, 3}
584
-
585
- cache.put(4, 4)
586
- assert len(cache) == 3
587
- assert set(cache.cache) == {2, 3, 4}
588
- assert cache._remove_counter == 1
589
-
590
- assert cache.get(2) == 2
591
- assert cache.stat() == CacheInfo(hits=1, total=1)
592
- assert cache.stat(delta=True) == CacheInfo(hits=1, total=1)
593
-
594
- assert cache[2] == 2
595
- assert cache.stat() == CacheInfo(hits=2, total=2)
596
- assert cache.stat(delta=True) == CacheInfo(hits=1, total=1)
597
-
598
- cache.put(5, 5)
599
- assert set(cache.cache) == {2, 4, 5}
600
- assert cache._remove_counter == 2
601
-
602
- assert cache.pop(5) == 5
603
- assert len(cache) == 2
604
- assert set(cache.cache) == {2, 4}
605
- assert cache._remove_counter == 3
606
-
607
- assert cache.get(-1) is None
608
- assert cache.stat() == CacheInfo(hits=2, total=3)
609
- assert cache.stat(delta=True) == CacheInfo(hits=0, total=1)
610
-
611
- cache.pop(10)
612
- assert len(cache) == 2
613
- assert set(cache.cache) == {2, 4}
614
- assert cache._remove_counter == 3
615
-
616
- cache.get(10)
617
- assert len(cache) == 2
618
- assert set(cache.cache) == {2, 4}
619
- assert cache._remove_counter == 3
620
-
621
- cache.put(6, 6)
622
- assert len(cache) == 3
623
- assert set(cache.cache) == {2, 4, 6}
624
- assert 2 in cache
625
- assert 4 in cache
626
- assert 6 in cache
627
-
628
- cache.remove_oldest()
629
- assert len(cache) == 2
630
- assert set(cache.cache) == {2, 6}
631
- assert cache._remove_counter == 4
632
-
633
- cache.clear()
634
- assert len(cache) == 0
635
- assert cache._remove_counter == 6
636
- assert cache.stat() == CacheInfo(hits=0, total=0)
637
- assert cache.stat(delta=True) == CacheInfo(hits=0, total=0)
638
-
639
- cache._remove_counter = 0
640
-
641
- cache[1] = 1
642
- assert len(cache) == 1
643
-
644
- cache[1] = 1
645
- assert len(cache) == 1
646
-
647
- cache[2] = 2
648
- assert len(cache) == 2
649
-
650
- cache[3] = 3
651
- assert len(cache) == 3
652
- assert set(cache.cache) == {1, 2, 3}
653
-
654
- cache[4] = 4
655
- assert len(cache) == 3
656
- assert set(cache.cache) == {2, 3, 4}
657
- assert cache._remove_counter == 1
658
- assert cache[2] == 2
659
-
660
- cache[5] = 5
661
- assert set(cache.cache) == {2, 4, 5}
662
- assert cache._remove_counter == 2
663
-
664
- del cache[5]
665
- assert len(cache) == 2
666
- assert set(cache.cache) == {2, 4}
667
- assert cache._remove_counter == 3
668
-
669
- cache.pop(10)
670
- assert len(cache) == 2
671
- assert set(cache.cache) == {2, 4}
672
- assert cache._remove_counter == 3
673
-
674
- cache[6] = 6
675
- assert len(cache) == 3
676
- assert set(cache.cache) == {2, 4, 6}
677
- assert 2 in cache
678
- assert 4 in cache
679
- assert 6 in cache
680
-
681
-
556
@pytest.mark.parametrize(
557
("src_dtype", "tgt_dtype", "expected_result"),
558
[
vllm/lora/models.py
+2
-1
@@ -32,7 +32,8 @@ from vllm.model_executor.models.interfaces import is_pooling_model
32
from vllm.model_executor.models.module_mapping import MultiModelKeys
33
from vllm.model_executor.models.utils import PPMissingLayer, WeightsMapper
34
from vllm.model_executor.utils import get_packed_modules_mapping
35
-from vllm.utils import LRUCache, is_pin_memory_available
35
+from vllm.utils import is_pin_memory_available
36
+from vllm.utils.cache import LRUCache
37
38
logger = init_logger(__name__)
39
vllm/multimodal/cache.py
+2
-1
@@ -17,7 +17,8 @@ from vllm.distributed.device_communicators.shm_object_storage import (
17
)
18
from vllm.envs import VLLM_OBJECT_STORAGE_SHM_BUFFER_NAME
19
from vllm.logger import init_logger
20
-from vllm.utils import GiB_bytes, LRUCache, MiB_bytes
20
+from vllm.utils import GiB_bytes, MiB_bytes
21
+from vllm.utils.cache import LRUCache
22
from vllm.utils.jsontree import json_count_leaves, json_map_leaves, json_reduce_leaves
23
24
from .inputs import (
vllm/utils/__init__.py
-250
@@ -51,7 +51,6 @@ from collections.abc import (
51
Hashable,
52
Iterable,
53
Iterator,
54
- KeysView,
54
Mapping,
55
Sequence,
56
)
@@ -60,24 +59,19 @@ from concurrent.futures.process import ProcessPoolExecutor
59
from dataclasses import dataclass, field
60
from functools import cache, lru_cache, partial, wraps
61
from pathlib import Path
63
-from types import MappingProxyType
62
from typing import (
63
TYPE_CHECKING,
64
Any,
65
Callable,
66
Generic,
67
Literal,
70
- NamedTuple,
68
TextIO,
69
TypeVar,
70
Union,
74
- cast,
75
- overload,
71
)
72
from urllib.parse import urlparse
73
from uuid import uuid4
74
80
-import cachetools
75
import cbor2
76
import cloudpickle
77
import numpy as np
@@ -183,13 +177,6 @@ U = TypeVar("U")
177
178
_K = TypeVar("_K", bound=Hashable)
179
_V = TypeVar("_V")
186
-_T = TypeVar("_T")
187
-
188
-
189
-class _Sentinel: ...
190
-
191
-
192
-ALL_PINNED_SENTINEL = _Sentinel()
180
181
182
class Device(enum.Enum):
@@ -215,243 +202,6 @@ class Counter:
202
self.counter = 0
203
204
218
-class _MappingOrderCacheView(UserDict[_K, _V]):
219
- def __init__(self, data: Mapping[_K, _V], ordered_keys: Mapping[_K, None]):
220
- super().__init__(data)
221
- self.ordered_keys = ordered_keys
222
-
223
- def __iter__(self) -> Iterator[_K]:
224
- return iter(self.ordered_keys)
225
-
226
- def keys(self) -> KeysView[_K]:
227
- return KeysView(self.ordered_keys)
228
-
229
-
230
-class CacheInfo(NamedTuple):
231
- hits: int
232
- total: int
233
-
234
- @property
235
- def hit_ratio(self) -> float:
236
- if self.total == 0:
237
- return 0
238
-
239
- return self.hits / self.total
240
-
241
- def __sub__(self, other: CacheInfo):
242
- return CacheInfo(
243
- hits=self.hits - other.hits,
244
- total=self.total - other.total,
245
- )
246
-
247
-
248
-class LRUCache(cachetools.LRUCache[_K, _V], Generic[_K, _V]):
249
- def __init__(self, capacity: float, getsizeof: Callable[[_V], float] | None = None):
250
- super().__init__(capacity, getsizeof)
251
-
252
- self.pinned_items = set[_K]()
253
-
254
- self._hits = 0
255
- self._total = 0
256
- self._last_info = CacheInfo(hits=0, total=0)
257
-
258
- def __getitem__(self, key: _K, *, update_info: bool = True) -> _V:
259
- value = super().__getitem__(key)
260
-
261
- if update_info:
262
- self._hits += 1
263
- self._total += 1
264
-
265
- return value
266
-
267
- def __delitem__(self, key: _K) -> None:
268
- run_on_remove = key in self
269
- value = self.__getitem__(key, update_info=False) # type: ignore[call-arg]
270
- super().__delitem__(key)
271
- if key in self.pinned_items:
272
- # Todo: add warning to inform that del pinned item
273
- self._unpin(key)
274
- if run_on_remove:
275
- self._on_remove(key, value)
276
-
277
- @property
278
- def cache(self) -> Mapping[_K, _V]:
279
- """Return the internal cache dictionary in order (read-only)."""
280
- return _MappingOrderCacheView(
281
- self._Cache__data, # type: ignore
282
- self.order,
283
- )
284
-
285
- @property
286
- def order(self) -> Mapping[_K, None]:
287
- """Return the internal order dictionary (read-only)."""
288
- return MappingProxyType(self._LRUCache__order) # type: ignore
289
-
290
- @property
291
- def capacity(self) -> float:
292
- return self.maxsize
293
-
294
- @property
295
- def usage(self) -> float:
296
- if self.maxsize == 0:
297
- return 0
298
-
299
- return self.currsize / self.maxsize
300
-
301
- def stat(self, *, delta: bool = False) -> CacheInfo:
302
- """
303
- Gets the cumulative number of hits and queries against this cache.
304
-
305
- If `delta=True`, instead gets these statistics
306
- since the last call that also passed `delta=True`.
307
- """
308
- info = CacheInfo(hits=self._hits, total=self._total)
309
-
310
- if delta:
311
- info_delta = info - self._last_info
312
- self._last_info = info
313
- info = info_delta
314
-
315
- return info
316
-
317
- def touch(self, key: _K) -> None:
318
- try:
319
- self._LRUCache__order.move_to_end(key) # type: ignore
320
- except KeyError:
321
- self._LRUCache__order[key] = None # type: ignore
322
-
323
- @overload
324
- def get(self, key: _K, /) -> _V | None: ...
325
-
326
- @overload
327
- def get(self, key: _K, /, default: Union[_V, _T]) -> Union[_V, _T]: ...
328
-
329
- def get(
330
- self, key: _K, /, default: Union[_V, _T] | None = None
331
- ) -> Union[_V, _T] | None:
332
- value: Union[_V, _T] | None
333
- if key in self:
334
- value = self.__getitem__(key, update_info=False) # type: ignore[call-arg]
335
-
336
- self._hits += 1
337
- else:
338
- value = default
339
-
340
- self._total += 1
341
- return value
342
-
343
- @overload
344
- def pop(self, key: _K) -> _V: ...
345
-
346
- @overload
347
- def pop(self, key: _K, default: Union[_V, _T]) -> Union[_V, _T]: ...
348
-
349
- def pop(
350
- self, key: _K, default: Union[_V, _T] | None = None
351
- ) -> Union[_V, _T] | None:
352
- value: Union[_V, _T] | None
353
- if key not in self:
354
- return default
355
-
356
- value = self.__getitem__(key, update_info=False) # type: ignore[call-arg]
357
- self.__delitem__(key)
358
- return value
359
-
360
- def put(self, key: _K, value: _V) -> None:
361
- self.__setitem__(key, value)
362
-
363
- def pin(self, key: _K) -> None:
364
- """
365
- Pins a key in the cache preventing it from being
366
- evicted in the LRU order.
367
- """
368
- if key not in self:
369
- raise ValueError(f"Cannot pin key: {key} not in cache.")
370
- self.pinned_items.add(key)
371
-
372
- def _unpin(self, key: _K) -> None:
373
- """
374
- Unpins a key in the cache allowing it to be
375
- evicted in the LRU order.
376
- """
377
- self.pinned_items.remove(key)
378
-
379
- def _on_remove(self, key: _K, value: _V | None) -> None:
380
- pass
381
-
382
- def remove_oldest(self, *, remove_pinned: bool = False) -> None:
383
- if len(self) == 0:
384
- return
385
-
386
- self.popitem(remove_pinned=remove_pinned)
387
-
388
- def _remove_old_if_needed(self) -> None:
389
- while self.currsize > self.capacity:
390
- self.remove_oldest()
391
-
392
- def popitem(self, remove_pinned: bool = False):
393
- """Remove and return the `(key, value)` pair least recently used."""
394
- if not remove_pinned:
395
- # pop the oldest item in the cache that is not pinned
396
- lru_key = next(
397
- (key for key in self.order if key not in self.pinned_items),
398
- ALL_PINNED_SENTINEL,
399
- )
400
- if lru_key is ALL_PINNED_SENTINEL:
401
- raise RuntimeError(
402
- "All items are pinned, cannot remove oldest from the cache."
403
- )
404
- else:
405
- lru_key = next(iter(self.order))
406
- value = self.pop(cast(_K, lru_key))
407
- return (lru_key, value)
408
-
409
- def clear(self) -> None:
410
- while len(self) > 0:
411
- self.remove_oldest(remove_pinned=True)
412
-
413
- self._hits = 0
414
- self._total = 0
415
- self._last_info = CacheInfo(hits=0, total=0)
416
-
417
-
418
-class PyObjectCache:
419
- """Used to cache python objects to avoid object allocations
420
- across scheduler iterations.
421
- """
422
-
423
- def __init__(self, obj_builder):
424
- self._obj_builder = obj_builder
425
- self._index = 0
426
-
427
- self._obj_cache = []
428
- for _ in range(128):
429
- self._obj_cache.append(self._obj_builder())
430
-
431
- def _grow_cache(self):
432
- # Double the size of the cache
433
- num_objs = len(self._obj_cache)
434
- for _ in range(num_objs):
435
- self._obj_cache.append(self._obj_builder())
436
-
437
- def get_object(self):
438
- """Returns a pre-allocated cached object. If there is not enough
439
- objects, then the cache size will double.
440
- """
441
- if self._index >= len(self._obj_cache):
442
- self._grow_cache()
443
- assert self._index < len(self._obj_cache)
444
-
445
- obj = self._obj_cache[self._index]
446
- self._index += 1
447
-
448
- return obj
449
-
450
- def reset(self):
451
- """Makes all cached-objects available for the next scheduler iteration."""
452
- self._index = 0
453
-
454
-
205
@cache
206
def get_max_shared_memory_bytes(gpu: int = 0) -> int:
207
"""Returns the maximum shared memory per thread block in bytes."""
vllm/utils/cache.py
new
+220
@@ -0,0 +1,220 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+from __future__ import annotations
4
+
5
+from collections import UserDict
6
+from collections.abc import Hashable, Iterator, KeysView, Mapping
7
+from types import MappingProxyType
8
+from typing import Callable, Generic, NamedTuple, TypeVar, Union, cast, overload
9
+
10
+import cachetools
11
+
12
+_K = TypeVar("_K", bound=Hashable)
13
+_V = TypeVar("_V")
14
+_T = TypeVar("_T")
15
+
16
+
17
+class _Sentinel: ...
18
+
19
+
20
+ALL_PINNED_SENTINEL = _Sentinel()
21
+
22
+
23
+class _MappingOrderCacheView(UserDict[_K, _V]):
24
+ def __init__(self, data: Mapping[_K, _V], ordered_keys: Mapping[_K, None]):
25
+ super().__init__(data)
26
+ self.ordered_keys = ordered_keys
27
+
28
+ def __iter__(self) -> Iterator[_K]:
29
+ return iter(self.ordered_keys)
30
+
31
+ def keys(self) -> KeysView[_K]:
32
+ return KeysView(self.ordered_keys)
33
+
34
+
35
+class CacheInfo(NamedTuple):
36
+ hits: int
37
+ total: int
38
+
39
+ @property
40
+ def hit_ratio(self) -> float:
41
+ if self.total == 0:
42
+ return 0
43
+
44
+ return self.hits / self.total
45
+
46
+ def __sub__(self, other: CacheInfo):
47
+ return CacheInfo(
48
+ hits=self.hits - other.hits,
49
+ total=self.total - other.total,
50
+ )
51
+
52
+
53
+class LRUCache(cachetools.LRUCache[_K, _V], Generic[_K, _V]):
54
+ def __init__(self, capacity: float, getsizeof: Callable[[_V], float] | None = None):
55
+ super().__init__(capacity, getsizeof)
56
+
57
+ self.pinned_items = set[_K]()
58
+
59
+ self._hits = 0
60
+ self._total = 0
61
+ self._last_info = CacheInfo(hits=0, total=0)
62
+
63
+ def __getitem__(self, key: _K, *, update_info: bool = True) -> _V:
64
+ value = super().__getitem__(key)
65
+
66
+ if update_info:
67
+ self._hits += 1
68
+ self._total += 1
69
+
70
+ return value
71
+
72
+ def __delitem__(self, key: _K) -> None:
73
+ run_on_remove = key in self
74
+ value = self.__getitem__(key, update_info=False) # type: ignore[call-arg]
75
+ super().__delitem__(key)
76
+ if key in self.pinned_items:
77
+ # Todo: add warning to inform that del pinned item
78
+ self._unpin(key)
79
+ if run_on_remove:
80
+ self._on_remove(key, value)
81
+
82
+ @property
83
+ def cache(self) -> Mapping[_K, _V]:
84
+ """Return the internal cache dictionary in order (read-only)."""
85
+ return _MappingOrderCacheView(
86
+ self._Cache__data, # type: ignore
87
+ self.order,
88
+ )
89
+
90
+ @property
91
+ def order(self) -> Mapping[_K, None]:
92
+ """Return the internal order dictionary (read-only)."""
93
+ return MappingProxyType(self._LRUCache__order) # type: ignore
94
+
95
+ @property
96
+ def capacity(self) -> float:
97
+ return self.maxsize
98
+
99
+ @property
100
+ def usage(self) -> float:
101
+ if self.maxsize == 0:
102
+ return 0
103
+
104
+ return self.currsize / self.maxsize
105
+
106
+ def stat(self, *, delta: bool = False) -> CacheInfo:
107
+ """
108
+ Gets the cumulative number of hits and queries against this cache.
109
+
110
+ If `delta=True`, instead gets these statistics
111
+ since the last call that also passed `delta=True`.
112
+ """
113
+ info = CacheInfo(hits=self._hits, total=self._total)
114
+
115
+ if delta:
116
+ info_delta = info - self._last_info
117
+ self._last_info = info
118
+ info = info_delta
119
+
120
+ return info
121
+
122
+ def touch(self, key: _K) -> None:
123
+ try:
124
+ self._LRUCache__order.move_to_end(key) # type: ignore
125
+ except KeyError:
126
+ self._LRUCache__order[key] = None # type: ignore
127
+
128
+ @overload
129
+ def get(self, key: _K, /) -> _V | None: ...
130
+
131
+ @overload
132
+ def get(self, key: _K, /, default: Union[_V, _T]) -> Union[_V, _T]: ...
133
+
134
+ def get(
135
+ self, key: _K, /, default: Union[_V, _T] | None = None
136
+ ) -> Union[_V, _T] | None:
137
+ value: Union[_V, _T] | None
138
+ if key in self:
139
+ value = self.__getitem__(key, update_info=False) # type: ignore[call-arg]
140
+
141
+ self._hits += 1
142
+ else:
143
+ value = default
144
+
145
+ self._total += 1
146
+ return value
147
+
148
+ @overload
149
+ def pop(self, key: _K) -> _V: ...
150
+
151
+ @overload
152
+ def pop(self, key: _K, default: Union[_V, _T]) -> Union[_V, _T]: ...
153
+
154
+ def pop(
155
+ self, key: _K, default: Union[_V, _T] | None = None
156
+ ) -> Union[_V, _T] | None:
157
+ value: Union[_V, _T] | None
158
+ if key not in self:
159
+ return default
160
+
161
+ value = self.__getitem__(key, update_info=False) # type: ignore[call-arg]
162
+ self.__delitem__(key)
163
+ return value
164
+
165
+ def put(self, key: _K, value: _V) -> None:
166
+ self.__setitem__(key, value)
167
+
168
+ def pin(self, key: _K) -> None:
169
+ """
170
+ Pins a key in the cache preventing it from being
171
+ evicted in the LRU order.
172
+ """
173
+ if key not in self:
174
+ raise ValueError(f"Cannot pin key: {key} not in cache.")
175
+ self.pinned_items.add(key)
176
+
177
+ def _unpin(self, key: _K) -> None:
178
+ """
179
+ Unpins a key in the cache allowing it to be
180
+ evicted in the LRU order.
181
+ """
182
+ self.pinned_items.remove(key)
183
+
184
+ def _on_remove(self, key: _K, value: _V | None) -> None:
185
+ pass
186
+
187
+ def remove_oldest(self, *, remove_pinned: bool = False) -> None:
188
+ if len(self) == 0:
189
+ return
190
+
191
+ self.popitem(remove_pinned=remove_pinned)
192
+
193
+ def _remove_old_if_needed(self) -> None:
194
+ while self.currsize > self.capacity:
195
+ self.remove_oldest()
196
+
197
+ def popitem(self, remove_pinned: bool = False):
198
+ """Remove and return the `(key, value)` pair least recently used."""
199
+ if not remove_pinned:
200
+ # pop the oldest item in the cache that is not pinned
201
+ lru_key = next(
202
+ (key for key in self.order if key not in self.pinned_items),
203
+ ALL_PINNED_SENTINEL,
204
+ )
205
+ if lru_key is ALL_PINNED_SENTINEL:
206
+ raise RuntimeError(
207
+ "All items are pinned, cannot remove oldest from the cache."
208
+ )
209
+ else:
210
+ lru_key = next(iter(self.order))
211
+ value = self.pop(cast(_K, lru_key))
212
+ return (lru_key, value)
213
+
214
+ def clear(self) -> None:
215
+ while len(self) > 0:
216
+ self.remove_oldest(remove_pinned=True)
217
+
218
+ self._hits = 0
219
+ self._total = 0
220
+ self._last_info = CacheInfo(hits=0, total=0)