A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 14:45 UTC
[Misc] Modify LRUCache touch (#16689)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Apr 16, 2025 at 12:51 UTC
0d7d05f4b6d79e1e5af15634b742dfd32f915f3f
1 file changed
+4
-1
vllm/utils.py
+4
-1
@@ -316,7 +316,10 @@ class LRUCache(cachetools.LRUCache[_K, _V], Generic[_K, _V]):
316
return info
317
318
def touch(self, key: _K) -> None:
319
- self._LRUCache__update(key) # type: ignore
319
+ try:
320
+ self._LRUCache__order.move_to_end(key) # type: ignore
321
+ except KeyError:
322
+ self._LRUCache__order[key] = None # type: ignore
323
324
@overload
325
def get(self, key: _K, /) -> Optional[_V]: