A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[PREFIX CACHING FOLLOW UP] OrderedDict-based evictor (#3431)
Co-authored-by: rsnm2 <rshaw@neuralmagic.com> Co-authored-by: Luka <luka@paperspace>
ElizaWszola committed
Mar 21, 2024 at 07:20 UTC
6ebd02bdef1eb08f9a7a11253a26cd49b5fb6d2d
1 file changed
+12
-13
vllm/core/evictor.py
+12
-13
@@ -1,5 +1,5 @@
1
import enum
2
-from typing import Dict
2
+from typing import OrderedDict
3
from abc import ABC, abstractmethod, abstractproperty
4
5
from vllm.block import PhysicalTokenBlock
@@ -58,27 +58,26 @@ class LRUEvictor(Evictor):
58
"""
59
60
def __init__(self):
61
- self.free_table: Dict[int, PhysicalTokenBlock] = {}
61
+ self.free_table: OrderedDict[int, PhysicalTokenBlock] = OrderedDict()
62
63
def __contains__(self, block_hash: int) -> bool:
64
return block_hash in self.free_table
65
66
- # TODO: The performance of this evict function can be optimized further.
66
def evict(self) -> PhysicalTokenBlock:
67
if len(self.free_table) == 0:
68
raise ValueError("No usable cache memory left")
70
- free_blocks = self.free_table.values()
69
72
- # Get evicted block
73
- evicted_block: PhysicalTokenBlock = next(iter(free_blocks))
74
-
75
- for block in free_blocks:
76
- if (block.last_accessed < evicted_block.last_accessed
77
- or block.last_accessed == evicted_block.last_accessed and
78
- block.num_hashed_tokens > evicted_block.num_hashed_tokens):
70
+ evicted_block = next(iter(self.free_table.values()))
71
+ # The blocks with the lowest timestamps should be placed consecutively
72
+ # at the start of OrderedDict. Loop through all these blocks to
73
+ # find the one with maximum number of hashed tokens.
74
+ for _, block in self.free_table.items():
75
+ if evicted_block.last_accessed < block.last_accessed:
76
+ break
77
+ if evicted_block.num_hashed_tokens < block.num_hashed_tokens:
78
evicted_block = block
79
81
- del self.free_table[evicted_block.block_hash]
80
+ self.free_table.pop(evicted_block.block_hash)
81
82
evicted_block.computed = False
83
return evicted_block
@@ -91,7 +90,7 @@ class LRUEvictor(Evictor):
90
raise ValueError(
91
"Attempting to remove block that's not in the evictor")
92
block: PhysicalTokenBlock = self.free_table[block_hash]
94
- del self.free_table[block_hash]
93
+ self.free_table.pop(block_hash)
94
return block
95
96
@property