A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Core] Optimize evictor-v2 performance (#7193)
xiaobochen123 committed
Aug 7, 2024 at 03:34 UTC
660470e5a36b8e52083615ad7c85e9b4fd4c72ce
1 file changed
+4
-2
vllm/core/evictor_v2.py
+4
-2
@@ -91,8 +91,9 @@ class LRUEvictor(Evictor):
91
# at the start of OrderedDict. Loop through all these blocks to
92
# find the one with maximum number of hashed tokens.
93
for _id, block in self.free_table.items():
94
- if evicted_block.last_accessed > block.last_accessed or (
95
- evicted_block.last_accessed == block.last_accessed and
94
+ if evicted_block.last_accessed < block.last_accessed:
95
+ break
96
+ if (evicted_block.last_accessed == block.last_accessed and
97
evicted_block.num_hashed_tokens < block.num_hashed_tokens):
98
evicted_block = block
99
evicted_block_id = _id
@@ -109,6 +110,7 @@ class LRUEvictor(Evictor):
110
111
def update(self, block_id: int, last_accessed: float):
112
self.free_table[block_id].last_accessed = last_accessed
113
+ self.free_table.move_to_end(block_id)
114
115
def remove(self, block_id: int):
116
if block_id not in self.free_table: