@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 660470e5a3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Core] Optimize evictor-v2 performance (#7193)

xiaobochen123 committed Aug 7, 2024 at 03:34 UTC 660470e5a36b8e52083615ad7c85e9b4fd4c72ce
1 file changed +4 -2
vllm/core/evictor_v2.py
+4 -2
@@ -91,8 +91,9 @@ class LRUEvictor(Evictor):
91 # at the start of OrderedDict. Loop through all these blocks to
92 # find the one with maximum number of hashed tokens.
93 for _id, block in self.free_table.items():
94 - if evicted_block.last_accessed > block.last_accessed or (
95 - evicted_block.last_accessed == block.last_accessed and
94 + if evicted_block.last_accessed < block.last_accessed:
95 + break
96 + if (evicted_block.last_accessed == block.last_accessed and
97 evicted_block.num_hashed_tokens < block.num_hashed_tokens):
98 evicted_block = block
99 evicted_block_id = _id
@@ -109,6 +110,7 @@ class LRUEvictor(Evictor):
110
111 def update(self, block_id: int, last_accessed: float):
112 self.free_table[block_id].last_accessed = last_accessed
113 + self.free_table.move_to_end(block_id)
114
115 def remove(self, block_id: int):
116 if block_id not in self.free_table: