@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6ebd02bdef
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[PREFIX CACHING FOLLOW UP] OrderedDict-based evictor (#3431)

Co-authored-by: rsnm2 <rshaw@neuralmagic.com> Co-authored-by: Luka <luka@paperspace>

ElizaWszola committed Mar 21, 2024 at 07:20 UTC 6ebd02bdef1eb08f9a7a11253a26cd49b5fb6d2d
1 file changed +12 -13
vllm/core/evictor.py
+12 -13
@@ -1,5 +1,5 @@
1 import enum
2 -from typing import Dict
2 +from typing import OrderedDict
3 from abc import ABC, abstractmethod, abstractproperty
4
5 from vllm.block import PhysicalTokenBlock
@@ -58,27 +58,26 @@ class LRUEvictor(Evictor):
58 """
59
60 def __init__(self):
61 - self.free_table: Dict[int, PhysicalTokenBlock] = {}
61 + self.free_table: OrderedDict[int, PhysicalTokenBlock] = OrderedDict()
62
63 def __contains__(self, block_hash: int) -> bool:
64 return block_hash in self.free_table
65
66 - # TODO: The performance of this evict function can be optimized further.
66 def evict(self) -> PhysicalTokenBlock:
67 if len(self.free_table) == 0:
68 raise ValueError("No usable cache memory left")
70 - free_blocks = self.free_table.values()
69
72 - # Get evicted block
73 - evicted_block: PhysicalTokenBlock = next(iter(free_blocks))
74 -
75 - for block in free_blocks:
76 - if (block.last_accessed < evicted_block.last_accessed
77 - or block.last_accessed == evicted_block.last_accessed and
78 - block.num_hashed_tokens > evicted_block.num_hashed_tokens):
70 + evicted_block = next(iter(self.free_table.values()))
71 + # The blocks with the lowest timestamps should be placed consecutively
72 + # at the start of OrderedDict. Loop through all these blocks to
73 + # find the one with maximum number of hashed tokens.
74 + for _, block in self.free_table.items():
75 + if evicted_block.last_accessed < block.last_accessed:
76 + break
77 + if evicted_block.num_hashed_tokens < block.num_hashed_tokens:
78 evicted_block = block
79
81 - del self.free_table[evicted_block.block_hash]
80 + self.free_table.pop(evicted_block.block_hash)
81
82 evicted_block.computed = False
83 return evicted_block
@@ -91,7 +90,7 @@ class LRUEvictor(Evictor):
90 raise ValueError(
91 "Attempting to remove block that's not in the evictor")
92 block: PhysicalTokenBlock = self.free_table[block_hash]
94 - del self.free_table[block_hash]
93 + self.free_table.pop(block_hash)
94 return block
95
96 @property