A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Performance][Core] Optimize the performance of evictor v1 and v2 by applying a priority queue and lazy deletion (#7209)
Sungjae Lee committed
Dec 15, 2024 at 04:38 UTC
886936837ca89e5645bc1f71cc0e1492b65b1590
1 file changed
+43
-20
vllm/core/evictor.py
+43
-20
@@ -1,6 +1,7 @@
1
import enum
2
+import heapq
3
from abc import ABC, abstractmethod
3
-from typing import OrderedDict, Tuple
4
+from typing import Dict, List, Tuple
5
6
7
class EvictionPolicy(enum.Enum):
@@ -75,8 +76,14 @@ class LRUEvictor(Evictor):
76
highest num_hashed_tokens value, then one will be chose arbitrarily
77
"""
78
79
+ # CLEANUP_THRESHOLD determines the maximum allowable size of the priority
80
+ # queue relative to the free table size. When this threshold is exceeded,
81
+ # a cleanup operation is triggered to reduce memory usage.
82
+ CLEANUP_THRESHOLD = 50
83
+
84
def __init__(self):
79
- self.free_table: OrderedDict[int, BlockMetaData] = OrderedDict()
85
+ self.free_table: Dict[int, BlockMetaData] = {}
86
+ self.priority_queue = []
87
88
def __contains__(self, block_id: int) -> bool:
89
return block_id in self.free_table
@@ -85,34 +92,50 @@ class LRUEvictor(Evictor):
92
if len(self.free_table) == 0:
93
raise ValueError("No usable cache memory left")
94
88
- evicted_block, evicted_block_id = None, None
89
- # The blocks with the lowest timestamps should be placed consecutively
90
- # at the start of OrderedDict. Loop through all these blocks to
91
- # find the one with maximum number of hashed tokens.
92
- for _id, block in self.free_table.items():
93
- if evicted_block is None:
94
- evicted_block, evicted_block_id = block, _id
95
- continue
96
- if evicted_block.last_accessed < block.last_accessed:
97
- break
98
- if evicted_block.num_hashed_tokens < block.num_hashed_tokens:
99
- evicted_block, evicted_block_id = block, _id
100
-
101
- assert evicted_block is not None
102
- assert evicted_block_id is not None
103
- self.free_table.pop(evicted_block_id)
104
-
105
- return evicted_block_id, evicted_block.content_hash
95
+ while self.priority_queue:
96
+ # We do not remove outdated entries from the priority queue at the
97
+ # time of updating the last_accessed timestamp. Instead, outdated
98
+ # entries are filtered out here during eviction. Outdated entries
99
+ # would either not in the free table, or have older last accessed
100
+ # time.
101
+ last_accessed, _, block_id, content_hash = heapq.heappop(
102
+ self.priority_queue)
103
+ if (block_id in self.free_table and
104
+ self.free_table[block_id].last_accessed == last_accessed):
105
+ self.free_table.pop(block_id)
106
+ return block_id, content_hash
107
+
108
+ raise ValueError("No usable cache memory left")
109
110
def add(self, block_id: int, content_hash: int, num_hashed_tokens: int,
111
last_accessed: float):
112
self.free_table[block_id] = BlockMetaData(content_hash,
113
num_hashed_tokens,
114
last_accessed)
115
+ heapq.heappush(
116
+ self.priority_queue,
117
+ (last_accessed, -num_hashed_tokens, block_id, content_hash))
118
+ self._cleanup_if_necessary()
119
120
def update(self, block_id: int, last_accessed: float):
121
self.free_table[block_id].last_accessed = last_accessed
122
123
+ def _cleanup_if_necessary(self):
124
+ if len(self.priority_queue) > LRUEvictor.CLEANUP_THRESHOLD * len(
125
+ self.free_table):
126
+ self._cleanup()
127
+
128
+ def _cleanup(self):
129
+ new_priority_queue: List[Tuple[float, int, int, int]] = []
130
+
131
+ for block_id, block in self.free_table.items():
132
+ new_priority_queue.append(
133
+ (block.last_accessed, -block.num_hashed_tokens, block_id,
134
+ block.content_hash))
135
+ heapq.heapify(new_priority_queue)
136
+
137
+ self.priority_queue = new_priority_queue
138
+
139
def remove(self, block_id: int):
140
if block_id not in self.free_table:
141
raise ValueError(