@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 886936837c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Performance][Core] Optimize the performance of evictor v1 and v2 by applying a priority queue and lazy deletion (#7209)

Sungjae Lee committed Dec 15, 2024 at 04:38 UTC 886936837ca89e5645bc1f71cc0e1492b65b1590
1 file changed +43 -20
vllm/core/evictor.py
+43 -20
@@ -1,6 +1,7 @@
1 import enum
2 +import heapq
3 from abc import ABC, abstractmethod
3 -from typing import OrderedDict, Tuple
4 +from typing import Dict, List, Tuple
5
6
7 class EvictionPolicy(enum.Enum):
@@ -75,8 +76,14 @@ class LRUEvictor(Evictor):
76 highest num_hashed_tokens value, then one will be chose arbitrarily
77 """
78
79 + # CLEANUP_THRESHOLD determines the maximum allowable size of the priority
80 + # queue relative to the free table size. When this threshold is exceeded,
81 + # a cleanup operation is triggered to reduce memory usage.
82 + CLEANUP_THRESHOLD = 50
83 +
84 def __init__(self):
79 - self.free_table: OrderedDict[int, BlockMetaData] = OrderedDict()
85 + self.free_table: Dict[int, BlockMetaData] = {}
86 + self.priority_queue = []
87
88 def __contains__(self, block_id: int) -> bool:
89 return block_id in self.free_table
@@ -85,34 +92,50 @@ class LRUEvictor(Evictor):
92 if len(self.free_table) == 0:
93 raise ValueError("No usable cache memory left")
94
88 - evicted_block, evicted_block_id = None, None
89 - # The blocks with the lowest timestamps should be placed consecutively
90 - # at the start of OrderedDict. Loop through all these blocks to
91 - # find the one with maximum number of hashed tokens.
92 - for _id, block in self.free_table.items():
93 - if evicted_block is None:
94 - evicted_block, evicted_block_id = block, _id
95 - continue
96 - if evicted_block.last_accessed < block.last_accessed:
97 - break
98 - if evicted_block.num_hashed_tokens < block.num_hashed_tokens:
99 - evicted_block, evicted_block_id = block, _id
100 -
101 - assert evicted_block is not None
102 - assert evicted_block_id is not None
103 - self.free_table.pop(evicted_block_id)
104 -
105 - return evicted_block_id, evicted_block.content_hash
95 + while self.priority_queue:
96 + # We do not remove outdated entries from the priority queue at the
97 + # time of updating the last_accessed timestamp. Instead, outdated
98 + # entries are filtered out here during eviction. Outdated entries
99 + # would either not in the free table, or have older last accessed
100 + # time.
101 + last_accessed, _, block_id, content_hash = heapq.heappop(
102 + self.priority_queue)
103 + if (block_id in self.free_table and
104 + self.free_table[block_id].last_accessed == last_accessed):
105 + self.free_table.pop(block_id)
106 + return block_id, content_hash
107 +
108 + raise ValueError("No usable cache memory left")
109
110 def add(self, block_id: int, content_hash: int, num_hashed_tokens: int,
111 last_accessed: float):
112 self.free_table[block_id] = BlockMetaData(content_hash,
113 num_hashed_tokens,
114 last_accessed)
115 + heapq.heappush(
116 + self.priority_queue,
117 + (last_accessed, -num_hashed_tokens, block_id, content_hash))
118 + self._cleanup_if_necessary()
119
120 def update(self, block_id: int, last_accessed: float):
121 self.free_table[block_id].last_accessed = last_accessed
122
123 + def _cleanup_if_necessary(self):
124 + if len(self.priority_queue) > LRUEvictor.CLEANUP_THRESHOLD * len(
125 + self.free_table):
126 + self._cleanup()
127 +
128 + def _cleanup(self):
129 + new_priority_queue: List[Tuple[float, int, int, int]] = []
130 +
131 + for block_id, block in self.free_table.items():
132 + new_priority_queue.append(
133 + (block.last_accessed, -block.num_hashed_tokens, block_id,
134 + block.content_hash))
135 + heapq.heapify(new_priority_queue)
136 +
137 + self.priority_queue = new_priority_queue
138 +
139 def remove(self, block_id: int):
140 if block_id not in self.free_table:
141 raise ValueError(