@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ca30c3c84b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Core] Remove evictor_v1 (#9572)

Kuntai Du committed Oct 21, 2024 at 23:55 UTC ca30c3c84b1c1a89b7083524854d81440e80c5bd
3 files changed +1 -107
vllm/core/block/prefix_caching_block.py
+1 -1
@@ -7,7 +7,7 @@ from vllm.core.block.common import (CacheMetricData, CopyOnWriteTracker,
7 from vllm.core.block.interfaces import Block, BlockAllocator, BlockId, Device
8 from vllm.core.block.naive_block import (BlockPool, NaiveBlock,
9 NaiveBlockAllocator)
10 -from vllm.core.evictor_v2 import EvictionPolicy, Evictor, make_evictor
10 +from vllm.core.evictor import EvictionPolicy, Evictor, make_evictor
11
12 PrefixHash = int
13
vllm/core/evictor.py renamed
vllm/core/evictor_v1.py deleted
-106
@@ -1,106 +0,0 @@
1 -import enum
2 -from abc import ABC, abstractmethod
3 -from typing import OrderedDict
4 -
5 -from vllm.block import PhysicalTokenBlock
6 -
7 -
8 -class EvictionPolicy(enum.Enum):
9 - """Enum for eviction policy used by make_evictor to instantiate the correct
10 - Evictor subclass.
11 - """
12 - LRU = enum.auto()
13 -
14 -
15 -class Evictor(ABC):
16 - """The Evictor subclasses should be used by the BlockAllocator class to
17 - handle eviction of freed PhysicalTokenBlocks.
18 - """
19 -
20 - @abstractmethod
21 - def __init__(self):
22 - pass
23 -
24 - @abstractmethod
25 - def __contains__(self, block_hash: int) -> bool:
26 - pass
27 -
28 - @abstractmethod
29 - def evict(self) -> PhysicalTokenBlock:
30 - """Runs the eviction algorithm and returns the evicted block"""
31 - pass
32 -
33 - @abstractmethod
34 - def add(self, block: PhysicalTokenBlock):
35 - """Adds block to the evictor, making it a candidate for eviction"""
36 - pass
37 -
38 - @abstractmethod
39 - def remove(self, block_hash: int) -> PhysicalTokenBlock:
40 - """Simply removes the block with the hash value block_hash from the
41 - evictor. Caller is responsible for making sure that block_hash is
42 - contained in the evictor before calling remove. Should be used to
43 - "bring back" blocks that have been freed but not evicted yet.
44 - """
45 - pass
46 -
47 - @property
48 - @abstractmethod
49 - def num_blocks(self) -> int:
50 - pass
51 -
52 -
53 -class LRUEvictor(Evictor):
54 - """Evicts in a least-recently-used order using the last_accessed timestamp
55 - that's recorded in the PhysicalTokenBlock. If there are multiple blocks with
56 - the same last_accessed time, then the one with the largest num_hashed_tokens
57 - will be evicted. If two blocks each have the lowest last_accessed time and
58 - highest num_hashed_tokens value, then one will be chose arbitrarily
59 - """
60 -
61 - def __init__(self):
62 - self.free_table: OrderedDict[int, PhysicalTokenBlock] = OrderedDict()
63 -
64 - def __contains__(self, block_hash: int) -> bool:
65 - return block_hash in self.free_table
66 -
67 - def evict(self) -> PhysicalTokenBlock:
68 - if len(self.free_table) == 0:
69 - raise ValueError("No usable cache memory left")
70 -
71 - evicted_block = next(iter(self.free_table.values()))
72 - # The blocks with the lowest timestamps should be placed consecutively
73 - # at the start of OrderedDict. Loop through all these blocks to
74 - # find the one with maximum number of hashed tokens.
75 - for _, block in self.free_table.items():
76 - if evicted_block.last_accessed < block.last_accessed:
77 - break
78 - if evicted_block.num_hashed_tokens < block.num_hashed_tokens:
79 - evicted_block = block
80 -
81 - self.free_table.pop(evicted_block.block_hash)
82 -
83 - evicted_block.computed = False
84 - return evicted_block
85 -
86 - def add(self, block: PhysicalTokenBlock):
87 - self.free_table[block.block_hash] = block
88 -
89 - def remove(self, block_hash: int) -> PhysicalTokenBlock:
90 - if block_hash not in self.free_table:
91 - raise ValueError(
92 - "Attempting to remove block that's not in the evictor")
93 - block: PhysicalTokenBlock = self.free_table[block_hash]
94 - self.free_table.pop(block_hash)
95 - return block
96 -
97 - @property
98 - def num_blocks(self) -> int:
99 - return len(self.free_table)
100 -
101 -
102 -def make_evictor(eviction_policy: EvictionPolicy) -> Evictor:
103 - if eviction_policy == EvictionPolicy.LRU:
104 - return LRUEvictor()
105 - else:
106 - raise ValueError(f"Unknown cache eviction policy: {eviction_policy}")