A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Core] Remove evictor_v1 (#9572)
Kuntai Du committed
Oct 21, 2024 at 23:55 UTC
ca30c3c84b1c1a89b7083524854d81440e80c5bd
3 files changed
+1
-107
vllm/core/block/prefix_caching_block.py
+1
-1
@@ -7,7 +7,7 @@ from vllm.core.block.common import (CacheMetricData, CopyOnWriteTracker,
7
from vllm.core.block.interfaces import Block, BlockAllocator, BlockId, Device
8
from vllm.core.block.naive_block import (BlockPool, NaiveBlock,
9
NaiveBlockAllocator)
10
-from vllm.core.evictor_v2 import EvictionPolicy, Evictor, make_evictor
10
+from vllm.core.evictor import EvictionPolicy, Evictor, make_evictor
11
12
PrefixHash = int
13
vllm/core/evictor.py
renamed
vllm/core/evictor_v1.py
deleted
-106
@@ -1,106 +0,0 @@
1
-import enum
2
-from abc import ABC, abstractmethod
3
-from typing import OrderedDict
4
-
5
-from vllm.block import PhysicalTokenBlock
6
-
7
-
8
-class EvictionPolicy(enum.Enum):
9
- """Enum for eviction policy used by make_evictor to instantiate the correct
10
- Evictor subclass.
11
- """
12
- LRU = enum.auto()
13
-
14
-
15
-class Evictor(ABC):
16
- """The Evictor subclasses should be used by the BlockAllocator class to
17
- handle eviction of freed PhysicalTokenBlocks.
18
- """
19
-
20
- @abstractmethod
21
- def __init__(self):
22
- pass
23
-
24
- @abstractmethod
25
- def __contains__(self, block_hash: int) -> bool:
26
- pass
27
-
28
- @abstractmethod
29
- def evict(self) -> PhysicalTokenBlock:
30
- """Runs the eviction algorithm and returns the evicted block"""
31
- pass
32
-
33
- @abstractmethod
34
- def add(self, block: PhysicalTokenBlock):
35
- """Adds block to the evictor, making it a candidate for eviction"""
36
- pass
37
-
38
- @abstractmethod
39
- def remove(self, block_hash: int) -> PhysicalTokenBlock:
40
- """Simply removes the block with the hash value block_hash from the
41
- evictor. Caller is responsible for making sure that block_hash is
42
- contained in the evictor before calling remove. Should be used to
43
- "bring back" blocks that have been freed but not evicted yet.
44
- """
45
- pass
46
-
47
- @property
48
- @abstractmethod
49
- def num_blocks(self) -> int:
50
- pass
51
-
52
-
53
-class LRUEvictor(Evictor):
54
- """Evicts in a least-recently-used order using the last_accessed timestamp
55
- that's recorded in the PhysicalTokenBlock. If there are multiple blocks with
56
- the same last_accessed time, then the one with the largest num_hashed_tokens
57
- will be evicted. If two blocks each have the lowest last_accessed time and
58
- highest num_hashed_tokens value, then one will be chose arbitrarily
59
- """
60
-
61
- def __init__(self):
62
- self.free_table: OrderedDict[int, PhysicalTokenBlock] = OrderedDict()
63
-
64
- def __contains__(self, block_hash: int) -> bool:
65
- return block_hash in self.free_table
66
-
67
- def evict(self) -> PhysicalTokenBlock:
68
- if len(self.free_table) == 0:
69
- raise ValueError("No usable cache memory left")
70
-
71
- evicted_block = next(iter(self.free_table.values()))
72
- # The blocks with the lowest timestamps should be placed consecutively
73
- # at the start of OrderedDict. Loop through all these blocks to
74
- # find the one with maximum number of hashed tokens.
75
- for _, block in self.free_table.items():
76
- if evicted_block.last_accessed < block.last_accessed:
77
- break
78
- if evicted_block.num_hashed_tokens < block.num_hashed_tokens:
79
- evicted_block = block
80
-
81
- self.free_table.pop(evicted_block.block_hash)
82
-
83
- evicted_block.computed = False
84
- return evicted_block
85
-
86
- def add(self, block: PhysicalTokenBlock):
87
- self.free_table[block.block_hash] = block
88
-
89
- def remove(self, block_hash: int) -> PhysicalTokenBlock:
90
- if block_hash not in self.free_table:
91
- raise ValueError(
92
- "Attempting to remove block that's not in the evictor")
93
- block: PhysicalTokenBlock = self.free_table[block_hash]
94
- self.free_table.pop(block_hash)
95
- return block
96
-
97
- @property
98
- def num_blocks(self) -> int:
99
- return len(self.free_table)
100
-
101
-
102
-def make_evictor(eviction_policy: EvictionPolicy) -> Evictor:
103
- if eviction_policy == EvictionPolicy.LRU:
104
- return LRUEvictor()
105
- else:
106
- raise ValueError(f"Unknown cache eviction policy: {eviction_policy}")