A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[PREFIX CACHING FOLLOW UP] A bunch of fixes to block allocator performance when automatic prefix caching is disabled (#3357)
Co-authored-by: Zhuohan Li <zhuohan123@gmail.com>
ElizaWszola committed
Mar 20, 2024 at 08:11 UTC
9474e89ba4ecae253b585eb6b3e1d85f4e108f01
4 files changed
+165
-121
tests/core/test_block_manager.py
+8
-6
@@ -4,7 +4,7 @@ from typing import List
4
5
from vllm import SamplingParams
6
from vllm.block import PhysicalTokenBlock
7
-from vllm.core.block_manager import (BlockAllocator, BlockSpaceManager,
7
+from vllm.core.block_manager import (UncachedBlockAllocator, BlockSpaceManager,
8
AllocStatus)
9
from vllm.utils import Device
10
from vllm.sequence import Sequence, SequenceGroup, SequenceStatus, Logprob
@@ -15,7 +15,8 @@ from .utils import create_dummy_prompt
15
def test_block_allocator_allocate():
16
block_size = 4
17
num_cpu_blocks = 4
18
- cpu_allocator = BlockAllocator(Device.CPU, block_size, num_cpu_blocks)
18
+ cpu_allocator = UncachedBlockAllocator(Device.CPU, block_size,
19
+ num_cpu_blocks)
20
21
# Allocate all available cpu blocks.
22
num_free = num_cpu_blocks
@@ -24,7 +25,7 @@ def test_block_allocator_allocate():
25
block = cpu_allocator.allocate()
26
num_free -= 1
27
27
- assert block.block_hash not in cpu_allocator.evictor
28
+ assert block not in cpu_allocator.free_blocks
29
assert cpu_allocator.get_num_free_blocks() == num_free
30
31
with pytest.raises(ValueError):
@@ -34,14 +35,15 @@ def test_block_allocator_allocate():
35
def test_block_allocator_free():
36
block_size = 4
37
num_cpu_blocks = 4
37
- cpu_allocator = BlockAllocator(Device.CPU, block_size, num_cpu_blocks)
38
+ cpu_allocator = UncachedBlockAllocator(Device.CPU, block_size,
39
+ num_cpu_blocks)
40
41
# Allocate all available cpu blocks.
42
blocks: List[PhysicalTokenBlock] = []
43
for _ in range(num_cpu_blocks):
44
block = cpu_allocator.allocate()
45
blocks.append(block)
44
- assert block.block_hash not in cpu_allocator.evictor
46
+ assert block not in cpu_allocator.free_blocks
47
48
# Free all allocated cpu blocks.
49
num_free = 0
@@ -49,7 +51,7 @@ def test_block_allocator_free():
51
for block in blocks:
52
cpu_allocator.free(block)
53
num_free += 1
52
- assert block.block_hash in cpu_allocator.evictor
54
+ assert block in cpu_allocator.free_blocks
55
assert cpu_allocator.get_num_free_blocks() == num_free
56
57
with pytest.raises(ValueError):
tests/prefix_caching/test_prefix_caching.py
+3
-9
@@ -4,7 +4,7 @@ Run `pytest tests/prefix_caching/test_prefix_caching.py`.
4
"""
5
import pytest
6
7
-from vllm.core.block_manager import BlockAllocator
7
+from vllm.core.block_manager import CachedBlockAllocator
8
from vllm.utils import Device
9
10
@@ -15,10 +15,7 @@ def test_block_allocator(
15
num_blocks: int,
16
):
17
block_hash = 1
18
- block_allocator = BlockAllocator(Device.CPU,
19
- block_size,
20
- num_blocks,
21
- enable_caching=True)
18
+ block_allocator = CachedBlockAllocator(Device.CPU, block_size, num_blocks)
19
20
# Allocate two PysicalTokenBlocks with the same hash and check
21
# that they are the same PhysicalTokenBlock
@@ -45,10 +42,7 @@ def test_block_allocator(
42
@pytest.mark.parametrize("num_blocks", [16])
43
def test_eviction(num_blocks: int, ):
44
block_size = 16
48
- block_allocator = BlockAllocator(Device.CPU,
49
- block_size,
50
- num_blocks,
51
- enable_caching=True)
45
+ block_allocator = CachedBlockAllocator(Device.CPU, block_size, num_blocks)
46
blocks = []
47
48
for i in range(num_blocks):
vllm/core/block_manager.py
+146
-43
@@ -3,6 +3,7 @@ import enum
3
from itertools import count, takewhile
4
from os.path import commonprefix
5
from typing import Dict, List, Optional, Set, Tuple
6
+from abc import ABC, abstractmethod
7
8
from vllm.block import BlockTable, PhysicalTokenBlock
9
from vllm.sequence import Sequence, SequenceGroup, SequenceStatus
@@ -10,7 +11,7 @@ from vllm.utils import Device
11
from vllm.core.evictor import Evictor, EvictionPolicy, make_evictor
12
13
13
-class BlockAllocator:
14
+class BlockAllocatorBase(ABC):
15
"""Manages free physical token blocks for a device.
16
17
The allocator maintains a list of free blocks and allocates a block when
@@ -18,23 +19,57 @@ class BlockAllocator:
19
the reference count becomes zero, the block is added back to the free list.
20
"""
21
22
+ @abstractmethod
23
def __init__(self,
24
device: Device,
25
block_size: int,
26
num_blocks: int,
25
- eviction_policy: EvictionPolicy = EvictionPolicy.LRU,
26
- enable_caching: bool = False) -> None:
27
+ eviction_policy: EvictionPolicy = EvictionPolicy.LRU):
28
+ pass
29
+
30
+ @abstractmethod
31
+ def allocate(self,
32
+ block_hash: Optional[int] = None,
33
+ num_hashed_tokens: int = 0) -> PhysicalTokenBlock:
34
+ pass
35
+
36
+ @abstractmethod
37
+ def free(self, block: PhysicalTokenBlock) -> None:
38
+ pass
39
+
40
+ @abstractmethod
41
+ def get_num_free_blocks(self) -> int:
42
+ pass
43
+
44
+ @abstractmethod
45
+ def contains_block(self, block_hash: int) -> bool:
46
+ pass
47
+
48
+ @abstractmethod
49
+ def update_hash(self, block_hash: int, block: PhysicalTokenBlock):
50
+ pass
51
+
52
+
53
+class CachedBlockAllocator(BlockAllocatorBase):
54
+ """Manages free physical token blocks for a device.
55
+
56
+ The allocator maintains a list of free blocks and allocates a block when
57
+ requested. When a block is freed, its reference count is decremented. If
58
+ the reference count becomes zero, the block is added back to the free list.
59
+ """
60
+
61
+ def __init__(self,
62
+ device: Device,
63
+ block_size: int,
64
+ num_blocks: int,
65
+ eviction_policy: EvictionPolicy = EvictionPolicy.LRU) -> None:
66
self.device = device
67
self.block_size = block_size
68
self.num_blocks = num_blocks
30
- self.enable_caching = enable_caching
69
70
self.current_num_blocks = 0
71
self.cached_blocks: Dict[int, PhysicalTokenBlock] = {}
72
35
- # Switch over to FIFO eviction when caching is disabled
36
- if not self.enable_caching:
37
- eviction_policy = EvictionPolicy.FIFO
73
self.evictor: Evictor = make_evictor(eviction_policy)
74
75
self.default_hash_ctr = count()
@@ -57,13 +92,6 @@ class BlockAllocator:
92
def allocate(self,
93
block_hash: Optional[int] = None,
94
num_hashed_tokens: int = 0) -> PhysicalTokenBlock:
60
- # If caching is disabled, just allocate a new block and return it
61
- if not self.enable_caching:
62
- block = self.allocate_block(next(self.default_hash_ctr),
63
- num_hashed_tokens)
64
- block.ref_count += 1
65
- return block
66
-
95
if block_hash is None:
96
block_hash = next(self.default_hash_ctr)
97
if block_hash in self.evictor:
@@ -90,9 +118,8 @@ class BlockAllocator:
118
assert block.block_hash not in self.evictor
119
self.evictor.add(block)
120
93
- # If caching is enabled, remove the block from the cached_blocks
94
- if self.enable_caching:
95
- del self.cached_blocks[block.block_hash]
121
+ # Remove the block from the cached_blocks
122
+ del self.cached_blocks[block.block_hash]
123
124
def get_num_free_blocks(self) -> int:
125
return (self.num_blocks - self.current_num_blocks +
@@ -102,14 +129,68 @@ class BlockAllocator:
129
return block_hash in self.cached_blocks or block_hash in self.evictor
130
131
def update_hash(self, block_hash: int, block: PhysicalTokenBlock):
105
- # If caching is enabled, update the hash of block and the
106
- # cached_blocks dictionary.
107
- if self.enable_caching:
108
- assert not self.contains_block(block_hash)
109
- old_hash = block.block_hash
110
- block.block_hash = block_hash
111
- del self.cached_blocks[old_hash]
112
- self.cached_blocks[block_hash] = block
132
+ # Update the hash of block and the cached_blocks dictionary.
133
+ assert not self.contains_block(block_hash)
134
+ old_hash = block.block_hash
135
+ block.block_hash = block_hash
136
+ del self.cached_blocks[old_hash]
137
+ self.cached_blocks[block_hash] = block
138
+
139
+
140
+class UncachedBlockAllocator(BlockAllocatorBase):
141
+ """Manages free physical token blocks for a device.
142
+
143
+ The allocator maintains a list of free blocks and allocates a block when
144
+ requested. When a block is freed, its reference count is decremented. If
145
+ the reference count becomes zero, the block is added back to the free list.
146
+ """
147
+
148
+ def __init__(
149
+ self,
150
+ device: Device,
151
+ block_size: int,
152
+ num_blocks: int,
153
+ ) -> None:
154
+ self.device = device
155
+ self.block_size = block_size
156
+ self.num_blocks = num_blocks
157
+
158
+ # Initialize the free blocks.
159
+ self.free_blocks: BlockTable = []
160
+ for i in range(num_blocks):
161
+ block = PhysicalTokenBlock(device=device,
162
+ block_number=i,
163
+ block_size=block_size,
164
+ block_hash=-1,
165
+ num_hashed_tokens=0)
166
+ self.free_blocks.append(block)
167
+
168
+ def allocate(self,
169
+ block_hash: Optional[int] = None,
170
+ num_hashed_tokens: int = 0) -> PhysicalTokenBlock:
171
+ if not self.free_blocks:
172
+ raise ValueError("Out of memory! No free blocks are available.")
173
+ block = self.free_blocks.pop()
174
+ block.ref_count = 1
175
+ return block
176
+
177
+ def free(self, block: PhysicalTokenBlock) -> None:
178
+ if block.ref_count == 0:
179
+ raise ValueError(f"Double free! {block} is already freed.")
180
+ block.ref_count -= 1
181
+ if block.ref_count == 0:
182
+ self.free_blocks.append(block)
183
+
184
+ def get_num_free_blocks(self) -> int:
185
+ return len(self.free_blocks)
186
+
187
+ def contains_block(self, block_hash: int) -> bool:
188
+ raise NotImplementedError(
189
+ "Invalid codepath for uncached block allocator.")
190
+
191
+ def update_hash(self, block_hash: int, block: PhysicalTokenBlock):
192
+ raise NotImplementedError(
193
+ "Invalid codepath for uncached block allocator.")
194
195
196
class AllocStatus(enum.Enum):
@@ -142,6 +223,10 @@ class BlockSpaceManager:
223
self.num_total_gpu_blocks = num_gpu_blocks
224
self.num_total_cpu_blocks = num_cpu_blocks
225
226
+ if enable_caching and sliding_window is not None:
227
+ raise NotImplementedError(
228
+ "Sliding window is not allowed with prefix caching enabled!")
229
+
230
self.block_sliding_window = None
231
if sliding_window is not None:
232
assert sliding_window % block_size == 0, (sliding_window,
@@ -154,14 +239,17 @@ class BlockSpaceManager:
239
self.enable_caching = enable_caching
240
241
self.watermark_blocks = int(watermark * num_gpu_blocks)
157
- self.gpu_allocator = BlockAllocator(Device.GPU,
158
- block_size,
159
- num_gpu_blocks,
160
- enable_caching=enable_caching)
161
- self.cpu_allocator = BlockAllocator(Device.CPU,
162
- block_size,
163
- num_cpu_blocks,
164
- enable_caching=enable_caching)
242
+
243
+ if self.enable_caching:
244
+ self.gpu_allocator = CachedBlockAllocator(Device.GPU, block_size,
245
+ num_gpu_blocks)
246
+ self.cpu_allocator = CachedBlockAllocator(Device.CPU, block_size,
247
+ num_cpu_blocks)
248
+ else:
249
+ self.gpu_allocator = UncachedBlockAllocator(
250
+ Device.GPU, block_size, num_gpu_blocks)
251
+ self.cpu_allocator = UncachedBlockAllocator(
252
+ Device.CPU, block_size, num_cpu_blocks)
253
# Mapping: seq_id -> BlockTable.
254
self.block_tables: Dict[int, BlockTable] = {}
255
@@ -198,10 +286,16 @@ class BlockSpaceManager:
286
if (self.block_sliding_window is not None
287
and logical_idx >= self.block_sliding_window):
288
block = block_table[logical_idx % self.block_sliding_window]
201
- else:
289
+ # Set the reference counts of the token blocks.
290
+ block.ref_count = seq_group.num_seqs()
291
+ elif self.enable_caching:
292
block = self.gpu_allocator.allocate(
293
seq.hash_of_block(logical_idx),
294
seq.num_hashed_tokens_of_block(logical_idx))
295
+ else:
296
+ block = self.gpu_allocator.allocate()
297
+ # Set the reference counts of the token blocks.
298
+ block.ref_count = seq_group.num_seqs()
299
block_table.append(block)
300
301
# Assign the block table for each sequence.
@@ -220,8 +314,10 @@ class BlockSpaceManager:
314
seq: Sequence,
315
last_block: PhysicalTokenBlock,
316
) -> PhysicalTokenBlock:
223
- # Compute a new hash for the block so that it can be shared by
224
- # other Sequences
317
+ assert self.enable_caching
318
+
319
+ # Compute a new hash for the block so that it can be shared by other
320
+ # Sequences
321
new_hash = seq.hash_of_block(len(seq.logical_token_blocks) - 1)
322
323
# if new_hash is already in the cached table, then free last_block
@@ -254,6 +350,8 @@ class BlockSpaceManager:
350
self,
351
seq: Sequence,
352
) -> PhysicalTokenBlock:
353
+ if not self.enable_caching:
354
+ return self.gpu_allocator.allocate()
355
block_hash: Optional[int] = None
356
if (self._is_last_block_full(seq)):
357
block_hash = seq.hash_of_block(len(seq.logical_token_blocks) - 1)
@@ -293,10 +391,12 @@ class BlockSpaceManager:
391
assert last_block.device == Device.GPU
392
if last_block.ref_count == 1:
393
# Not shared with other sequences. Appendable.
296
- # If the last block is now complete, promote it to a full block so
297
- # that it can be shared
298
- new_block = self._maybe_promote_last_block(seq, last_block)
299
- block_table[-1] = new_block
394
+ if self.enable_caching:
395
+ # If the last block is now complete, we may reuse an old block
396
+ # to save memory.
397
+ maybe_new_block = self._maybe_promote_last_block(
398
+ seq, last_block)
399
+ block_table[-1] = maybe_new_block
400
return None
401
else:
402
# The last block is shared with other sequences.
@@ -440,9 +540,12 @@ class BlockSpaceManager:
540
seq: Sequence,
541
access_time: float,
542
) -> None:
443
- block_table = self.block_tables[seq.seq_id]
444
- for block in block_table:
445
- block.last_accessed = access_time
543
+ if self.enable_caching:
544
+ # Update the last accessed time of all the blocks accessed
545
+ # in this step.
546
+ block_table = self.block_tables[seq.seq_id]
547
+ for block in block_table:
548
+ block.last_accessed = access_time
549
550
def compute_full_blocks_in_seq(self, seq: Sequence):
551
if seq.seq_id not in self.block_tables:
vllm/core/evictor.py
+8
-63
@@ -1,5 +1,5 @@
1
import enum
2
-from typing import Dict, List, Optional
2
+from typing import Dict
3
from abc import ABC, abstractmethod, abstractproperty
4
5
from vllm.block import PhysicalTokenBlock
@@ -10,7 +10,6 @@ class EvictionPolicy(enum.Enum):
10
Evictor subclass.
11
"""
12
LRU = enum.auto()
13
- FIFO = enum.auto()
13
14
15
class Evictor(ABC):
@@ -66,37 +65,18 @@ class LRUEvictor(Evictor):
65
66
# TODO: The performance of this evict function can be optimized further.
67
def evict(self) -> PhysicalTokenBlock:
69
- free_blocks: List[PhysicalTokenBlock] = list(self.free_table.values())
70
- if len(free_blocks) == 0:
68
+ if len(self.free_table) == 0:
69
raise ValueError("No usable cache memory left")
70
+ free_blocks = self.free_table.values()
71
73
- # Find lowest timestamp
74
- lowest_timestamp = free_blocks[0].last_accessed
75
- for block in free_blocks:
76
- if block.last_accessed < lowest_timestamp:
77
- lowest_timestamp = block.last_accessed
72
+ # Get evicted block
73
+ evicted_block: PhysicalTokenBlock = next(iter(free_blocks))
74
79
- # Find all blocks with the lowest timestamp
80
- least_recent: List[PhysicalTokenBlock] = []
75
for block in free_blocks:
82
- if block.last_accessed == lowest_timestamp:
83
- least_recent.append(block)
84
-
85
- # Find highest prefix count per block
86
- highest_num_hashed_tokens = 0
87
- for block in least_recent:
88
- if block.num_hashed_tokens > highest_num_hashed_tokens:
89
- highest_num_hashed_tokens = block.num_hashed_tokens
90
-
91
- evicted_block: Optional[PhysicalTokenBlock] = None
92
-
93
- # Find the first block with the lowest timestamp
94
- for block in least_recent:
95
- if block.num_hashed_tokens == highest_num_hashed_tokens:
76
+ if (block.last_accessed < evicted_block.last_accessed
77
+ or block.last_accessed == evicted_block.last_accessed and
78
+ block.num_hashed_tokens > evicted_block.num_hashed_tokens):
79
evicted_block = block
97
- break
98
-
99
- assert evicted_block is not None
80
81
del self.free_table[evicted_block.block_hash]
82
@@ -119,43 +99,8 @@ class LRUEvictor(Evictor):
99
return len(self.free_table)
100
101
122
-class RandomEvictor(Evictor):
123
- """Evicts in a first-in-first-out order"""
124
-
125
- def __init__(self):
126
- self.free_table: Dict[int, PhysicalTokenBlock] = {}
127
-
128
- def __contains__(self, block_hash: int) -> bool:
129
- return block_hash in self.free_table
130
-
131
- def evict(self) -> PhysicalTokenBlock:
132
- if len(self.free_table) == 0:
133
- raise ValueError("No usable cache memory left")
134
- evicted_block = next(iter(self.free_table.values()))
135
- evicted_block.computed = False
136
- del self.free_table[evicted_block.block_hash]
137
- return evicted_block
138
-
139
- def add(self, block: PhysicalTokenBlock):
140
- self.free_table[block.block_hash] = block
141
-
142
- def remove(self, block_hash: int) -> PhysicalTokenBlock:
143
- if block_hash not in self.free_table:
144
- raise ValueError(
145
- "Attempting to remove block that's not in the evictor")
146
- block: PhysicalTokenBlock = self.free_table[block_hash]
147
- del self.free_table[block_hash]
148
- return block
149
-
150
- @property
151
- def num_blocks(self) -> int:
152
- return len(self.free_table)
153
-
154
-
102
def make_evictor(eviction_policy: EvictionPolicy) -> Evictor:
103
if eviction_policy == EvictionPolicy.LRU:
104
return LRUEvictor()
158
- elif eviction_policy == EvictionPolicy.FIFO:
159
- return RandomEvictor()
105
else:
106
raise ValueError(f"Unknown cache eviction policy: {eviction_policy}")