A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Core] Enable prefix caching with block manager v2 enabled (#4142)
Co-authored-by: Lei Wen <wenlei03@qiyi.com> Co-authored-by: Sage Moore <sagemoore@utexas.edu>
leiwen83 committed
May 2, 2024 at 02:20 UTC
24750f4cadd15a2b3a52f982e39eb9803749efbc
11 files changed
+584
-57
benchmarks/benchmark_prefix_caching.py
+13
-3
@@ -16,20 +16,22 @@ def test_prefix(llm=None, sampling_params=None, prompts=None):
16
17
18
def main(args):
19
- llm = LLM(model="baichuan-inc/Baichuan2-13B-Chat",
19
+ llm = LLM(model=args.model,
20
tokenizer_mode='auto',
21
trust_remote_code=True,
22
enforce_eager=True,
23
+ use_v2_block_manager=args.use_v2_block_manager,
24
+ tensor_parallel_size=args.tensor_parallel_size,
25
enable_prefix_caching=args.enable_prefix_caching)
26
27
num_prompts = 100
28
prompts = [PROMPT] * num_prompts
27
- sampling_params = SamplingParams(temperature=0, max_tokens=100)
29
+ sampling_params = SamplingParams(temperature=0, max_tokens=args.output_len)
30
31
print("------warm up------")
32
test_prefix(
33
llm=llm,
32
- prompts=prompts[:1],
34
+ prompts=prompts,
35
sampling_params=sampling_params,
36
)
37
@@ -45,8 +47,16 @@ if __name__ == "__main__":
47
parser = argparse.ArgumentParser(
48
description='Benchmark the performance with or without automatic '
49
'prefix caching.')
50
+ parser.add_argument('--model',
51
+ type=str,
52
+ default='baichuan-inc/Baichuan2-13B-Chat')
53
+ parser.add_argument('--tensor-parallel-size', '-tp', type=int, default=1)
54
+ parser.add_argument('--output-len', type=int, default=10)
55
parser.add_argument('--enable-prefix-caching',
56
action='store_true',
57
help='enable prefix caching')
58
+ parser.add_argument('--use-v2-block-manager',
59
+ action='store_true',
60
+ help='Use BlockSpaceMangerV2')
61
args = parser.parse_args()
62
main(args)
tests/core/block/e2e/test_correctness.py
+146
@@ -300,6 +300,152 @@ def test_chunked_prefill_block_manager_v2(baseline_llm_generator,
300
assert baseline_token_ids == test_token_ids
301
302
303
+@pytest.mark.parametrize(
304
+ "common_llm_kwargs",
305
+ [{
306
+ # Use a small model for a fast test.
307
+ "model": "facebook/opt-125m",
308
+
309
+ # skip cuda graph creation for fast test.
310
+ "enforce_eager": True,
311
+
312
+ # Allow only 5 sequences of ~1024 tokens in worst case.
313
+ "block_size": 16,
314
+ "num_gpu_blocks_override": 5 * (64 + 1),
315
+
316
+ # Enable prefill cache
317
+ "enable_prefix_caching": True,
318
+ }])
319
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
320
+@pytest.mark.parametrize("baseline_llm_kwargs", [{
321
+ "use_v2_block_manager": False
322
+}])
323
+@pytest.mark.parametrize("test_llm_kwargs", [{"use_v2_block_manager": True}])
324
+@pytest.mark.parametrize("batch_size", [10])
325
+@pytest.mark.parametrize("seed", [1])
326
+def test_v1_v2_greedy_equality_prefix_caching_enabled_with_preemption(
327
+ baseline_llm_generator, test_llm_generator, batch_size):
328
+ """Verify block manager v2 produces same outputs as block manager v1, even
329
+ when there is preemption.
330
+
331
+ This constructs two LLM, each with limited number of GPU blocks. The limit
332
+ is decided such that as the sequences in the batch grow, sequences must be
333
+ preempted and removed from cache.
334
+
335
+ If the output token ids are equivalent, then we have confidence that the KV
336
+ cache is not corrupted in the v2 block manager.
337
+
338
+ NOTE: We want a significant number of generated tokens so that any incorrect
339
+ KV mapping has time to build up error.
340
+ """
341
+ output_len = 1024
342
+ temperature = 0.0
343
+
344
+ # We want to ensure equality even with preemption.
345
+ # We force the total block size to be 1 + cdiv(output_len, block_size)
346
+ # so that only one sequence can fit at a time (once the sequences grow).
347
+
348
+ prompts = [
349
+ "Hello, my name is",
350
+ "The president of the United States is",
351
+ "The capital of France is",
352
+ "The future of AI is",
353
+ ]
354
+
355
+ prompts = [prompt for prompt, _ in zip(cycle(prompts), range(batch_size))]
356
+
357
+ sampling_params = SamplingParams(
358
+ max_tokens=output_len,
359
+ ignore_eos=True,
360
+ temperature=temperature,
361
+ )
362
+
363
+ print('Getting token ids from block manager v1')
364
+ baseline_token_ids = get_token_ids_from_llm_generator(
365
+ baseline_llm_generator, prompts, sampling_params)
366
+
367
+ print('Getting token ids from block manager v2')
368
+ test_token_ids = get_token_ids_from_llm_generator(test_llm_generator,
369
+ prompts, sampling_params)
370
+
371
+ for expected_token_ids, actual_token_ids in zip(baseline_token_ids,
372
+ test_token_ids):
373
+ assert expected_token_ids == actual_token_ids
374
+
375
+ assert baseline_token_ids == test_token_ids
376
+
377
+
378
+@pytest.mark.parametrize(
379
+ "common_llm_kwargs",
380
+ [{
381
+ # Use a small model for a fast test.
382
+ "model": "facebook/opt-125m",
383
+
384
+ # skip cuda graph creation for fast test.
385
+ "enforce_eager": True,
386
+
387
+ # Allow only 5 sequences of ~1024 tokens in worst case.
388
+ "block_size": 16,
389
+ "num_gpu_blocks_override": 5 * (64 + 1),
390
+
391
+ # Test APC in v2 block
392
+ "use_v2_block_manager": True,
393
+ }])
394
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
395
+@pytest.mark.parametrize("baseline_llm_kwargs", [{
396
+ "enable_prefix_caching": False
397
+}])
398
+@pytest.mark.parametrize("test_llm_kwargs", [{"enable_prefix_caching": True}])
399
+@pytest.mark.parametrize("batch_size", [10])
400
+@pytest.mark.parametrize("seed", [1])
401
+def test_auto_prefix_caching_with_preemption(baseline_llm_generator,
402
+ test_llm_generator, batch_size):
403
+ """Verify block manager v2 with auto prefix caching enabled produces same
404
+ outputs as auto prefix caching disabled, even when there is preemption.
405
+
406
+ This constructs two LLM, each with limited number of GPU blocks. The limit
407
+ is decided such that as the sequences in the batch grow, sequences must be
408
+ preempted and removed from cache.
409
+
410
+ If the output token ids are equivalent, then we have confidence that auto
411
+ prefix caching itself at least don't cause result error.
412
+ """
413
+ output_len = 1024
414
+ temperature = 0.0
415
+
416
+ # We want to ensure equality even with preemption.
417
+ # We force the total block size to be 1 + cdiv(output_len, block_size)
418
+ # so that only one sequence can fit at a time (once the sequences grow).
419
+ prompts = [
420
+ "Hello, my name is",
421
+ "The president of the United States is",
422
+ "The capital of France is",
423
+ "The future of AI is",
424
+ ]
425
+
426
+ prompts = [prompt for prompt, _ in zip(cycle(prompts), range(batch_size))]
427
+
428
+ sampling_params = SamplingParams(
429
+ max_tokens=output_len,
430
+ ignore_eos=True,
431
+ temperature=temperature,
432
+ )
433
+
434
+ print('Getting token ids with APC disabled')
435
+ baseline_token_ids = get_token_ids_from_llm_generator(
436
+ baseline_llm_generator, prompts, sampling_params)
437
+
438
+ print('Getting token ids with APC enabled')
439
+ test_token_ids = get_token_ids_from_llm_generator(test_llm_generator,
440
+ prompts, sampling_params)
441
+
442
+ for expected_token_ids, actual_token_ids in zip(baseline_token_ids,
443
+ test_token_ids):
444
+ assert expected_token_ids == actual_token_ids
445
+
446
+ assert baseline_token_ids == test_token_ids
447
+
448
+
449
def get_token_ids_from_llm_generator(llm_generator, prompts, sampling_params):
450
for llm in llm_generator:
451
outputs = llm.generate(prompts, sampling_params, use_tqdm=True)
tests/core/block/test_prefix_caching_block.py
+125
@@ -358,6 +358,131 @@ class TestPrefixCachingBlockAllocator:
358
i)
359
allocator.free(block)
360
361
+ @staticmethod
362
+ @pytest.mark.parametrize("num_blocks", [1024])
363
+ @pytest.mark.parametrize("block_size", [16])
364
+ @pytest.mark.parametrize("seed", list(range(20)))
365
+ def test_get_common_computed_block_ids(num_blocks: int, block_size: int,
366
+ seed: int):
367
+ """Verify get_common_computed_block_ids could get correct result
368
+ by create two immutable chain sharing prefix at specified pos,
369
+ and compare whether we also could get right result
370
+ from get_common_computed_block_ids.
371
+ """
372
+ random.seed(seed)
373
+ allocator = PrefixCachingBlockAllocator(num_blocks=num_blocks * 2,
374
+ block_size=block_size)
375
+ num_blocks_to_consume = random.randint(1, num_blocks - 1)
376
+
377
+ # Create token ids that will exhaust all blocks.
378
+ token_ids = list(range(num_blocks_to_consume * block_size))
379
+ blocks = list(range(num_blocks_to_consume))
380
+
381
+ first_chain = TestPrefixCachingBlockAllocator.create_immutable_chain(
382
+ block_size=block_size,
383
+ token_ids=token_ids,
384
+ allocator=allocator,
385
+ )
386
+
387
+ # mark all blocks in first chain as computed
388
+ allocator.mark_blocks_as_computed(blocks)
389
+
390
+ # After zero_point, second_chain's token_ids would be set -1, which
391
+ # make it different from here comparing with first_chain
392
+ zero_point = random.randint(1, len(token_ids) - 1)
393
+ zero_point_blocks = zero_point // block_size
394
+ token_ids[zero_point:] = [-1] * (len(token_ids) - zero_point)
395
+
396
+ second_chain = TestPrefixCachingBlockAllocator.create_immutable_chain(
397
+ block_size=block_size,
398
+ token_ids=token_ids,
399
+ allocator=allocator,
400
+ )
401
+
402
+ first_computed_ids = [
403
+ first_chain[i].block_id for i in range(num_blocks_to_consume)
404
+ ]
405
+ second_computed_ids = [
406
+ second_chain[i].block_id for i in range(num_blocks_to_consume)
407
+ ]
408
+ res = allocator.get_common_computed_block_ids(
409
+ [first_computed_ids, second_computed_ids])
410
+
411
+ assert (len(res) == zero_point_blocks)
412
+
413
+ # Test case where two last accessed times are equal
414
+ @staticmethod
415
+ @pytest.mark.parametrize("num_blocks", [1024])
416
+ @pytest.mark.parametrize("block_size", [16])
417
+ @pytest.mark.parametrize("seed", list(range(20)))
418
+ def test_eviction_order(num_blocks: int, block_size: int, seed: int):
419
+ """This test case simulate the two chain created and free in order,
420
+ and together they would exhaust the initial freed blocks.
421
+
422
+ So the next block created after those two chain shall use the block
423
+ from the first chain as that block has long access time.
424
+ While first chain has two blocks, it shall pick up the last one, as
425
+ it has larger token number.
426
+ """
427
+
428
+ random.seed(seed)
429
+ allocator = PrefixCachingBlockAllocator(num_blocks=num_blocks,
430
+ block_size=block_size)
431
+ num_blocks_to_consume = num_blocks + 1
432
+
433
+ token_ids = list(range(num_blocks_to_consume * block_size))
434
+
435
+ num_blocks_in_first_chain = 2
436
+ num_tokens_in_first_chain = block_size * num_blocks_in_first_chain
437
+ # First chain takes the first block
438
+ first_chain = TestPrefixCachingBlockAllocator.create_immutable_chain(
439
+ block_size=block_size,
440
+ token_ids=token_ids[:num_tokens_in_first_chain],
441
+ allocator=allocator,
442
+ )
443
+ # There should only be one block allocated at this point
444
+ assert allocator.get_num_free_blocks() == (num_blocks -
445
+ num_blocks_in_first_chain)
446
+
447
+ # Set the last accessed time of the first block to 1
448
+ blocks_ids = [block.block_id for block in first_chain]
449
+ allocator.mark_blocks_as_accessed(blocks_ids, 1)
450
+
451
+ # Second chain takes the rest of the blocks
452
+ second_chain = TestPrefixCachingBlockAllocator.create_immutable_chain(
453
+ block_size=block_size,
454
+ token_ids=token_ids[num_tokens_in_first_chain:-block_size],
455
+ allocator=allocator,
456
+ )
457
+
458
+ # There shouldn't be any blocks left at this point
459
+ assert allocator.get_num_free_blocks() == (0)
460
+
461
+ assert len(first_chain) == num_blocks_in_first_chain
462
+ last_block_id = first_chain[-1].block_id
463
+ # Free each block in the first chain.
464
+ for i, block in enumerate(first_chain):
465
+ allocator.free(block)
466
+
467
+ # Set the last accessed time on all of the blocks in the second chain
468
+ # to 2
469
+ blocks_ids = [block.block_id for block in second_chain]
470
+ allocator.mark_blocks_as_accessed(blocks_ids, 2)
471
+
472
+ # Free each block in the second chain.
473
+ for i, block in enumerate(second_chain):
474
+ allocator.free(block)
475
+
476
+ # Allocate a new block and check that it's the least recently used block
477
+ # from the first chain.
478
+ new_block = TestPrefixCachingBlockAllocator.create_immutable_chain(
479
+ block_size=block_size,
480
+ token_ids=token_ids[-block_size:],
481
+ allocator=allocator,
482
+ )
483
+
484
+ assert new_block[0].block_id == last_block_id
485
+
486
@staticmethod
487
def create_immutable_chain(
488
block_size: int,
vllm/core/block/cpu_gpu_block_allocator.py
+10
-2
@@ -190,10 +190,18 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
190
device = Device.GPU
191
return self._allocators[device].clear_copy_on_writes()
192
193
- def mark_blocks_as_computed(self) -> None:
193
+ def mark_blocks_as_accessed(self, block_ids: List[int],
194
+ now: float) -> None:
195
+ """Mark blocks as accessed, only use for prefix caching."""
196
# Prefix caching only supported on GPU.
197
device = Device.GPU
196
- return self._allocators[device].mark_blocks_as_computed()
198
+ return self._allocators[device].mark_blocks_as_accessed(block_ids, now)
199
+
200
+ def mark_blocks_as_computed(self, block_ids: List[int]) -> None:
201
+ """Mark blocks as accessed, only use for prefix caching."""
202
+ # Prefix caching only supported on GPU.
203
+ device = Device.GPU
204
+ return self._allocators[device].mark_blocks_as_computed(block_ids)
205
206
def get_common_computed_block_ids(
207
self, seq_block_ids: List[List[int]]) -> List[int]:
vllm/core/block/interfaces.py
+4
@@ -81,6 +81,10 @@ class BlockAllocator(ABC):
81
def clear_copy_on_writes(self) -> Dict[int, List[int]]:
82
pass
83
84
+ @abstractmethod
85
+ def mark_blocks_as_accessed(self) -> None:
86
+ pass
87
+
88
@abstractmethod
89
def mark_blocks_as_computed(self) -> None:
90
pass
vllm/core/block/naive_block.py
+10
-1
@@ -174,7 +174,16 @@ class NaiveBlockAllocator(BlockAllocator):
174
"""
175
return self._cow_tracker.clear_cows()
176
177
- def mark_blocks_as_computed(self) -> None:
177
+ def mark_blocks_as_accessed(self, block_ids: List[int],
178
+ now: float) -> None:
179
+ """Mark blocks as accessed, used in prefix caching.
180
+
181
+ Since the naive allocator does not implement prefix caching, we do
182
+ nothing.
183
+ """
184
+ pass
185
+
186
+ def mark_blocks_as_computed(self, block_ids: List[int]) -> None:
187
"""Mark blocks as computed, used in prefix caching.
188
189
Since the naive allocator does not implement prefix caching, we do
vllm/core/block/prefix_caching_block.py
+134
-38
@@ -7,10 +7,16 @@ from vllm.core.block.common import (CopyOnWriteTracker,
7
get_all_blocks_recursively)
8
from vllm.core.block.interfaces import Block, BlockAllocator
9
from vllm.core.block.naive_block import NaiveBlock, NaiveBlockAllocator
10
+from vllm.core.evictor_v2 import EvictionPolicy, Evictor, make_evictor
11
12
PrefixHash = int
13
BlockId = int
14
15
+# By default, we init our block access time as _DEFAULT_LAST_ACCESSED_TIME
16
+# so that if we find one block is still hold _DEFAULT_LAST_ACCESSED_TIME,
17
+# then we know this block hasn't been accessed yet.
18
+_DEFAULT_LAST_ACCESSED_TIME = -1
19
+
20
21
class PrefixCachingBlockAllocator(BlockAllocator):
22
"""A block allocator that implements prefix caching.
@@ -27,22 +33,19 @@ class PrefixCachingBlockAllocator(BlockAllocator):
33
from 0 to num_blocks - 1.
34
"""
35
30
- # TODO last access time / evictor integration
31
-
36
def __init__(
37
self,
38
num_blocks: int,
39
block_size: int,
40
block_ids: Optional[Iterable[int]] = None,
41
+ eviction_policy: Optional[EvictionPolicy] = EvictionPolicy.LRU,
42
):
43
# A mapping of prefix hash to block index. All blocks which have a
44
# prefix hash will be in this dict, even if they have refcount 0.
45
self._cached_blocks: Dict[PrefixHash, BlockId] = {}
46
42
- # A mapping of prefix hash to block index. All blocks which have a
43
- # prefix hash AND refcount 0 will be in this dict. Thus, it is a subset
44
- # of self._cached_blocks.
45
- self._unused_cached_blocks: Dict[PrefixHash, BlockId] = {}
47
+ # A mapping of blockId to Block to track those cached blocks
48
+ self._blocks: Dict[BlockId, Block] = {}
49
50
# An allocator for blocks that do not have prefix hashes.
51
self._hashless_allocator = NaiveBlockAllocator(
@@ -54,6 +57,10 @@ class PrefixCachingBlockAllocator(BlockAllocator):
57
58
self._block_size = block_size
59
60
+ # Evitor used to maintain how we want to handle those computed blocks
61
+ # if we find memory pressure is high.
62
+ self.evictor: Evictor = make_evictor(eviction_policy)
63
+
64
# We share the refcounter between allocators. This allows us to promote
65
# blocks originally allocated in the hashless allocator to immutable
66
# blocks.
@@ -72,6 +79,7 @@ class PrefixCachingBlockAllocator(BlockAllocator):
79
block_size: int,
80
allocator: BlockAllocator,
81
block_id: Optional[int] = None,
82
+ computed: Optional[bool] = False,
83
) -> Block:
84
# Bind block to self.
85
allocator = self
@@ -82,6 +90,7 @@ class PrefixCachingBlockAllocator(BlockAllocator):
90
block_size=block_size,
91
block_id=block_id,
92
prefix_caching_allocator=allocator,
93
+ computed=computed,
94
)
95
96
def allocate_immutable(self, prev_block: Optional[Block],
@@ -109,14 +118,12 @@ class PrefixCachingBlockAllocator(BlockAllocator):
118
cached_block_id = self._cached_blocks.get(block.content_hash, None)
119
if cached_block_id is not None:
120
block.block_id = cached_block_id
112
- self._incr_refcount_cached_block(block.content_hash,
113
- block.block_id)
121
+ self._incr_refcount_cached_block(block, block.block_id)
122
return block
123
124
block = self.allocate_mutable(prev_block)
125
block.append_token_ids(token_ids)
126
assert block.content_hash is not None
119
- # TODO computed bit
127
128
return block
129
@@ -133,41 +140,67 @@ class PrefixCachingBlockAllocator(BlockAllocator):
140
assert_prefix_caching_block_or_none(prev_block)
141
142
try:
136
- return self._hashless_allocator.allocate_mutable(
143
+ block = self._hashless_allocator.allocate_mutable(
144
prev_block=prev_block)
145
+
146
+ assert block.block_id not in self._blocks
147
+ self._blocks[block.block_id] = block
148
+ return block
149
except BlockAllocator.NoFreeBlocksError:
150
# We must check the unused cached blocks before raising OOM.
151
pass
152
142
- if self._unused_cached_blocks:
143
- # TODO policy for selecting block to remove
144
- content_hash_to_evict = next(iter(self._unused_cached_blocks))
153
+ # If the evictor has blocks available for eviction, evict a block
154
+ # and return it.
155
+ if self.evictor.num_blocks > 0:
156
+ block_id, content_hash_to_evict = self.evictor.evict()
157
+
158
+ # Here we may have scenario that several blocks have
159
+ # the same content hash, but due to the latter coming block
160
+ # is coming from mutable to immutable path, their physical
161
+ # block is added into evictor.
162
+ # However in this case, we shall not pop the _cached_blocks,
163
+ # as the same content is still used by others, which means
164
+ # we need to check ref before decide to pop the list.
165
146
- # Clear content hash mapping; the block will be overwritten.
147
- del self._cached_blocks[content_hash_to_evict]
166
+ _block_id = self._cached_blocks[content_hash_to_evict]
167
+ refcount = self._refcounter.get(_block_id)
168
+ if refcount == 1:
169
+ self._cached_blocks.pop(content_hash_to_evict)
170
+ assert _block_id == block_id
171
149
- block_id = self._unused_cached_blocks.pop(content_hash_to_evict)
150
- refcount = self._refcounter.incr(block_id)
151
- assert refcount == 1
172
+ self._refcounter.incr(block_id)
173
+
174
+ # the block comes from evictor already contain computed result
175
block = self._create_block(
176
prev_block=prev_block,
177
token_ids=[],
178
block_size=self._block_size,
179
allocator=self,
180
block_id=block_id,
181
+ computed=True,
182
)
183
assert block.content_hash is None
184
+
185
+ assert block.block_id not in self._blocks
186
+ self._blocks[block.block_id] = block
187
return block
188
189
# No block available in hashless allocator, nor in unused cache blocks.
190
raise BlockAllocator.NoFreeBlocksError()
191
165
- def _incr_refcount_cached_block(self, content_hash: int,
192
+ def _incr_refcount_cached_block(self, block: Block,
193
block_id: BlockId) -> None:
194
+ # since block is already computed, mark it
195
+ block.computed = True
196
+
197
refcount = self._refcounter.incr(block_id)
198
if refcount == 1:
169
- assert content_hash in self._unused_cached_blocks
170
- del self._unused_cached_blocks[content_hash]
199
+ # if block get referred, then it shall not be in evictor
200
+ # and put it into _blocks for tracking
201
+ if block_id in self.evictor:
202
+ self.evictor.remove(block_id)
203
+ self._blocks[block_id] = block
204
205
def free(self, block: Block) -> None:
206
"""Decrement the refcount of the block. If the decremented refcount is
@@ -180,6 +213,7 @@ class PrefixCachingBlockAllocator(BlockAllocator):
213
is not None), "freeing unallocated block is undefined"
214
215
self._free_block_id_for_block(block.block_id, block)
216
+
217
block.block_id = None
218
219
def _free_block_id_for_block(self, block_id: BlockId,
@@ -187,15 +221,21 @@ class PrefixCachingBlockAllocator(BlockAllocator):
221
assert isinstance(block, PrefixCachingBlock)
222
223
if block.content_hash is None:
224
+ refcount = self._refcounter.get(block_id)
225
+ # We have fork case where block would get more than one ref,
226
+ # so we cannot free it from tracking if ref cnt large than 1
227
+ if refcount <= 1:
228
+ del self._blocks[block.block_id]
229
return self._hashless_allocator.free(block)
230
231
refcount = self._refcounter.decr(block_id)
232
194
- # If no longer used, add the block to the unused cached blocks.
233
+ # If no longer used, add the block to the evictor.
234
if refcount == 0:
196
- assert block.content_hash not in self._unused_cached_blocks
235
assert block.content_hash in self._cached_blocks
198
- self._unused_cached_blocks[block.content_hash] = block_id
236
+ del self._blocks[block.block_id]
237
+ self.evictor.add(block.block_id, block.content_hash,
238
+ block.num_tokens_total, block.last_accessed)
239
240
def fork(self, last_block: Block) -> List[Block]:
241
"""Creates a new sequence of blocks that shares the same underlying
@@ -230,9 +270,9 @@ class PrefixCachingBlockAllocator(BlockAllocator):
270
271
def get_num_free_blocks(self) -> int:
272
# The number of free blocks is the number of hashless free blocks
233
- # plus the number of hashful blocks that are unused.
234
- return self._hashless_allocator.get_num_free_blocks() + len(
235
- self._unused_cached_blocks)
273
+ # plus the number of blocks evictor could free from its list.
274
+ return self._hashless_allocator.get_num_free_blocks(
275
+ ) + self.evictor.num_blocks
276
277
@property
278
def all_block_ids(self) -> frozenset[int]:
@@ -266,7 +306,7 @@ class PrefixCachingBlockAllocator(BlockAllocator):
306
else:
307
self._free_block_id_for_block(block.block_id, block)
308
self._incr_refcount_cached_block(
269
- block.content_hash, self._cached_blocks[block.content_hash])
309
+ block, self._cached_blocks[block.content_hash])
310
311
return self._cached_blocks[block.content_hash]
312
@@ -293,29 +333,60 @@ class PrefixCachingBlockAllocator(BlockAllocator):
333
"""
334
return self._cow_tracker.clear_cows()
335
296
- def mark_blocks_as_computed(self) -> None:
336
+ def mark_blocks_as_accessed(self, block_ids: List[int],
337
+ now: float) -> None:
338
+ """Mark blocks as accessed, used in prefix caching.
339
+
340
+ If the block is added into evictor, we need to update corresponding
341
+ info in evictor's metadata.
342
+ """
343
+
344
+ for block_id in block_ids:
345
+ if block_id in self._blocks:
346
+ self._blocks[block_id].last_accessed = now
347
+ elif block_id in self.evictor:
348
+ self.evictor.update(block_id, now)
349
+ else:
350
+ raise ValueError(
351
+ "Mark block as accessed which is not belonged to GPU")
352
+
353
+ def mark_blocks_as_computed(self, block_ids: List[int]) -> None:
354
"""Mark blocks as computed, used in prefix caching."""
298
- # TODO Track computed blocks.
299
- pass
355
+
356
+ for block_id in block_ids:
357
+ if block_id in self._blocks:
358
+ # only those full block is valid for prefix caching
359
+ if self._blocks[block_id].is_full:
360
+ self._blocks[block_id].computed = True
361
+ elif block_id not in self.evictor:
362
+ raise ValueError(f"Mark {block_id=} as computed which "
363
+ "is not belonged to GPU")
364
+
365
+ def block_is_computed(self, block_id: int) -> bool:
366
+ if block_id in self._blocks:
367
+ return self._blocks[block_id].computed
368
+ else:
369
+ return block_id in self.evictor
370
371
def get_common_computed_block_ids(
372
self, seq_block_ids: List[List[int]]) -> List[int]:
373
"""Return the block ids that are common for a given sequence group.
374
305
- Used in prefill (can skip prefill of some blocks).
375
+ Only those blocks that are immutable and already be marked
376
+ compyted would be taken consideration.
377
"""
378
308
- # TODO: Track computed blocks.
309
- computed = lambda block_id: False
310
-
379
# NOTE We exclude the last block to avoid the case where the entire
380
# prompt is cached. This would cause erroneous behavior in model
381
# runner.
382
+
383
ids_list = [
315
- takewhile(lambda block_id: computed(block_id), seq[:-1])
316
- for seq in seq_block_ids
384
+ list(
385
+ takewhile(lambda block_id: self.block_is_computed(block_id),
386
+ seq[:-1])) for seq in seq_block_ids
387
]
318
- return commonprefix([ids for ids in ids_list if ids != []])
388
+ res = commonprefix([ids for ids in ids_list if ids != []])
389
+ return res
390
391
392
class PrefixCachingBlock(Block):
@@ -345,12 +416,16 @@ class PrefixCachingBlock(Block):
416
block_size: int,
417
prefix_caching_allocator: PrefixCachingBlockAllocator,
418
block_id: Optional[int] = None,
419
+ computed: Optional[bool] = False,
420
):
421
assert_prefix_caching_block_or_none(prev_block)
422
423
self._prev_block = prev_block
424
self._cached_content_hash: Optional[int] = None
425
+ self._cached_num_tokens_total: Optional[int] = None
426
self._prefix_caching_allocator = prefix_caching_allocator
427
+ self.last_accessed = _DEFAULT_LAST_ACCESSED_TIME
428
+ self.computed = computed
429
430
self._block = NaiveBlock(
431
prev_block=prev_block,
@@ -398,6 +473,27 @@ class PrefixCachingBlock(Block):
473
def num_empty_slots(self) -> int:
474
return self._block.num_empty_slots
475
476
+ @property
477
+ def num_tokens_total(self) -> int:
478
+ """return the total tokens so far.
479
+
480
+ Here we iterate the block chain till to the first block, while
481
+ cache the result in local to prevent repeated computations.
482
+ """
483
+ if self._cached_num_tokens_total is not None:
484
+ return self._cached_num_tokens_total
485
+
486
+ _block = self
487
+ self._cached_num_tokens_total = 0
488
+
489
+ # TODO: current implement here take O(N^2), we expect future
490
+ # we have O(1) here
491
+ while _block is not None:
492
+ self._cached_num_tokens_total += len(_block.token_ids)
493
+ _block = _block.prev_block
494
+
495
+ return self._cached_num_tokens_total
496
+
497
@property
498
def block_size(self) -> int:
499
return self._block.block_size
vllm/core/block_manager_v1.py
+1
-1
@@ -8,7 +8,7 @@ from typing import Sequence as GenericSequence
8
from typing import Set
9
10
from vllm.block import BlockTable, PhysicalTokenBlock
11
-from vllm.core.evictor import EvictionPolicy, Evictor, make_evictor
11
+from vllm.core.evictor_v1 import EvictionPolicy, Evictor, make_evictor
12
from vllm.core.interfaces import AllocStatus, BlockSpaceManager
13
from vllm.logger import init_logger
14
from vllm.sequence import Sequence, SequenceGroup, SequenceStatus
vllm/core/block_manager_v2.py
+19
-12
@@ -72,14 +72,12 @@ class BlockSpaceManagerV2(BlockSpaceManager):
72
self.watermark = watermark
73
assert watermark >= 0.0
74
75
- assert not enable_caching, "Prefix caching not yet supported"
75
self.enable_caching = enable_caching
76
77
self.watermark_blocks = int(watermark * num_gpu_blocks)
78
79
self.block_allocator = CpuGpuBlockAllocator.create(
81
- # Currently, only naive blocks are supported (no prefix caching).
82
- allocator_type="naive",
80
+ allocator_type="prefix_caching" if enable_caching else "naive",
81
num_gpu_blocks=num_gpu_blocks,
82
num_cpu_blocks=num_cpu_blocks,
83
block_size=block_size,
@@ -194,17 +192,26 @@ class BlockSpaceManagerV2(BlockSpaceManager):
192
assert all(b is not None for b in block_ids)
193
return block_ids
194
197
- def access_all_blocks_in_seq(self, seq, now):
198
- # TODO add prefix caching support.
199
- # Tracked here https://github.com/vllm-project/vllm/issues/3667
200
- pass
195
+ def access_all_blocks_in_seq(self, seq: Sequence, now: float):
196
+ # Update the last accessed time of all the blocks accessed
197
+ # in this step.
198
+ # And the accessed time is only useful for prefix caching now,
199
+ # as it support internal evictor policy for which cached
200
+ # block could be refilled, to keep cached content could be reused
201
+ # at max extend.
202
+ if self.enable_caching:
203
+ block_table = self.block_tables[seq.seq_id]
204
+ block_ids = []
205
+ for block_id in block_table.physical_block_ids:
206
+ block_ids.append(block_id)
207
+ self.block_allocator.mark_blocks_as_accessed(block_ids, now)
208
209
def mark_blocks_as_computed(self, seq_group: SequenceGroup):
203
- # We ignore the sequence group as its not necessary. After the batch is
204
- # formed by the scheduler, we do not need to mark blocks from individual
205
- # sequence groups as computed -- all blocks in the batch can be marked
206
- # as computed.
207
- self.block_allocator.mark_blocks_as_computed()
210
+ # The only need for mark block as computed is for prefix caching,
211
+ # while currently we could determine whether one block is computed
212
+ # or not by check whether it has content hash.
213
+ # So this function is useless for block_v2.
214
+ pass
215
216
def get_common_computed_block_ids(
217
self, seqs: List[Sequence]) -> GenericSequence[int]:
vllm/core/evictor_v1.py
renamed
vllm/core/evictor_v2.py
new
+122
@@ -0,0 +1,122 @@
1
+import enum
2
+from abc import ABC, abstractmethod, abstractproperty
3
+from typing import OrderedDict, Tuple
4
+
5
+
6
+class EvictionPolicy(enum.Enum):
7
+ """Enum for eviction policy used by make_evictor to instantiate the correct
8
+ Evictor subclass.
9
+ """
10
+ LRU = enum.auto()
11
+
12
+
13
+class Evictor(ABC):
14
+ """The Evictor subclasses should be used by the BlockAllocator class to
15
+ handle eviction of freed PhysicalTokenBlocks.
16
+ """
17
+
18
+ @abstractmethod
19
+ def __init__(self):
20
+ pass
21
+
22
+ @abstractmethod
23
+ def __contains__(self, block_id: int) -> bool:
24
+ pass
25
+
26
+ @abstractmethod
27
+ def evict(self) -> Tuple[int, int]:
28
+ """Runs the eviction algorithm and returns the evicted block's
29
+ content hash along with physical block id along with physical block id
30
+ """
31
+ pass
32
+
33
+ @abstractmethod
34
+ def add(self, block_id: int, content_hash: int, num_hashed_tokens: int,
35
+ last_accessed: int):
36
+ """Adds block to the evictor, making it a candidate for eviction"""
37
+ pass
38
+
39
+ @abstractmethod
40
+ def update(self, block_id: int, last_accessed: int):
41
+ """Update corresponding block's access time in metadata"""
42
+ pass
43
+
44
+ @abstractproperty
45
+ def num_blocks(self) -> int:
46
+ pass
47
+
48
+
49
+class BlockMetaData():
50
+ """Data structure for storing key data describe cached block, so that
51
+ evitor could use to make its decision which one to choose for eviction
52
+
53
+ Here we use physical block id as the dict key, as there maybe several
54
+ blocks with the same content hash, but their physical id is unique.
55
+ """
56
+
57
+ def __init__(self, content_hash: int, num_hashed_tokens: int,
58
+ last_accessed: int):
59
+ self.content_hash = content_hash
60
+ self.num_hashed_tokens = num_hashed_tokens
61
+ self.last_accessed = last_accessed
62
+
63
+
64
+class LRUEvictor(Evictor):
65
+ """Evicts in a least-recently-used order using the last_accessed timestamp
66
+ that's recorded in the PhysicalTokenBlock. If there are multiple blocks with
67
+ the same last_accessed time, then the one with the largest num_hashed_tokens
68
+ will be evicted. If two blocks each have the lowest last_accessed time and
69
+ highest num_hashed_tokens value, then one will be chose arbitrarily
70
+ """
71
+
72
+ def __init__(self):
73
+ self.free_table: OrderedDict[int, BlockMetaData] = OrderedDict()
74
+
75
+ def __contains__(self, block_id: int) -> bool:
76
+ return block_id in self.free_table
77
+
78
+ def evict(self) -> Tuple[int, int]:
79
+ if len(self.free_table) == 0:
80
+ raise ValueError("No usable cache memory left")
81
+
82
+ evicted_block = next(iter(self.free_table.values()))
83
+ evicted_block_id = next(iter(self.free_table.keys()))
84
+ # The blocks with the lowest timestamps should be placed consecutively
85
+ # at the start of OrderedDict. Loop through all these blocks to
86
+ # find the one with maximum number of hashed tokens.
87
+ for _id, block in self.free_table.items():
88
+ if evicted_block.last_accessed > block.last_accessed or (
89
+ evicted_block.last_accessed == block.last_accessed and
90
+ evicted_block.num_hashed_tokens < block.num_hashed_tokens):
91
+ evicted_block = block
92
+ evicted_block_id = _id
93
+
94
+ self.free_table.pop(evicted_block_id)
95
+
96
+ return evicted_block_id, evicted_block.content_hash
97
+
98
+ def add(self, block_id: int, content_hash: int, num_hashed_tokens: int,
99
+ last_accessed: int):
100
+ self.free_table[block_id] = BlockMetaData(content_hash,
101
+ num_hashed_tokens,
102
+ last_accessed)
103
+
104
+ def update(self, block_id: int, last_accessed: int):
105
+ self.free_table[block_id].last_accessed = last_accessed
106
+
107
+ def remove(self, block_id: int):
108
+ if block_id not in self.free_table:
109
+ raise ValueError(
110
+ "Attempting to remove block that's not in the evictor")
111
+ self.free_table.pop(block_id)
112
+
113
+ @property
114
+ def num_blocks(self) -> int:
115
+ return len(self.free_table)
116
+
117
+
118
+def make_evictor(eviction_policy: EvictionPolicy) -> Evictor:
119
+ if eviction_policy == EvictionPolicy.LRU:
120
+ return LRUEvictor()
121
+ else:
122
+ raise ValueError(f"Unknown cache eviction policy: {eviction_policy}")