@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 24750f4cad
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Core] Enable prefix caching with block manager v2 enabled (#4142)

Co-authored-by: Lei Wen <wenlei03@qiyi.com> Co-authored-by: Sage Moore <sagemoore@utexas.edu>

leiwen83 committed May 2, 2024 at 02:20 UTC 24750f4cadd15a2b3a52f982e39eb9803749efbc
11 files changed +584 -57
benchmarks/benchmark_prefix_caching.py
+13 -3
@@ -16,20 +16,22 @@ def test_prefix(llm=None, sampling_params=None, prompts=None):
16
17
18 def main(args):
19 - llm = LLM(model="baichuan-inc/Baichuan2-13B-Chat",
19 + llm = LLM(model=args.model,
20 tokenizer_mode='auto',
21 trust_remote_code=True,
22 enforce_eager=True,
23 + use_v2_block_manager=args.use_v2_block_manager,
24 + tensor_parallel_size=args.tensor_parallel_size,
25 enable_prefix_caching=args.enable_prefix_caching)
26
27 num_prompts = 100
28 prompts = [PROMPT] * num_prompts
27 - sampling_params = SamplingParams(temperature=0, max_tokens=100)
29 + sampling_params = SamplingParams(temperature=0, max_tokens=args.output_len)
30
31 print("------warm up------")
32 test_prefix(
33 llm=llm,
32 - prompts=prompts[:1],
34 + prompts=prompts,
35 sampling_params=sampling_params,
36 )
37
@@ -45,8 +47,16 @@ if __name__ == "__main__":
47 parser = argparse.ArgumentParser(
48 description='Benchmark the performance with or without automatic '
49 'prefix caching.')
50 + parser.add_argument('--model',
51 + type=str,
52 + default='baichuan-inc/Baichuan2-13B-Chat')
53 + parser.add_argument('--tensor-parallel-size', '-tp', type=int, default=1)
54 + parser.add_argument('--output-len', type=int, default=10)
55 parser.add_argument('--enable-prefix-caching',
56 action='store_true',
57 help='enable prefix caching')
58 + parser.add_argument('--use-v2-block-manager',
59 + action='store_true',
60 + help='Use BlockSpaceMangerV2')
61 args = parser.parse_args()
62 main(args)
tests/core/block/e2e/test_correctness.py
+146
@@ -300,6 +300,152 @@ def test_chunked_prefill_block_manager_v2(baseline_llm_generator,
300 assert baseline_token_ids == test_token_ids
301
302
303 +@pytest.mark.parametrize(
304 + "common_llm_kwargs",
305 + [{
306 + # Use a small model for a fast test.
307 + "model": "facebook/opt-125m",
308 +
309 + # skip cuda graph creation for fast test.
310 + "enforce_eager": True,
311 +
312 + # Allow only 5 sequences of ~1024 tokens in worst case.
313 + "block_size": 16,
314 + "num_gpu_blocks_override": 5 * (64 + 1),
315 +
316 + # Enable prefill cache
317 + "enable_prefix_caching": True,
318 + }])
319 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
320 +@pytest.mark.parametrize("baseline_llm_kwargs", [{
321 + "use_v2_block_manager": False
322 +}])
323 +@pytest.mark.parametrize("test_llm_kwargs", [{"use_v2_block_manager": True}])
324 +@pytest.mark.parametrize("batch_size", [10])
325 +@pytest.mark.parametrize("seed", [1])
326 +def test_v1_v2_greedy_equality_prefix_caching_enabled_with_preemption(
327 + baseline_llm_generator, test_llm_generator, batch_size):
328 + """Verify block manager v2 produces same outputs as block manager v1, even
329 + when there is preemption.
330 +
331 + This constructs two LLM, each with limited number of GPU blocks. The limit
332 + is decided such that as the sequences in the batch grow, sequences must be
333 + preempted and removed from cache.
334 +
335 + If the output token ids are equivalent, then we have confidence that the KV
336 + cache is not corrupted in the v2 block manager.
337 +
338 + NOTE: We want a significant number of generated tokens so that any incorrect
339 + KV mapping has time to build up error.
340 + """
341 + output_len = 1024
342 + temperature = 0.0
343 +
344 + # We want to ensure equality even with preemption.
345 + # We force the total block size to be 1 + cdiv(output_len, block_size)
346 + # so that only one sequence can fit at a time (once the sequences grow).
347 +
348 + prompts = [
349 + "Hello, my name is",
350 + "The president of the United States is",
351 + "The capital of France is",
352 + "The future of AI is",
353 + ]
354 +
355 + prompts = [prompt for prompt, _ in zip(cycle(prompts), range(batch_size))]
356 +
357 + sampling_params = SamplingParams(
358 + max_tokens=output_len,
359 + ignore_eos=True,
360 + temperature=temperature,
361 + )
362 +
363 + print('Getting token ids from block manager v1')
364 + baseline_token_ids = get_token_ids_from_llm_generator(
365 + baseline_llm_generator, prompts, sampling_params)
366 +
367 + print('Getting token ids from block manager v2')
368 + test_token_ids = get_token_ids_from_llm_generator(test_llm_generator,
369 + prompts, sampling_params)
370 +
371 + for expected_token_ids, actual_token_ids in zip(baseline_token_ids,
372 + test_token_ids):
373 + assert expected_token_ids == actual_token_ids
374 +
375 + assert baseline_token_ids == test_token_ids
376 +
377 +
378 +@pytest.mark.parametrize(
379 + "common_llm_kwargs",
380 + [{
381 + # Use a small model for a fast test.
382 + "model": "facebook/opt-125m",
383 +
384 + # skip cuda graph creation for fast test.
385 + "enforce_eager": True,
386 +
387 + # Allow only 5 sequences of ~1024 tokens in worst case.
388 + "block_size": 16,
389 + "num_gpu_blocks_override": 5 * (64 + 1),
390 +
391 + # Test APC in v2 block
392 + "use_v2_block_manager": True,
393 + }])
394 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
395 +@pytest.mark.parametrize("baseline_llm_kwargs", [{
396 + "enable_prefix_caching": False
397 +}])
398 +@pytest.mark.parametrize("test_llm_kwargs", [{"enable_prefix_caching": True}])
399 +@pytest.mark.parametrize("batch_size", [10])
400 +@pytest.mark.parametrize("seed", [1])
401 +def test_auto_prefix_caching_with_preemption(baseline_llm_generator,
402 + test_llm_generator, batch_size):
403 + """Verify block manager v2 with auto prefix caching enabled produces same
404 + outputs as auto prefix caching disabled, even when there is preemption.
405 +
406 + This constructs two LLM, each with limited number of GPU blocks. The limit
407 + is decided such that as the sequences in the batch grow, sequences must be
408 + preempted and removed from cache.
409 +
410 + If the output token ids are equivalent, then we have confidence that auto
411 + prefix caching itself at least don't cause result error.
412 + """
413 + output_len = 1024
414 + temperature = 0.0
415 +
416 + # We want to ensure equality even with preemption.
417 + # We force the total block size to be 1 + cdiv(output_len, block_size)
418 + # so that only one sequence can fit at a time (once the sequences grow).
419 + prompts = [
420 + "Hello, my name is",
421 + "The president of the United States is",
422 + "The capital of France is",
423 + "The future of AI is",
424 + ]
425 +
426 + prompts = [prompt for prompt, _ in zip(cycle(prompts), range(batch_size))]
427 +
428 + sampling_params = SamplingParams(
429 + max_tokens=output_len,
430 + ignore_eos=True,
431 + temperature=temperature,
432 + )
433 +
434 + print('Getting token ids with APC disabled')
435 + baseline_token_ids = get_token_ids_from_llm_generator(
436 + baseline_llm_generator, prompts, sampling_params)
437 +
438 + print('Getting token ids with APC enabled')
439 + test_token_ids = get_token_ids_from_llm_generator(test_llm_generator,
440 + prompts, sampling_params)
441 +
442 + for expected_token_ids, actual_token_ids in zip(baseline_token_ids,
443 + test_token_ids):
444 + assert expected_token_ids == actual_token_ids
445 +
446 + assert baseline_token_ids == test_token_ids
447 +
448 +
449 def get_token_ids_from_llm_generator(llm_generator, prompts, sampling_params):
450 for llm in llm_generator:
451 outputs = llm.generate(prompts, sampling_params, use_tqdm=True)
tests/core/block/test_prefix_caching_block.py
+125
@@ -358,6 +358,131 @@ class TestPrefixCachingBlockAllocator:
358 i)
359 allocator.free(block)
360
361 + @staticmethod
362 + @pytest.mark.parametrize("num_blocks", [1024])
363 + @pytest.mark.parametrize("block_size", [16])
364 + @pytest.mark.parametrize("seed", list(range(20)))
365 + def test_get_common_computed_block_ids(num_blocks: int, block_size: int,
366 + seed: int):
367 + """Verify get_common_computed_block_ids could get correct result
368 + by create two immutable chain sharing prefix at specified pos,
369 + and compare whether we also could get right result
370 + from get_common_computed_block_ids.
371 + """
372 + random.seed(seed)
373 + allocator = PrefixCachingBlockAllocator(num_blocks=num_blocks * 2,
374 + block_size=block_size)
375 + num_blocks_to_consume = random.randint(1, num_blocks - 1)
376 +
377 + # Create token ids that will exhaust all blocks.
378 + token_ids = list(range(num_blocks_to_consume * block_size))
379 + blocks = list(range(num_blocks_to_consume))
380 +
381 + first_chain = TestPrefixCachingBlockAllocator.create_immutable_chain(
382 + block_size=block_size,
383 + token_ids=token_ids,
384 + allocator=allocator,
385 + )
386 +
387 + # mark all blocks in first chain as computed
388 + allocator.mark_blocks_as_computed(blocks)
389 +
390 + # After zero_point, second_chain's token_ids would be set -1, which
391 + # make it different from here comparing with first_chain
392 + zero_point = random.randint(1, len(token_ids) - 1)
393 + zero_point_blocks = zero_point // block_size
394 + token_ids[zero_point:] = [-1] * (len(token_ids) - zero_point)
395 +
396 + second_chain = TestPrefixCachingBlockAllocator.create_immutable_chain(
397 + block_size=block_size,
398 + token_ids=token_ids,
399 + allocator=allocator,
400 + )
401 +
402 + first_computed_ids = [
403 + first_chain[i].block_id for i in range(num_blocks_to_consume)
404 + ]
405 + second_computed_ids = [
406 + second_chain[i].block_id for i in range(num_blocks_to_consume)
407 + ]
408 + res = allocator.get_common_computed_block_ids(
409 + [first_computed_ids, second_computed_ids])
410 +
411 + assert (len(res) == zero_point_blocks)
412 +
413 + # Test case where two last accessed times are equal
414 + @staticmethod
415 + @pytest.mark.parametrize("num_blocks", [1024])
416 + @pytest.mark.parametrize("block_size", [16])
417 + @pytest.mark.parametrize("seed", list(range(20)))
418 + def test_eviction_order(num_blocks: int, block_size: int, seed: int):
419 + """This test case simulate the two chain created and free in order,
420 + and together they would exhaust the initial freed blocks.
421 +
422 + So the next block created after those two chain shall use the block
423 + from the first chain as that block has long access time.
424 + While first chain has two blocks, it shall pick up the last one, as
425 + it has larger token number.
426 + """
427 +
428 + random.seed(seed)
429 + allocator = PrefixCachingBlockAllocator(num_blocks=num_blocks,
430 + block_size=block_size)
431 + num_blocks_to_consume = num_blocks + 1
432 +
433 + token_ids = list(range(num_blocks_to_consume * block_size))
434 +
435 + num_blocks_in_first_chain = 2
436 + num_tokens_in_first_chain = block_size * num_blocks_in_first_chain
437 + # First chain takes the first block
438 + first_chain = TestPrefixCachingBlockAllocator.create_immutable_chain(
439 + block_size=block_size,
440 + token_ids=token_ids[:num_tokens_in_first_chain],
441 + allocator=allocator,
442 + )
443 + # There should only be one block allocated at this point
444 + assert allocator.get_num_free_blocks() == (num_blocks -
445 + num_blocks_in_first_chain)
446 +
447 + # Set the last accessed time of the first block to 1
448 + blocks_ids = [block.block_id for block in first_chain]
449 + allocator.mark_blocks_as_accessed(blocks_ids, 1)
450 +
451 + # Second chain takes the rest of the blocks
452 + second_chain = TestPrefixCachingBlockAllocator.create_immutable_chain(
453 + block_size=block_size,
454 + token_ids=token_ids[num_tokens_in_first_chain:-block_size],
455 + allocator=allocator,
456 + )
457 +
458 + # There shouldn't be any blocks left at this point
459 + assert allocator.get_num_free_blocks() == (0)
460 +
461 + assert len(first_chain) == num_blocks_in_first_chain
462 + last_block_id = first_chain[-1].block_id
463 + # Free each block in the first chain.
464 + for i, block in enumerate(first_chain):
465 + allocator.free(block)
466 +
467 + # Set the last accessed time on all of the blocks in the second chain
468 + # to 2
469 + blocks_ids = [block.block_id for block in second_chain]
470 + allocator.mark_blocks_as_accessed(blocks_ids, 2)
471 +
472 + # Free each block in the second chain.
473 + for i, block in enumerate(second_chain):
474 + allocator.free(block)
475 +
476 + # Allocate a new block and check that it's the least recently used block
477 + # from the first chain.
478 + new_block = TestPrefixCachingBlockAllocator.create_immutable_chain(
479 + block_size=block_size,
480 + token_ids=token_ids[-block_size:],
481 + allocator=allocator,
482 + )
483 +
484 + assert new_block[0].block_id == last_block_id
485 +
486 @staticmethod
487 def create_immutable_chain(
488 block_size: int,
vllm/core/block/cpu_gpu_block_allocator.py
+10 -2
@@ -190,10 +190,18 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
190 device = Device.GPU
191 return self._allocators[device].clear_copy_on_writes()
192
193 - def mark_blocks_as_computed(self) -> None:
193 + def mark_blocks_as_accessed(self, block_ids: List[int],
194 + now: float) -> None:
195 + """Mark blocks as accessed, only use for prefix caching."""
196 # Prefix caching only supported on GPU.
197 device = Device.GPU
196 - return self._allocators[device].mark_blocks_as_computed()
198 + return self._allocators[device].mark_blocks_as_accessed(block_ids, now)
199 +
200 + def mark_blocks_as_computed(self, block_ids: List[int]) -> None:
201 + """Mark blocks as accessed, only use for prefix caching."""
202 + # Prefix caching only supported on GPU.
203 + device = Device.GPU
204 + return self._allocators[device].mark_blocks_as_computed(block_ids)
205
206 def get_common_computed_block_ids(
207 self, seq_block_ids: List[List[int]]) -> List[int]:
vllm/core/block/interfaces.py
+4
@@ -81,6 +81,10 @@ class BlockAllocator(ABC):
81 def clear_copy_on_writes(self) -> Dict[int, List[int]]:
82 pass
83
84 + @abstractmethod
85 + def mark_blocks_as_accessed(self) -> None:
86 + pass
87 +
88 @abstractmethod
89 def mark_blocks_as_computed(self) -> None:
90 pass
vllm/core/block/naive_block.py
+10 -1
@@ -174,7 +174,16 @@ class NaiveBlockAllocator(BlockAllocator):
174 """
175 return self._cow_tracker.clear_cows()
176
177 - def mark_blocks_as_computed(self) -> None:
177 + def mark_blocks_as_accessed(self, block_ids: List[int],
178 + now: float) -> None:
179 + """Mark blocks as accessed, used in prefix caching.
180 +
181 + Since the naive allocator does not implement prefix caching, we do
182 + nothing.
183 + """
184 + pass
185 +
186 + def mark_blocks_as_computed(self, block_ids: List[int]) -> None:
187 """Mark blocks as computed, used in prefix caching.
188
189 Since the naive allocator does not implement prefix caching, we do
vllm/core/block/prefix_caching_block.py
+134 -38
@@ -7,10 +7,16 @@ from vllm.core.block.common import (CopyOnWriteTracker,
7 get_all_blocks_recursively)
8 from vllm.core.block.interfaces import Block, BlockAllocator
9 from vllm.core.block.naive_block import NaiveBlock, NaiveBlockAllocator
10 +from vllm.core.evictor_v2 import EvictionPolicy, Evictor, make_evictor
11
12 PrefixHash = int
13 BlockId = int
14
15 +# By default, we init our block access time as _DEFAULT_LAST_ACCESSED_TIME
16 +# so that if we find one block is still hold _DEFAULT_LAST_ACCESSED_TIME,
17 +# then we know this block hasn't been accessed yet.
18 +_DEFAULT_LAST_ACCESSED_TIME = -1
19 +
20
21 class PrefixCachingBlockAllocator(BlockAllocator):
22 """A block allocator that implements prefix caching.
@@ -27,22 +33,19 @@ class PrefixCachingBlockAllocator(BlockAllocator):
33 from 0 to num_blocks - 1.
34 """
35
30 - # TODO last access time / evictor integration
31 -
36 def __init__(
37 self,
38 num_blocks: int,
39 block_size: int,
40 block_ids: Optional[Iterable[int]] = None,
41 + eviction_policy: Optional[EvictionPolicy] = EvictionPolicy.LRU,
42 ):
43 # A mapping of prefix hash to block index. All blocks which have a
44 # prefix hash will be in this dict, even if they have refcount 0.
45 self._cached_blocks: Dict[PrefixHash, BlockId] = {}
46
42 - # A mapping of prefix hash to block index. All blocks which have a
43 - # prefix hash AND refcount 0 will be in this dict. Thus, it is a subset
44 - # of self._cached_blocks.
45 - self._unused_cached_blocks: Dict[PrefixHash, BlockId] = {}
47 + # A mapping of blockId to Block to track those cached blocks
48 + self._blocks: Dict[BlockId, Block] = {}
49
50 # An allocator for blocks that do not have prefix hashes.
51 self._hashless_allocator = NaiveBlockAllocator(
@@ -54,6 +57,10 @@ class PrefixCachingBlockAllocator(BlockAllocator):
57
58 self._block_size = block_size
59
60 + # Evitor used to maintain how we want to handle those computed blocks
61 + # if we find memory pressure is high.
62 + self.evictor: Evictor = make_evictor(eviction_policy)
63 +
64 # We share the refcounter between allocators. This allows us to promote
65 # blocks originally allocated in the hashless allocator to immutable
66 # blocks.
@@ -72,6 +79,7 @@ class PrefixCachingBlockAllocator(BlockAllocator):
79 block_size: int,
80 allocator: BlockAllocator,
81 block_id: Optional[int] = None,
82 + computed: Optional[bool] = False,
83 ) -> Block:
84 # Bind block to self.
85 allocator = self
@@ -82,6 +90,7 @@ class PrefixCachingBlockAllocator(BlockAllocator):
90 block_size=block_size,
91 block_id=block_id,
92 prefix_caching_allocator=allocator,
93 + computed=computed,
94 )
95
96 def allocate_immutable(self, prev_block: Optional[Block],
@@ -109,14 +118,12 @@ class PrefixCachingBlockAllocator(BlockAllocator):
118 cached_block_id = self._cached_blocks.get(block.content_hash, None)
119 if cached_block_id is not None:
120 block.block_id = cached_block_id
112 - self._incr_refcount_cached_block(block.content_hash,
113 - block.block_id)
121 + self._incr_refcount_cached_block(block, block.block_id)
122 return block
123
124 block = self.allocate_mutable(prev_block)
125 block.append_token_ids(token_ids)
126 assert block.content_hash is not None
119 - # TODO computed bit
127
128 return block
129
@@ -133,41 +140,67 @@ class PrefixCachingBlockAllocator(BlockAllocator):
140 assert_prefix_caching_block_or_none(prev_block)
141
142 try:
136 - return self._hashless_allocator.allocate_mutable(
143 + block = self._hashless_allocator.allocate_mutable(
144 prev_block=prev_block)
145 +
146 + assert block.block_id not in self._blocks
147 + self._blocks[block.block_id] = block
148 + return block
149 except BlockAllocator.NoFreeBlocksError:
150 # We must check the unused cached blocks before raising OOM.
151 pass
152
142 - if self._unused_cached_blocks:
143 - # TODO policy for selecting block to remove
144 - content_hash_to_evict = next(iter(self._unused_cached_blocks))
153 + # If the evictor has blocks available for eviction, evict a block
154 + # and return it.
155 + if self.evictor.num_blocks > 0:
156 + block_id, content_hash_to_evict = self.evictor.evict()
157 +
158 + # Here we may have scenario that several blocks have
159 + # the same content hash, but due to the latter coming block
160 + # is coming from mutable to immutable path, their physical
161 + # block is added into evictor.
162 + # However in this case, we shall not pop the _cached_blocks,
163 + # as the same content is still used by others, which means
164 + # we need to check ref before decide to pop the list.
165
146 - # Clear content hash mapping; the block will be overwritten.
147 - del self._cached_blocks[content_hash_to_evict]
166 + _block_id = self._cached_blocks[content_hash_to_evict]
167 + refcount = self._refcounter.get(_block_id)
168 + if refcount == 1:
169 + self._cached_blocks.pop(content_hash_to_evict)
170 + assert _block_id == block_id
171
149 - block_id = self._unused_cached_blocks.pop(content_hash_to_evict)
150 - refcount = self._refcounter.incr(block_id)
151 - assert refcount == 1
172 + self._refcounter.incr(block_id)
173 +
174 + # the block comes from evictor already contain computed result
175 block = self._create_block(
176 prev_block=prev_block,
177 token_ids=[],
178 block_size=self._block_size,
179 allocator=self,
180 block_id=block_id,
181 + computed=True,
182 )
183 assert block.content_hash is None
184 +
185 + assert block.block_id not in self._blocks
186 + self._blocks[block.block_id] = block
187 return block
188
189 # No block available in hashless allocator, nor in unused cache blocks.
190 raise BlockAllocator.NoFreeBlocksError()
191
165 - def _incr_refcount_cached_block(self, content_hash: int,
192 + def _incr_refcount_cached_block(self, block: Block,
193 block_id: BlockId) -> None:
194 + # since block is already computed, mark it
195 + block.computed = True
196 +
197 refcount = self._refcounter.incr(block_id)
198 if refcount == 1:
169 - assert content_hash in self._unused_cached_blocks
170 - del self._unused_cached_blocks[content_hash]
199 + # if block get referred, then it shall not be in evictor
200 + # and put it into _blocks for tracking
201 + if block_id in self.evictor:
202 + self.evictor.remove(block_id)
203 + self._blocks[block_id] = block
204
205 def free(self, block: Block) -> None:
206 """Decrement the refcount of the block. If the decremented refcount is
@@ -180,6 +213,7 @@ class PrefixCachingBlockAllocator(BlockAllocator):
213 is not None), "freeing unallocated block is undefined"
214
215 self._free_block_id_for_block(block.block_id, block)
216 +
217 block.block_id = None
218
219 def _free_block_id_for_block(self, block_id: BlockId,
@@ -187,15 +221,21 @@ class PrefixCachingBlockAllocator(BlockAllocator):
221 assert isinstance(block, PrefixCachingBlock)
222
223 if block.content_hash is None:
224 + refcount = self._refcounter.get(block_id)
225 + # We have fork case where block would get more than one ref,
226 + # so we cannot free it from tracking if ref cnt large than 1
227 + if refcount <= 1:
228 + del self._blocks[block.block_id]
229 return self._hashless_allocator.free(block)
230
231 refcount = self._refcounter.decr(block_id)
232
194 - # If no longer used, add the block to the unused cached blocks.
233 + # If no longer used, add the block to the evictor.
234 if refcount == 0:
196 - assert block.content_hash not in self._unused_cached_blocks
235 assert block.content_hash in self._cached_blocks
198 - self._unused_cached_blocks[block.content_hash] = block_id
236 + del self._blocks[block.block_id]
237 + self.evictor.add(block.block_id, block.content_hash,
238 + block.num_tokens_total, block.last_accessed)
239
240 def fork(self, last_block: Block) -> List[Block]:
241 """Creates a new sequence of blocks that shares the same underlying
@@ -230,9 +270,9 @@ class PrefixCachingBlockAllocator(BlockAllocator):
270
271 def get_num_free_blocks(self) -> int:
272 # The number of free blocks is the number of hashless free blocks
233 - # plus the number of hashful blocks that are unused.
234 - return self._hashless_allocator.get_num_free_blocks() + len(
235 - self._unused_cached_blocks)
273 + # plus the number of blocks evictor could free from its list.
274 + return self._hashless_allocator.get_num_free_blocks(
275 + ) + self.evictor.num_blocks
276
277 @property
278 def all_block_ids(self) -> frozenset[int]:
@@ -266,7 +306,7 @@ class PrefixCachingBlockAllocator(BlockAllocator):
306 else:
307 self._free_block_id_for_block(block.block_id, block)
308 self._incr_refcount_cached_block(
269 - block.content_hash, self._cached_blocks[block.content_hash])
309 + block, self._cached_blocks[block.content_hash])
310
311 return self._cached_blocks[block.content_hash]
312
@@ -293,29 +333,60 @@ class PrefixCachingBlockAllocator(BlockAllocator):
333 """
334 return self._cow_tracker.clear_cows()
335
296 - def mark_blocks_as_computed(self) -> None:
336 + def mark_blocks_as_accessed(self, block_ids: List[int],
337 + now: float) -> None:
338 + """Mark blocks as accessed, used in prefix caching.
339 +
340 + If the block is added into evictor, we need to update corresponding
341 + info in evictor's metadata.
342 + """
343 +
344 + for block_id in block_ids:
345 + if block_id in self._blocks:
346 + self._blocks[block_id].last_accessed = now
347 + elif block_id in self.evictor:
348 + self.evictor.update(block_id, now)
349 + else:
350 + raise ValueError(
351 + "Mark block as accessed which is not belonged to GPU")
352 +
353 + def mark_blocks_as_computed(self, block_ids: List[int]) -> None:
354 """Mark blocks as computed, used in prefix caching."""
298 - # TODO Track computed blocks.
299 - pass
355 +
356 + for block_id in block_ids:
357 + if block_id in self._blocks:
358 + # only those full block is valid for prefix caching
359 + if self._blocks[block_id].is_full:
360 + self._blocks[block_id].computed = True
361 + elif block_id not in self.evictor:
362 + raise ValueError(f"Mark {block_id=} as computed which "
363 + "is not belonged to GPU")
364 +
365 + def block_is_computed(self, block_id: int) -> bool:
366 + if block_id in self._blocks:
367 + return self._blocks[block_id].computed
368 + else:
369 + return block_id in self.evictor
370
371 def get_common_computed_block_ids(
372 self, seq_block_ids: List[List[int]]) -> List[int]:
373 """Return the block ids that are common for a given sequence group.
374
305 - Used in prefill (can skip prefill of some blocks).
375 + Only those blocks that are immutable and already be marked
376 + compyted would be taken consideration.
377 """
378
308 - # TODO: Track computed blocks.
309 - computed = lambda block_id: False
310 -
379 # NOTE We exclude the last block to avoid the case where the entire
380 # prompt is cached. This would cause erroneous behavior in model
381 # runner.
382 +
383 ids_list = [
315 - takewhile(lambda block_id: computed(block_id), seq[:-1])
316 - for seq in seq_block_ids
384 + list(
385 + takewhile(lambda block_id: self.block_is_computed(block_id),
386 + seq[:-1])) for seq in seq_block_ids
387 ]
318 - return commonprefix([ids for ids in ids_list if ids != []])
388 + res = commonprefix([ids for ids in ids_list if ids != []])
389 + return res
390
391
392 class PrefixCachingBlock(Block):
@@ -345,12 +416,16 @@ class PrefixCachingBlock(Block):
416 block_size: int,
417 prefix_caching_allocator: PrefixCachingBlockAllocator,
418 block_id: Optional[int] = None,
419 + computed: Optional[bool] = False,
420 ):
421 assert_prefix_caching_block_or_none(prev_block)
422
423 self._prev_block = prev_block
424 self._cached_content_hash: Optional[int] = None
425 + self._cached_num_tokens_total: Optional[int] = None
426 self._prefix_caching_allocator = prefix_caching_allocator
427 + self.last_accessed = _DEFAULT_LAST_ACCESSED_TIME
428 + self.computed = computed
429
430 self._block = NaiveBlock(
431 prev_block=prev_block,
@@ -398,6 +473,27 @@ class PrefixCachingBlock(Block):
473 def num_empty_slots(self) -> int:
474 return self._block.num_empty_slots
475
476 + @property
477 + def num_tokens_total(self) -> int:
478 + """return the total tokens so far.
479 +
480 + Here we iterate the block chain till to the first block, while
481 + cache the result in local to prevent repeated computations.
482 + """
483 + if self._cached_num_tokens_total is not None:
484 + return self._cached_num_tokens_total
485 +
486 + _block = self
487 + self._cached_num_tokens_total = 0
488 +
489 + # TODO: current implement here take O(N^2), we expect future
490 + # we have O(1) here
491 + while _block is not None:
492 + self._cached_num_tokens_total += len(_block.token_ids)
493 + _block = _block.prev_block
494 +
495 + return self._cached_num_tokens_total
496 +
497 @property
498 def block_size(self) -> int:
499 return self._block.block_size
vllm/core/block_manager_v1.py
+1 -1
@@ -8,7 +8,7 @@ from typing import Sequence as GenericSequence
8 from typing import Set
9
10 from vllm.block import BlockTable, PhysicalTokenBlock
11 -from vllm.core.evictor import EvictionPolicy, Evictor, make_evictor
11 +from vllm.core.evictor_v1 import EvictionPolicy, Evictor, make_evictor
12 from vllm.core.interfaces import AllocStatus, BlockSpaceManager
13 from vllm.logger import init_logger
14 from vllm.sequence import Sequence, SequenceGroup, SequenceStatus
vllm/core/block_manager_v2.py
+19 -12
@@ -72,14 +72,12 @@ class BlockSpaceManagerV2(BlockSpaceManager):
72 self.watermark = watermark
73 assert watermark >= 0.0
74
75 - assert not enable_caching, "Prefix caching not yet supported"
75 self.enable_caching = enable_caching
76
77 self.watermark_blocks = int(watermark * num_gpu_blocks)
78
79 self.block_allocator = CpuGpuBlockAllocator.create(
81 - # Currently, only naive blocks are supported (no prefix caching).
82 - allocator_type="naive",
80 + allocator_type="prefix_caching" if enable_caching else "naive",
81 num_gpu_blocks=num_gpu_blocks,
82 num_cpu_blocks=num_cpu_blocks,
83 block_size=block_size,
@@ -194,17 +192,26 @@ class BlockSpaceManagerV2(BlockSpaceManager):
192 assert all(b is not None for b in block_ids)
193 return block_ids
194
197 - def access_all_blocks_in_seq(self, seq, now):
198 - # TODO add prefix caching support.
199 - # Tracked here https://github.com/vllm-project/vllm/issues/3667
200 - pass
195 + def access_all_blocks_in_seq(self, seq: Sequence, now: float):
196 + # Update the last accessed time of all the blocks accessed
197 + # in this step.
198 + # And the accessed time is only useful for prefix caching now,
199 + # as it support internal evictor policy for which cached
200 + # block could be refilled, to keep cached content could be reused
201 + # at max extend.
202 + if self.enable_caching:
203 + block_table = self.block_tables[seq.seq_id]
204 + block_ids = []
205 + for block_id in block_table.physical_block_ids:
206 + block_ids.append(block_id)
207 + self.block_allocator.mark_blocks_as_accessed(block_ids, now)
208
209 def mark_blocks_as_computed(self, seq_group: SequenceGroup):
203 - # We ignore the sequence group as its not necessary. After the batch is
204 - # formed by the scheduler, we do not need to mark blocks from individual
205 - # sequence groups as computed -- all blocks in the batch can be marked
206 - # as computed.
207 - self.block_allocator.mark_blocks_as_computed()
210 + # The only need for mark block as computed is for prefix caching,
211 + # while currently we could determine whether one block is computed
212 + # or not by check whether it has content hash.
213 + # So this function is useless for block_v2.
214 + pass
215
216 def get_common_computed_block_ids(
217 self, seqs: List[Sequence]) -> GenericSequence[int]:
vllm/core/evictor_v1.py renamed
vllm/core/evictor_v2.py new
+122
@@ -0,0 +1,122 @@
1 +import enum
2 +from abc import ABC, abstractmethod, abstractproperty
3 +from typing import OrderedDict, Tuple
4 +
5 +
6 +class EvictionPolicy(enum.Enum):
7 + """Enum for eviction policy used by make_evictor to instantiate the correct
8 + Evictor subclass.
9 + """
10 + LRU = enum.auto()
11 +
12 +
13 +class Evictor(ABC):
14 + """The Evictor subclasses should be used by the BlockAllocator class to
15 + handle eviction of freed PhysicalTokenBlocks.
16 + """
17 +
18 + @abstractmethod
19 + def __init__(self):
20 + pass
21 +
22 + @abstractmethod
23 + def __contains__(self, block_id: int) -> bool:
24 + pass
25 +
26 + @abstractmethod
27 + def evict(self) -> Tuple[int, int]:
28 + """Runs the eviction algorithm and returns the evicted block's
29 + content hash along with physical block id along with physical block id
30 + """
31 + pass
32 +
33 + @abstractmethod
34 + def add(self, block_id: int, content_hash: int, num_hashed_tokens: int,
35 + last_accessed: int):
36 + """Adds block to the evictor, making it a candidate for eviction"""
37 + pass
38 +
39 + @abstractmethod
40 + def update(self, block_id: int, last_accessed: int):
41 + """Update corresponding block's access time in metadata"""
42 + pass
43 +
44 + @abstractproperty
45 + def num_blocks(self) -> int:
46 + pass
47 +
48 +
49 +class BlockMetaData():
50 + """Data structure for storing key data describe cached block, so that
51 + evitor could use to make its decision which one to choose for eviction
52 +
53 + Here we use physical block id as the dict key, as there maybe several
54 + blocks with the same content hash, but their physical id is unique.
55 + """
56 +
57 + def __init__(self, content_hash: int, num_hashed_tokens: int,
58 + last_accessed: int):
59 + self.content_hash = content_hash
60 + self.num_hashed_tokens = num_hashed_tokens
61 + self.last_accessed = last_accessed
62 +
63 +
64 +class LRUEvictor(Evictor):
65 + """Evicts in a least-recently-used order using the last_accessed timestamp
66 + that's recorded in the PhysicalTokenBlock. If there are multiple blocks with
67 + the same last_accessed time, then the one with the largest num_hashed_tokens
68 + will be evicted. If two blocks each have the lowest last_accessed time and
69 + highest num_hashed_tokens value, then one will be chose arbitrarily
70 + """
71 +
72 + def __init__(self):
73 + self.free_table: OrderedDict[int, BlockMetaData] = OrderedDict()
74 +
75 + def __contains__(self, block_id: int) -> bool:
76 + return block_id in self.free_table
77 +
78 + def evict(self) -> Tuple[int, int]:
79 + if len(self.free_table) == 0:
80 + raise ValueError("No usable cache memory left")
81 +
82 + evicted_block = next(iter(self.free_table.values()))
83 + evicted_block_id = next(iter(self.free_table.keys()))
84 + # The blocks with the lowest timestamps should be placed consecutively
85 + # at the start of OrderedDict. Loop through all these blocks to
86 + # find the one with maximum number of hashed tokens.
87 + for _id, block in self.free_table.items():
88 + if evicted_block.last_accessed > block.last_accessed or (
89 + evicted_block.last_accessed == block.last_accessed and
90 + evicted_block.num_hashed_tokens < block.num_hashed_tokens):
91 + evicted_block = block
92 + evicted_block_id = _id
93 +
94 + self.free_table.pop(evicted_block_id)
95 +
96 + return evicted_block_id, evicted_block.content_hash
97 +
98 + def add(self, block_id: int, content_hash: int, num_hashed_tokens: int,
99 + last_accessed: int):
100 + self.free_table[block_id] = BlockMetaData(content_hash,
101 + num_hashed_tokens,
102 + last_accessed)
103 +
104 + def update(self, block_id: int, last_accessed: int):
105 + self.free_table[block_id].last_accessed = last_accessed
106 +
107 + def remove(self, block_id: int):
108 + if block_id not in self.free_table:
109 + raise ValueError(
110 + "Attempting to remove block that's not in the evictor")
111 + self.free_table.pop(block_id)
112 +
113 + @property
114 + def num_blocks(self) -> int:
115 + return len(self.free_table)
116 +
117 +
118 +def make_evictor(eviction_policy: EvictionPolicy) -> Evictor:
119 + if eviction_policy == EvictionPolicy.LRU:
120 + return LRUEvictor()
121 + else:
122 + raise ValueError(f"Unknown cache eviction policy: {eviction_policy}")