@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3476ed0809
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[Core] Optimize block_manager_v2 vs block_manager_v1 (to make V2 default) (#5602)

Alexander Matveev committed Jul 1, 2024 at 23:10 UTC 3476ed0809ec91a3457da0cb90543133a4f4b519
19 files changed +1189 -532
benchmarks/benchmark_latency.py
+4
@@ -46,6 +46,7 @@ def main(args: argparse.Namespace):
46 load_format=args.load_format,
47 distributed_executor_backend=args.distributed_executor_backend,
48 otlp_traces_endpoint=args.otlp_traces_endpoint,
49 + enable_prefix_caching=args.enable_prefix_caching,
50 )
51
52 sampling_params = SamplingParams(
@@ -220,6 +221,9 @@ if __name__ == '__main__':
221 action='store_true',
222 help='If True, the prefill requests can be chunked based on the '
223 'max_num_batched_tokens')
224 + parser.add_argument("--enable-prefix-caching",
225 + action='store_true',
226 + help="Enable automatic prefix caching")
227 parser.add_argument('--use-v2-block-manager', action='store_true')
228 parser.add_argument(
229 "--ray-workers-use-nsight",
tests/conftest.py
+1 -1
@@ -474,7 +474,7 @@ class VllmRunner:
474 req_sample_output_strs: List[str] = []
475 for sample in req_output.outputs:
476 output_str = sample.text
477 - output_ids = sample.token_ids
477 + output_ids = list(sample.token_ids)
478 req_sample_output_ids.append(prompt_ids + output_ids)
479 req_sample_output_strs.append(prompt_str + output_str)
480 outputs.append((req_sample_output_ids, req_sample_output_strs))
tests/core/block/test_block_table.py
+3 -2
@@ -373,8 +373,9 @@ def test_cow(block_size: int, sequence_len: int, append_len: int,
373 block_size) - (sequence_len // block_size)
374
375 original_block_table.allocate(token_ids=token_ids, device=Device.GPU)
376 - original_block_ids = original_block_table.physical_block_ids
376 + original_block_ids = original_block_table.physical_block_ids[:]
377
378 + print("original_block_ids = {}".format(original_block_ids))
379 forked_block_table = original_block_table.fork()
380
381 # Expect no additional allocation (copy on _write_).
@@ -457,7 +458,7 @@ def test_cow_lookahead_simple(block_size: int, sequence_len: int,
458
459 # Allocate lookahead slots.
460 original_block_table.ensure_num_empty_slots(lookahead_slots)
460 - original_block_ids = original_block_table.physical_block_ids
461 + original_block_ids = original_block_table.physical_block_ids[:]
462
463 forked_block_table = original_block_table.fork()
464
tests/core/block/test_cpu_gpu_block_allocator.py
+12 -12
@@ -8,8 +8,8 @@ from vllm.utils import Device, chunk_list
8 @pytest.mark.parametrize("num_gpu_blocks", [1024])
9 @pytest.mark.parametrize("block_size", [16])
10 @pytest.mark.parametrize("allocator_type", ["naive", "prefix_caching"])
11 -def test_allocate_mutable(num_cpu_blocks: int, num_gpu_blocks: int,
12 - block_size: int, allocator_type: str):
11 +def test_allocate_mutable_block(num_cpu_blocks: int, num_gpu_blocks: int,
12 + block_size: int, allocator_type: str):
13 allocator = CpuGpuBlockAllocator.create(
14 allocator_type=allocator_type,
15 num_gpu_blocks=num_gpu_blocks,
@@ -21,14 +21,14 @@ def test_allocate_mutable(num_cpu_blocks: int, num_gpu_blocks: int,
21 assert allocator.get_num_free_blocks(Device.GPU) == num_gpu_blocks
22
23 cpu_blocks = [
24 - allocator.allocate_mutable(prev_block=None, device=Device.CPU)
24 + allocator.allocate_mutable_block(prev_block=None, device=Device.CPU)
25 for _ in range(num_cpu_blocks)
26 ]
27 assert allocator.get_num_free_blocks(Device.CPU) == 0
28 assert allocator.get_num_free_blocks(Device.GPU) == num_gpu_blocks
29
30 gpu_blocks = [
31 - allocator.allocate_mutable(prev_block=None, device=Device.GPU)
31 + allocator.allocate_mutable_block(prev_block=None, device=Device.GPU)
32 for _ in range(num_gpu_blocks)
33 ]
34 assert allocator.get_num_free_blocks(Device.CPU) == 0
@@ -47,8 +47,8 @@ def test_allocate_mutable(num_cpu_blocks: int, num_gpu_blocks: int,
47 @pytest.mark.parametrize("num_gpu_blocks", [1024])
48 @pytest.mark.parametrize("block_size", [2])
49 @pytest.mark.parametrize("allocator_type", ["naive", "prefix_caching"])
50 -def test_allocate_immutable(num_cpu_blocks: int, num_gpu_blocks: int,
51 - block_size: int, allocator_type: str):
50 +def test_allocate_immutable_block(num_cpu_blocks: int, num_gpu_blocks: int,
51 + block_size: int, allocator_type: str):
52 allocator = CpuGpuBlockAllocator.create(
53 allocator_type=allocator_type,
54 num_gpu_blocks=num_gpu_blocks,
@@ -67,18 +67,18 @@ def test_allocate_immutable(num_cpu_blocks: int, num_gpu_blocks: int,
67 assert allocator.get_num_free_blocks(Device.GPU) == num_gpu_blocks
68
69 cpu_blocks = [
70 - allocator.allocate_immutable(prev_block=None,
71 - token_ids=token_ids,
72 - device=Device.CPU)
70 + allocator.allocate_immutable_block(prev_block=None,
71 + token_ids=token_ids,
72 + device=Device.CPU)
73 for token_ids in cpu_token_ids
74 ]
75 assert allocator.get_num_free_blocks(Device.CPU) == 0
76 assert allocator.get_num_free_blocks(Device.GPU) == num_gpu_blocks
77
78 gpu_blocks = [
79 - allocator.allocate_immutable(prev_block=None,
80 - token_ids=token_ids,
81 - device=Device.GPU)
79 + allocator.allocate_immutable_block(prev_block=None,
80 + token_ids=token_ids,
81 + device=Device.GPU)
82 for token_ids in gpu_token_ids
83 ]
84 assert allocator.get_num_free_blocks(Device.CPU) == 0
tests/core/block/test_naive_block.py
+3 -3
@@ -14,11 +14,11 @@ class TestNaiveBlockAllocator:
14 prev_block: Optional[Block],
15 token_ids: List[int]):
16 if allocate_type == "immutable":
17 - allocate_block = lambda: allocator.allocate_immutable(
17 + allocate_block = lambda: allocator.allocate_immutable_block(
18 prev_block=prev_block, token_ids=token_ids)
19 elif allocate_type == "mutable":
20 - allocate_block = lambda: allocator.allocate_mutable(prev_block=
21 - prev_block)
20 + allocate_block = lambda: allocator.allocate_mutable_block(
21 + prev_block=prev_block)
22 else:
23 raise ValueError()
24
tests/core/block/test_prefix_caching_block.py
+67 -39
@@ -26,11 +26,10 @@ class TestPrefixCachingBlock:
26 token_ids = list(range(num_to_fill))
27 mock_allocator = MagicMock(spec=PrefixCachingBlockAllocator)
28
29 - block_with_prev = PrefixCachingBlock(
30 - prev_block=None,
31 - token_ids=token_ids,
32 - block_size=block_size,
33 - prefix_caching_allocator=mock_allocator)
29 + block_with_prev = PrefixCachingBlock(prev_block=None,
30 + token_ids=token_ids,
31 + block_size=block_size,
32 + allocator=mock_allocator)
33
34 if is_curr_block_full:
35 # Expect hash since block is full.
@@ -71,7 +70,7 @@ class TestPrefixCachingBlock:
70 prev_block=previous_block,
71 token_ids=token_ids,
72 block_size=block_size,
74 - prefix_caching_allocator=mock_allocator,
73 + allocator=mock_allocator,
74 )
75
76 if is_curr_block_full and prev_block_has_hash:
@@ -138,7 +137,7 @@ class TestPrefixCachingBlock:
137 prev_block=prev_block,
138 token_ids=[],
139 block_size=block_size,
141 - prefix_caching_allocator=allocator,
140 + allocator=allocator,
141 )
142
143 tokens_to_append = token_ids[block_number *
@@ -159,11 +158,11 @@ class TestPrefixCachingBlockAllocator:
158 prev_block: Optional[Block],
159 token_ids: List[int]):
160 if allocate_type == "immutable":
162 - allocate_block = lambda: allocator.allocate_immutable(
161 + allocate_block = lambda: allocator.allocate_immutable_block(
162 prev_block=prev_block, token_ids=token_ids)
163 elif allocate_type == "mutable":
165 - allocate_block = lambda: allocator.allocate_mutable(prev_block=
166 - prev_block)
164 + allocate_block = lambda: allocator.allocate_mutable_block(
165 + prev_block=prev_block)
166 else:
167 raise ValueError()
168
@@ -233,12 +232,13 @@ class TestPrefixCachingBlockAllocator:
232
233 # Expect allocation with unseen hash to fail.
234 with pytest.raises(BlockAllocator.NoFreeBlocksError):
236 - allocator.allocate_immutable(prev_block=chain[-1],
237 - token_ids=list(range(block_size)))
235 + allocator.allocate_immutable_block(prev_block=chain[-1],
236 + token_ids=list(
237 + range(block_size)))
238
239 # Expect mutable allocation to fail.
240 with pytest.raises(BlockAllocator.NoFreeBlocksError):
241 - allocator.allocate_mutable(prev_block=chain[-1])
241 + allocator.allocate_mutable_block(prev_block=chain[-1])
242
243 # Expect allocation of exact same chain to pass.
244 second_chain = TestPrefixCachingBlockAllocator.create_immutable_chain(
@@ -270,7 +270,7 @@ class TestPrefixCachingBlockAllocator:
270
271 # Expect mutable allocation to fail.
272 with pytest.raises(BlockAllocator.NoFreeBlocksError):
273 - allocator.allocate_mutable(prev_block=None)
273 + allocator.allocate_mutable_block(prev_block=None)
274
275 block_to_free = chain[-1]
276
@@ -280,11 +280,11 @@ class TestPrefixCachingBlockAllocator:
280 allocator.free(block_to_free)
281 assert block_to_free.block_id is None, i
282
283 - new_block = allocator.allocate_mutable(prev_block=None)
283 + new_block = allocator.allocate_mutable_block(prev_block=None)
284 assert new_block.block_id == block_id, i
285
286 with pytest.raises(BlockAllocator.NoFreeBlocksError):
287 - allocator.allocate_mutable(prev_block=None)
287 + allocator.allocate_mutable_block(prev_block=None)
288
289 block_to_free = new_block
290
@@ -376,7 +376,6 @@ class TestPrefixCachingBlockAllocator:
376
377 # Create token ids that will exhaust all blocks.
378 token_ids = list(range(num_blocks_to_consume * block_size))
379 - blocks = list(range(num_blocks_to_consume))
379
380 first_chain = TestPrefixCachingBlockAllocator.create_immutable_chain(
381 block_size=block_size,
@@ -384,9 +383,6 @@ class TestPrefixCachingBlockAllocator:
383 allocator=allocator,
384 )
385
387 - # mark all blocks in first chain as computed
388 - allocator.mark_blocks_as_computed(blocks)
389 -
386 # After zero_point, second_chain's token_ids would be set -1, which
387 # make it different from here comparing with first_chain
388 zero_point = random.randint(1, len(token_ids) - 1)
@@ -424,15 +420,16 @@ class TestPrefixCachingBlockAllocator:
420 block_size=block_size)
421 token_ids = list(range(block_size))
422
427 - block = allocator.allocate_immutable(prev_block=None,
428 - token_ids=token_ids)
423 + block = allocator.allocate_immutable_block(prev_block=None,
424 + token_ids=token_ids)
425
426 assert allocator._refcounter.get(block.block_id) == 1
431 - m = allocator.allocate_mutable(prev_block=None)
427 + m = allocator.allocate_mutable_block(prev_block=None)
428
429 block_id = m.block_id
430 for i in range(block_size):
431 m.append_token_ids([i])
432 +
433 # After block get promoted to immutable from mutable, if there is
434 # already same content hash block, then it shall be released into
435 # hashless_allocator
@@ -452,48 +449,79 @@ class TestPrefixCachingBlockAllocator:
449
450 all_blocks_list = [i for i in range(num_blocks)]
451 zero_ref = {i: 0 for i in range(num_blocks)}
452 + one_ref = {i: 1 for i in range(num_blocks)}
453 allocator = PrefixCachingBlockAllocator(num_blocks=num_blocks,
454 block_size=block_size)
455 token_ids = list(range(num_blocks * block_size))
456
459 - # now we have num_blocks free blocks in hashless allocator
460 - # with internal tracking list _blocks _cached_blocks and evictor
461 - # empty and block's ref shall be 0
457 + # Verify initial/pre-alloc state
458 +
459 + # Ensure all blocks are free inside hashless allocator
460 assert list(allocator._hashless_allocator._free_block_indices
461 ) == all_blocks_list
464 - assert len(allocator._blocks.keys()) == 0
462 + # Ensure no tracked blocks
463 + assert len(allocator._block_tracker.keys()) == num_blocks
464 + for block_id in range(num_blocks):
465 + assert not allocator._block_tracker[block_id].active
466 + # Ensure no cached blocks
467 assert len(allocator._cached_blocks.values()) == 0
468 + # Ensure no evicted blocks
469 assert len(allocator.evictor.free_table.keys()) == 0
470 + # Ensure 0s ref counts for all blocks
471 assert allocator._refcounter._refcounts == zero_ref
472
473 # Allocate immutable chains with only one block residuled in
474 new_block = []
475 for i in range(num_blocks):
472 - block = allocator.allocate_immutable(
476 + block = allocator.allocate_immutable_block(
477 prev_block=None,
478 token_ids=token_ids[block_size * i:block_size * (i + 1)])
479 new_block.append(block)
480
481 + # Verify post-alloc state
482 +
483 + # Ensure no blocks are free inside hashless allocator
484 + assert (len(allocator._hashless_allocator._free_block_indices) == 0)
485 + # Ensure all blocks are tracked
486 + assert len(allocator._block_tracker.keys()) == num_blocks
487 + for block_id in range(num_blocks):
488 + assert allocator._block_tracker[block_id].active
489 + # Ensure all blocks are cached (all promoted)
490 + assert len(allocator._cached_blocks.values()) == num_blocks
491 + # Ensure no evicted blocks
492 + assert len(allocator.evictor.free_table.keys()) == 0
493 + # Ensure 1s ref counts for all blocks
494 + assert allocator._refcounter._refcounts == one_ref
495 +
496 # Free all blocks, and now all blocks shall be in the evictor
478 - # there shall be no tracking data left in _blocks
497 + # there shall be no tracking data left in _block_tracker
498 # all blocks shall be tracked in _cached_blocks
499 # all blocks' ref shall be zero
500 for block in new_block:
501 allocator.free(block)
502
484 - assert len(allocator._blocks.keys()) == 0
503 + # Verify post-free state
504 +
505 + # Ensure no tracked blocks
506 + assert len(allocator._block_tracker.keys()) == num_blocks
507 + for block_id in range(num_blocks):
508 + assert not allocator._block_tracker[block_id].active
509 + # Ensure no blocks in hashless allocator (all promoted)
510 assert len(allocator._hashless_allocator._free_block_indices) == 0
511 + # Ensure all blocks are cached
512 assert list(allocator._cached_blocks.values()) == all_blocks_list
513 + # Ensure all blocks are inside the evictor
514 assert list(allocator.evictor.free_table.keys()) == all_blocks_list
515 + # Ensure 0s refcounts
516 assert allocator._refcounter._refcounts == zero_ref
517
518 # Allocate a mutable block, and the first block shall be evicted
519 # and set its content hash into None, ref to 1
492 - mutable = allocator.allocate_mutable(prev_block=None)
520 + mutable = allocator.allocate_mutable_block(prev_block=None)
521
522 assert mutable.block_id == 0
523 assert mutable.content_hash is None
496 - assert 0 in allocator._blocks
524 + assert allocator._block_tracker[0].active
525 assert allocator._refcounter.get(0) == 1
526 assert 0 not in allocator._cached_blocks
527 assert 0 not in allocator.evictor
@@ -502,27 +530,27 @@ class TestPrefixCachingBlockAllocator:
530 # hashless allocator
531 allocator.free(mutable)
532
505 - assert len(allocator._blocks.keys()) == 0
533 + assert not allocator._block_tracker[0].active
534 assert allocator._refcounter._refcounts == zero_ref
535 assert 0 not in allocator._cached_blocks
536 assert 0 not in allocator.evictor
537 assert 0 in allocator._hashless_allocator._free_block_indices
538
511 - # when allocate immutable with first block_size tokens, we
539 + # When allocate immutable with first block_size tokens, we
540 # shall get free block from hashless allocator, thus no block left
541 # in hashless
514 - block = allocator.allocate_immutable(prev_block=None,
515 - token_ids=token_ids[:block_size])
542 + block = allocator.allocate_immutable_block(
543 + prev_block=None, token_ids=token_ids[:block_size])
544
545 assert block.block_id == 0
546 assert len(allocator._hashless_allocator._free_block_indices) == 0
519 - assert 0 in allocator._blocks
547 + assert allocator._block_tracker[0].active
548 assert 0 in allocator._cached_blocks.values()
549 assert allocator._refcounter.get(0) == 1
550 assert 0 not in allocator.evictor
551
552 # allocate mutable block again, it shall be popped from evictor
525 - mutable = allocator.allocate_mutable(prev_block=None)
553 + mutable = allocator.allocate_mutable_block(prev_block=None)
554 assert len(allocator._hashless_allocator._free_block_indices) == 0
555 assert mutable.block_id not in allocator.evictor.free_table
556 assert allocator._refcounter.get(mutable.block_id) == 1
@@ -619,7 +647,7 @@ class TestPrefixCachingBlockAllocator:
647 block_token_ids = token_ids[block_number *
648 block_size:(block_number + 1) *
649 block_size]
622 - prev_block = allocator.allocate_immutable(
650 + prev_block = allocator.allocate_immutable_block(
651 prev_block=prev_block, token_ids=block_token_ids)
652 blocks.append(prev_block)
653
tests/spec_decode/test_batch_expansion.py
+4 -4
@@ -90,10 +90,10 @@ def test_create_single_target_seq_group_metadata(k: int):
90
91 assert output.request_id == input_seq_group_metadata.request_id
92 assert len(output.seq_data) == 1
93 - assert output.seq_data[target_seq_id].get_prompt_token_ids(
94 - ) == prompt_tokens
95 - assert output.seq_data[target_seq_id].get_output_token_ids(
96 - ) == prev_output_tokens + token_ids
93 + assert output.seq_data[target_seq_id].get_prompt_token_ids() == tuple(
94 + prompt_tokens)
95 + assert output.seq_data[target_seq_id].get_output_token_ids() == tuple(
96 + prev_output_tokens + token_ids)
97
98 assert len(output.block_tables) == 1
99 assert output.block_tables[
vllm/core/block/block_table.py
+56 -29
@@ -1,5 +1,6 @@
1 from typing import List, Optional
2
3 +from vllm.core.block.common import BlockList
4 from vllm.core.block.interfaces import Block, DeviceAwareBlockAllocator
5 from vllm.utils import Device, cdiv, chunk_list
6
@@ -47,12 +48,10 @@ class BlockTable:
48 self._allocator = block_allocator
49 if _blocks is None:
50 _blocks = []
50 - self._blocks: List[Block] = _blocks
51 + self._blocks: BlockList = BlockList(_blocks)
52
53 self._max_block_sliding_window = max_block_sliding_window
53 - # Use helper method instead of directly calculating, as blocks
54 - # may not be allocated.
55 - self._num_full_slots = len(self._get_all_token_ids())
54 + self._num_full_slots = self._get_num_token_ids()
55
56 @staticmethod
57 def get_num_required_blocks(token_ids: List[int], block_size: int) -> int:
@@ -88,11 +87,18 @@ class BlockTable:
87 """
88 assert not self._is_allocated
89 assert token_ids
91 - self._blocks = self._allocate_blocks_for_token_ids(prev_block=None,
92 - token_ids=token_ids,
93 - device=device)
90 + blocks = self._allocate_blocks_for_token_ids(prev_block=None,
91 + token_ids=token_ids,
92 + device=device)
93 + self.update(blocks)
94 self._num_full_slots = len(token_ids)
95
96 + def update(self, blocks: List[Block]) -> None:
97 + """Resets the table to the newly provided blocks
98 + (with their corresponding block ids)
99 + """
100 + self._blocks.update(blocks)
101 +
102 def append_token_ids(self,
103 token_ids: List[int],
104 num_lookahead_slots: int = 0,
@@ -140,11 +146,11 @@ class BlockTable:
146 num_lookahead_slots)
147
148 # Update the blocks with the new tokens
143 - blocks = self._blocks[self._num_full_slots // self._block_size:]
149 + first_block_idx = self._num_full_slots // self._block_size
150 token_blocks = self._chunk_token_blocks_for_append(token_ids)
151
146 - for block, token_block in zip(blocks, token_blocks):
147 - block.append_token_ids(token_block)
152 + for i, token_block in enumerate(token_blocks):
153 + self._blocks.append_token_ids(first_block_idx + i, token_block)
154
155 self._num_full_slots += len(token_ids)
156
@@ -174,8 +180,8 @@ class BlockTable:
180 for _ in range(blocks_to_allocate):
181 assert len(self._blocks) > 0
182 self._blocks.append(
177 - self._allocator.allocate_mutable(prev_block=self._blocks[-1],
178 - device=device))
183 + self._allocator.allocate_mutable_block(
184 + prev_block=self._blocks[-1], device=device))
185
186 def fork(self) -> "BlockTable":
187 """Creates a new BlockTable instance with a copy of the blocks from the
@@ -209,12 +215,12 @@ class BlockTable:
215 is set to `None`.
216 """
217 assert self._is_allocated
212 - for block in self._blocks:
218 + for block in self.blocks:
219 self._allocator.free(block)
214 - self._blocks = []
220 + self._blocks.reset()
221
222 @property
217 - def physical_block_ids(self) -> List[Optional[int]]:
223 + def physical_block_ids(self) -> List[int]:
224 """Returns a list of physical block indices for the blocks in the
225 BlockTable.
226
@@ -228,7 +234,7 @@ class BlockTable:
234 BlockTable.
235 """
236 assert self._is_allocated
231 - return [block.block_id for block in self._blocks]
237 + return self._blocks.ids()
238
239 def get_unseen_token_ids(self, sequence_token_ids: List[int]) -> List[int]:
240 """Get the number of "unseen" tokens in the sequence.
@@ -253,17 +259,31 @@ class BlockTable:
259 token_ids: List[int],
260 device: Device) -> List[Block]:
261 blocks: List[Block] = []
256 - for block_token_ids in chunk_list(token_ids, self._block_size):
257 - if len(block_token_ids) == self._block_size:
258 - # If the block is full, create an immutable block.
259 - prev_block = self._allocator.allocate_immutable(
260 - prev_block, token_ids=block_token_ids, device=device)
262 +
263 + block_token_ids = []
264 + tail_token_ids = []
265 + for cur_token_ids in chunk_list(token_ids, self._block_size):
266 + if len(cur_token_ids) == self._block_size:
267 + block_token_ids.append(cur_token_ids)
268 else:
262 - # Else, partially fill a mutable block with token ids.
263 - prev_block = self._allocator.allocate_mutable(
264 - prev_block=prev_block, device=device)
265 - prev_block.append_token_ids(block_token_ids)
266 - blocks.append(prev_block)
269 + tail_token_ids.append(cur_token_ids)
270 +
271 + if block_token_ids:
272 + blocks.extend(
273 + self._allocator.allocate_immutable_blocks(
274 + prev_block, block_token_ids=block_token_ids,
275 + device=device))
276 + prev_block = blocks[-1]
277 +
278 + if tail_token_ids:
279 + assert len(tail_token_ids) == 1
280 + cur_token_ids = tail_token_ids[0]
281 +
282 + block = self._allocator.allocate_mutable_block(
283 + prev_block=prev_block, device=device)
284 + block.append_token_ids(cur_token_ids)
285 +
286 + blocks.append(block)
287
288 return blocks
289
@@ -274,18 +294,25 @@ class BlockTable:
294 if not self._is_allocated:
295 return token_ids
296
277 - for block in self._blocks:
297 + for block in self.blocks:
298 token_ids.extend(block.token_ids)
299
300 return token_ids
301
302 + def _get_num_token_ids(self) -> int:
303 + res = 0
304 + for block in self.blocks:
305 + res += len(block.token_ids)
306 +
307 + return res
308 +
309 @property
310 def _is_allocated(self) -> bool:
311 return len(self._blocks) > 0
312
313 @property
287 - def blocks(self) -> Optional[List[Block]]:
288 - return self._blocks
314 + def blocks(self) -> List[Block]:
315 + return self._blocks.list()
316
317 @property
318 def _num_empty_slots(self) -> int:
vllm/core/block/common.py
+154 -44
@@ -1,4 +1,5 @@
1 -from typing import Dict, Iterable, List, Optional, Protocol, Tuple
1 +from collections import deque
2 +from typing import Deque, Dict, Iterable, List, Optional, Protocol, Tuple
3
4 from vllm.core.block.interfaces import Block, BlockAllocator
5
@@ -95,64 +96,40 @@ class CopyOnWriteTracker:
96
97 The CopyOnWriteTracker class maintains a mapping of source block indices to
98 their corresponding copy-on-write destination block indices. It works in
98 - conjunction with a RefCounter and a BlockAllocator to handle reference
99 - counting and block allocation.
99 + conjunction with a RefCounter.
100
101 Args:
102 refcounter (RefCounter): The reference counter used to track block
103 reference counts.
104 - allocator (BlockAllocator): The block allocator used to allocate and
105 - free blocks.
104 """
105
108 - def __init__(
109 - self,
110 - refcounter: RefCounterProtocol,
111 - allocator: BlockAllocator,
112 - ):
106 + def __init__(self, refcounter: RefCounterProtocol):
107 self._copy_on_writes: List[Tuple[BlockId, BlockId]] = []
108 self._refcounter = refcounter
115 - self._allocator = allocator
116 -
117 - def cow_block_if_not_appendable(self, block: Block) -> Optional[BlockId]:
118 - """Performs a copy-on-write operation on the given block if it is not
119 - appendable.
120 -
121 - This method checks the reference count of the given block. If the
122 - reference count is greater than 1, indicating that the block is shared,
123 - a copy-on-write operation is performed. The original block is freed,
124 - and a new block is allocated with the same content. The new block index
125 - is returned.
126 -
127 - Args:
128 - block (Block): The block to check for copy-on-write.
109
130 - Returns:
131 - Optional[BlockId]: The block index of the new block if a copy-on
132 - -write operation was performed, or the original block index if
133 - no copy-on-write was necessary.
110 + def is_appendable(self, block: Block) -> bool:
111 + """Checks if the block is shared or not. If shared, then it cannot
112 + be appended and needs to be duplicated via copy-on-write
113 """
114 block_id = block.block_id
115 if block_id is None:
137 - return block_id
116 + return True
117
118 refcount = self._refcounter.get(block_id)
140 - assert refcount != 0
141 - if refcount > 1:
142 - src_block_id = block_id
143 - # Decrement refcount of the old block.
144 - self._allocator.free(block)
145 -
146 - # Allocate a fresh new block.
147 - block_id = self._allocator.allocate_mutable(
148 - prev_block=block.prev_block).block_id
119 + return refcount <= 1
120
150 - # Track src/dst copy.
151 - assert src_block_id is not None
152 - assert block_id is not None
153 - self._copy_on_writes.append((src_block_id, block_id))
154 -
155 - return block_id
121 + def record_cow(self, src_block_id: Optional[BlockId],
122 + trg_block_id: Optional[BlockId]) -> None:
123 + """Records a copy-on-write operation from source to target block id
124 + Args:
125 + src_block_id (BlockId): The source block id from which to copy
126 + the data
127 + trg_block_id (BlockId): The target block id to which the data
128 + is copied
129 + """
130 + assert src_block_id is not None
131 + assert trg_block_id is not None
132 + self._copy_on_writes.append((src_block_id, trg_block_id))
133
134 def clear_cows(self) -> List[Tuple[BlockId, BlockId]]:
135 """Clears the copy-on-write tracking information and returns the current
@@ -172,6 +149,139 @@ class CopyOnWriteTracker:
149 return cows
150
151
152 +class BlockPool:
153 + """Used to pre-allocate block objects, in order to avoid excessive python
154 + object allocations/deallocations.
155 + The pool starts from "pool_size" objects and will increase to more objects
156 + if necessary
157 +
158 + Note that multiple block objects may point to the same physical block id,
159 + which is why this pool is needed, so that it will be easier to support
160 + prefix caching and more complicated sharing of physical blocks.
161 + """
162 +
163 + def __init__(self, block_size: int, create_block: Block.Factory,
164 + allocator: BlockAllocator, pool_size: int):
165 + self._block_size = block_size
166 + self._create_block = create_block
167 + self._allocator = allocator
168 + self._pool_size = pool_size
169 + assert self._pool_size >= 0
170 +
171 + self._free_ids: Deque[int] = deque(range(self._pool_size))
172 + self._pool = []
173 + for i in range(self._pool_size):
174 + self._pool.append(
175 + self._create_block(prev_block=None,
176 + token_ids=[],
177 + block_size=self._block_size,
178 + allocator=self._allocator,
179 + block_id=None))
180 +
181 + def increase_pool(self):
182 + """Doubles the internal pool size
183 + """
184 + cur_pool_size = self._pool_size
185 + new_pool_size = cur_pool_size * 2
186 + self._pool_size = new_pool_size
187 +
188 + self._free_ids += deque(range(cur_pool_size, new_pool_size))
189 +
190 + for i in range(cur_pool_size, new_pool_size):
191 + self._pool.append(
192 + self._create_block(prev_block=None,
193 + token_ids=[],
194 + block_size=self._block_size,
195 + allocator=self._allocator,
196 + block_id=None))
197 +
198 + def init_block(self, prev_block: Optional[Block], token_ids: List[int],
199 + block_size: int, physical_block_id: Optional[int]) -> Block:
200 + if len(self._free_ids) == 0:
201 + self.increase_pool()
202 + assert len(self._free_ids) > 0
203 +
204 + pool_id = self._free_ids.popleft()
205 +
206 + block = self._pool[pool_id]
207 + block.__init__( # type: ignore[misc]
208 + prev_block=prev_block,
209 + token_ids=token_ids,
210 + block_size=block_size,
211 + allocator=block._allocator, # type: ignore[attr-defined]
212 + block_id=physical_block_id)
213 + block.pool_id = pool_id # type: ignore[attr-defined]
214 + return block
215 +
216 + def free_block(self, block: Block) -> None:
217 + self._free_ids.appendleft(block.pool_id) # type: ignore[attr-defined]
218 +
219 +
220 +class BlockList:
221 + """This class is an optimization to allow fast-access to physical
222 + block ids. It maintains a block id list that is updated with the
223 + block list and this avoids the need to reconstruct the block id
224 + list on every iteration of the block manager
225 + """
226 +
227 + def __init__(self, blocks: List[Block]):
228 + self._blocks: List[Block] = []
229 + self._block_ids: List[int] = []
230 +
231 + self.update(blocks)
232 +
233 + def _add_block_id(self, block_id: Optional[BlockId]) -> None:
234 + assert block_id is not None
235 + self._block_ids.append(block_id)
236 +
237 + def _update_block_id(self, block_index: int,
238 + new_block_id: Optional[BlockId]) -> None:
239 + assert new_block_id is not None
240 + self._block_ids[block_index] = new_block_id
241 +
242 + def update(self, blocks: List[Block]):
243 + self._blocks = blocks
244 +
245 + # Cache block ids for fast query
246 + self._block_ids = []
247 + for block in self._blocks:
248 + self._add_block_id(block.block_id)
249 +
250 + def append_token_ids(self, block_index: int, token_ids: List[int]) -> None:
251 + block = self._blocks[block_index]
252 + prev_block_id = block.block_id
253 +
254 + block.append_token_ids(token_ids)
255 +
256 + # CoW or promotion may update the internal block_id
257 + if prev_block_id != block.block_id:
258 + self._update_block_id(block_index, block.block_id)
259 +
260 + def append(self, new_block: Block):
261 + self._blocks.append(new_block)
262 + self._add_block_id(new_block.block_id)
263 +
264 + def __len__(self) -> int:
265 + return len(self._blocks)
266 +
267 + def __getitem__(self, block_index: int) -> Block:
268 + return self._blocks[block_index]
269 +
270 + def __setitem__(self, block_index: int, new_block: Block) -> None:
271 + self._blocks[block_index] = new_block
272 + self._update_block_id(block_index, new_block.block_id)
273 +
274 + def reset(self):
275 + self._blocks = []
276 + self._block_ids = []
277 +
278 + def list(self) -> List[Block]:
279 + return self._blocks
280 +
281 + def ids(self) -> List[int]:
282 + return self._block_ids
283 +
284 +
285 def get_all_blocks_recursively(last_block: Block) -> List[Block]:
286 """Retrieves all the blocks in a sequence starting from the last block.
287
vllm/core/block/cpu_gpu_block_allocator.py
+56 -28
@@ -113,11 +113,11 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
113 def allocate_or_get_null_block(self) -> Block:
114 if self._null_block is None:
115 self._null_block = NullBlock(
116 - self.allocate_mutable(None, Device.GPU))
116 + self.allocate_mutable_block(None, Device.GPU))
117 return self._null_block
118
119 - def allocate_mutable(self, prev_block: Optional[Block],
120 - device: Device) -> Block:
119 + def allocate_mutable_block(self, prev_block: Optional[Block],
120 + device: Device) -> Block:
121 """Allocates a new mutable block on the specified device.
122
123 Args:
@@ -128,10 +128,31 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
128 Returns:
129 Block: The newly allocated mutable block.
130 """
131 - return self._allocators[device].allocate_mutable(prev_block)
131 + return self._allocators[device].allocate_mutable_block(prev_block)
132
133 - def allocate_immutable(self, prev_block: Optional[Block],
134 - token_ids: List[int], device: Device) -> Block:
133 + def allocate_immutable_blocks(self, prev_block: Optional[Block],
134 + block_token_ids: List[List[int]],
135 + device: Optional[Device]) -> List[Block]:
136 + """Allocates a new group of immutable blocks with the provided block
137 + token IDs on the specified device.
138 +
139 + Args:
140 + prev_block (Optional[Block]): The previous block in the sequence.
141 + Used for prefix hashing.
142 + block_token_ids (List[int]): The list of block token IDs to be
143 + stored in the new blocks.
144 + device (Device): The device on which to allocate the new block.
145 +
146 + Returns:
147 + List[Block]: The newly allocated list of immutable blocks
148 + containing the provided block token IDs.
149 + """
150 + return self._allocators[device].allocate_immutable_blocks(
151 + prev_block, block_token_ids)
152 +
153 + def allocate_immutable_block(self, prev_block: Optional[Block],
154 + token_ids: List[int],
155 + device: Device) -> Block:
156 """Allocates a new immutable block with the provided token IDs on the
157 specified device.
158
@@ -146,7 +167,7 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
167 Block: The newly allocated immutable block containing the provided
168 token IDs.
169 """
149 - return self._allocators[device].allocate_immutable(
170 + return self._allocators[device].allocate_immutable_block(
171 prev_block, token_ids)
172
173 def free(self, block: Block) -> None:
@@ -161,7 +182,7 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
182 block_id = block.block_id
183 assert block_id is not None
184 allocator = self._block_ids_to_allocator[block_id]
164 - return allocator.free(block)
185 + allocator.free(block)
186
187 def fork(self, last_block: Block) -> List[Block]:
188 """Creates a new sequence of blocks that shares the same underlying
@@ -210,8 +231,8 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
231 """
232 return self._allocators[device].get_physical_block_id(absolute_id)
233
213 - def swap(self, blocks: List[Block], source_device: Device,
214 - dest_device: Device) -> Dict[int, int]:
234 + def swap(self, blocks: List[Block], src_device: Device,
235 + dst_device: Device) -> Dict[int, int]:
236 """Execute the swap for the given blocks from source_device
237 on to dest_device, save the current swap mapping and append
238 them to the accumulated `self._swap_mapping` for each
@@ -219,23 +240,23 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
240
241 Args:
242 blocks: List of blocks to be swapped.
222 - source_device (Device): Device to swap the 'blocks' from.
223 - dest_device (Device): Device to swap the 'blocks' to.
243 + src_device (Device): Device to swap the 'blocks' from.
244 + dst_device (Device): Device to swap the 'blocks' to.
245
246 Returns:
247 Dict[int, int]: Swap mapping from source_device
248 on to dest_device.
249 """
229 - source_block_ids = [block.block_id for block in blocks]
230 - self._allocators[source_device].swap_out(blocks)
231 - self._allocators[dest_device].swap_in(blocks)
232 - dest_block_ids = [block.block_id for block in blocks]
250 + src_block_ids = [block.block_id for block in blocks]
251 + self._allocators[src_device].swap_out(blocks)
252 + self._allocators[dst_device].swap_in(blocks)
253 + dst_block_ids = [block.block_id for block in blocks]
254
255 current_swap_mapping: Dict[int, int] = {}
235 - for src, dest in zip(source_block_ids, dest_block_ids):
236 - if src is not None and dest is not None:
237 - self._swap_mapping[src] = dest
238 - current_swap_mapping[src] = dest
256 + for src_block_id, dst_block_id in zip(src_block_ids, dst_block_ids):
257 + if src_block_id is not None and dst_block_id is not None:
258 + self._swap_mapping[src_block_id] = dst_block_id
259 + current_swap_mapping[src_block_id] = dst_block_id
260 return current_swap_mapping
261
262 def get_num_blocks_touched(self,
@@ -283,23 +304,25 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
304 device = Device.GPU
305 return self._allocators[device].mark_blocks_as_computed(block_ids)
306
307 + def get_computed_block_ids(self, prev_computed_block_ids: List[int],
308 + block_ids: List[int],
309 + skip_last_block_id: bool) -> List[int]:
310 + # Prefix caching only supported on GPU.
311 + device = Device.GPU
312 + return self._allocators[device].get_computed_block_ids(
313 + prev_computed_block_ids, block_ids, skip_last_block_id)
314 +
315 def get_common_computed_block_ids(
287 - self, seq_block_ids: List[List[int]]) -> List[int]:
316 + self, computed_seq_block_ids: List[List[int]]) -> List[int]:
317 # Prefix caching only supported on GPU.
318 device = Device.GPU
319 return self._allocators[device].get_common_computed_block_ids(
291 - seq_block_ids)
320 + computed_seq_block_ids)
321
322 @property
323 def all_block_ids(self) -> FrozenSet[int]:
324 return frozenset(self._block_ids_to_allocator.keys())
325
297 - def promote_to_immutable_block(self, block: Block) -> BlockId:
298 - raise NotImplementedError
299 -
300 - def cow_block_if_not_appendable(self, block: Block) -> Optional[BlockId]:
301 - raise NotImplementedError
302 -
326 def get_and_reset_swaps(self) -> List[Tuple[int, int]]:
327 """Returns and clears the mapping of source to destination block IDs.
328 Will be called after every swapping operations for now, and after every
@@ -341,6 +364,11 @@ class NullBlock(Block):
364 def token_ids(self) -> List[BlockId]:
365 return self._proxy.token_ids
366
367 + @property
368 + def num_tokens_total(self) -> int:
369 + raise NotImplementedError(
370 + "num_tokens_total is not used for null block")
371 +
372 @property
373 def num_empty_slots(self) -> BlockId:
374 return self._proxy.num_empty_slots
vllm/core/block/interfaces.py
+44 -12
@@ -28,6 +28,13 @@ class Block(ABC):
28 def token_ids(self) -> List[int]:
29 pass
30
31 + @property
32 + @abstractmethod
33 + def num_tokens_total(self) -> int:
34 + """The number of tokens till the current block (inclusive)
35 + """
36 + pass
37 +
38 @property
39 @abstractmethod
40 def num_empty_slots(self) -> int:
@@ -92,12 +99,18 @@ class Block(ABC):
99 class BlockAllocator(ABC):
100
101 @abstractmethod
95 - def allocate_mutable(self, prev_block: Optional[Block]) -> Block:
102 + def allocate_mutable_block(self, prev_block: Optional[Block]) -> Block:
103 pass
104
105 @abstractmethod
99 - def allocate_immutable(self, prev_block: Optional[Block],
100 - token_ids: List[int]) -> Block:
106 + def allocate_immutable_block(self, prev_block: Optional[Block],
107 + token_ids: List[int]) -> Block:
108 + pass
109 +
110 + @abstractmethod
111 + def allocate_immutable_blocks(
112 + self, prev_block: Optional[Block],
113 + block_token_ids: List[List[int]]) -> List[Block]:
114 pass
115
116 @abstractmethod
@@ -146,13 +159,19 @@ class BlockAllocator(ABC):
159 def mark_blocks_as_computed(self, block_ids: List[int]) -> None:
160 pass
161
162 + @abstractmethod
163 + def get_computed_block_ids(self, prev_computed_block_ids: List[int],
164 + block_ids: List[int],
165 + skip_last_block_id: bool) -> List[int]:
166 + pass
167 +
168 @abstractmethod
169 def get_common_computed_block_ids(
151 - self, seq_block_ids: List[List[int]]) -> List[int]:
170 + self, computed_seq_block_ids: List[List[int]]) -> List[int]:
171 pass
172
173 @abstractmethod
155 - def cow_block_if_not_appendable(self, block: Block) -> Optional["BlockId"]:
174 + def cow_block_if_not_appendable(self, block: Block) -> BlockId:
175 """NOTE: This should not be used besides Block"""
176 pass
177
@@ -174,13 +193,20 @@ class BlockAllocator(ABC):
193 class DeviceAwareBlockAllocator(ABC):
194
195 @abstractmethod
177 - def allocate_mutable(self, prev_block: Optional[Block],
178 - device: Device) -> Block:
196 + def allocate_mutable_block(self, prev_block: Optional[Block],
197 + device: Device) -> Block:
198 + pass
199 +
200 + @abstractmethod
201 + def allocate_immutable_block(self, prev_block: Optional[Block],
202 + token_ids: List[int],
203 + device: Device) -> Block:
204 pass
205
206 @abstractmethod
182 - def allocate_immutable(self, prev_block: Optional[Block],
183 - token_ids: List[int], device: Device) -> Block:
207 + def allocate_immutable_blocks(self, prev_block: Optional[Block],
208 + block_token_ids: List[List[int]],
209 + device: Device) -> List[Block]:
210 pass
211
212 @abstractmethod
@@ -217,9 +243,15 @@ class DeviceAwareBlockAllocator(ABC):
243 def mark_blocks_as_computed(self, block_ids: List[int]) -> None:
244 pass
245
246 + @abstractmethod
247 + def get_computed_block_ids(self, prev_computed_block_ids: List[int],
248 + block_ids: List[int],
249 + skip_last_block_id: bool) -> List[int]:
250 + pass
251 +
252 @abstractmethod
253 def get_common_computed_block_ids(
222 - self, seq_block_ids: List[List[int]]) -> List[int]:
254 + self, computed_seq_block_ids: List[List[int]]) -> List[int]:
255 pass
256
257 @abstractmethod
@@ -230,8 +262,8 @@ class DeviceAwareBlockAllocator(ABC):
262 pass
263
264 @abstractmethod
233 - def swap(self, blocks: List[Block], source_device: Device,
234 - dest_device: Device) -> Dict[int, int]:
265 + def swap(self, blocks: List[Block], src_device: Device,
266 + dst_device: Device) -> Dict[int, int]:
267 pass
268
269 @abstractmethod
vllm/core/block/naive_block.py
+149 -67
@@ -1,6 +1,7 @@
1 -from typing import FrozenSet, Iterable, List, Optional, Set, Tuple
1 +from collections import deque
2 +from typing import Deque, FrozenSet, Iterable, List, Optional, Tuple
3
3 -from vllm.core.block.common import (CopyOnWriteTracker, RefCounter,
4 +from vllm.core.block.common import (BlockPool, CopyOnWriteTracker, RefCounter,
5 get_all_blocks_recursively)
6 from vllm.core.block.interfaces import Block, BlockAllocator, BlockId, Device
7 from vllm.utils import cdiv
@@ -31,28 +32,39 @@ class NaiveBlockAllocator(BlockAllocator):
32 num_blocks: int,
33 block_size: int,
34 block_ids: Optional[Iterable[int]] = None,
35 + block_pool: Optional[BlockPool] = None,
36 ):
37 if block_ids is None:
38 block_ids = range(num_blocks)
39
38 - self._free_block_indices: Set[BlockId] = set(block_ids)
40 + self._free_block_indices: Deque[BlockId] = deque(block_ids)
41 self._all_block_indices = frozenset(block_ids)
42 assert len(self._all_block_indices) == num_blocks
43
44 self._refcounter = RefCounter(
45 all_block_indices=self._free_block_indices)
44 - self._create_block = create_block
46 self._block_size = block_size
47
48 self._cow_tracker = CopyOnWriteTracker(
48 - refcounter=self._refcounter.as_readonly(),
49 - allocator=self,
50 - )
51 -
52 - def allocate_immutable(self,
53 - prev_block: Optional[Block],
54 - token_ids: List[int],
55 - device: Optional[Device] = None) -> Block:
49 + refcounter=self._refcounter.as_readonly())
50 +
51 + if block_pool is None:
52 + extra_factor = 4
53 + # Pre-allocate "num_blocks * extra_factor" block objects.
54 + # The "* extra_factor" is a buffer to allow more block objects
55 + # than physical blocks
56 + self._block_pool = BlockPool(self._block_size, create_block, self,
57 + num_blocks * extra_factor)
58 + else:
59 + # In this case, the block pool is provided by the caller,
60 + # which means that there is most likely a need to share
61 + # a block pool between allocators
62 + self._block_pool = block_pool
63 +
64 + def allocate_immutable_block(self,
65 + prev_block: Optional[Block],
66 + token_ids: List[int],
67 + device: Optional[Device] = None) -> Block:
68 """Allocates a new immutable block with the given token IDs, linked to
69 the previous block.
70
@@ -66,13 +78,36 @@ class NaiveBlockAllocator(BlockAllocator):
78 Block: The newly allocated immutable block.
79 """
80 assert device is None
69 - block = self.allocate_mutable(prev_block=prev_block)
81 + block = self.allocate_mutable_block(prev_block=prev_block)
82 block.append_token_ids(token_ids)
83 return block
84
73 - def allocate_mutable(self,
74 - prev_block: Optional[Block],
75 - device: Optional[Device] = None) -> Block:
85 + def allocate_immutable_blocks(
86 + self,
87 + prev_block: Optional[Block],
88 + block_token_ids: List[List[int]],
89 + device: Optional[Device] = None) -> List[Block]:
90 + assert device is None
91 + num_blocks = len(block_token_ids)
92 +
93 + block_ids = []
94 + for i in range(num_blocks):
95 + block_ids.append(self._allocate_block_id())
96 +
97 + blocks = []
98 + for i in range(num_blocks):
99 + prev_block = self._block_pool.init_block(
100 + prev_block=prev_block,
101 + token_ids=block_token_ids[i],
102 + block_size=self._block_size,
103 + physical_block_id=block_ids[i])
104 + blocks.append(prev_block)
105 +
106 + return blocks
107 +
108 + def allocate_mutable_block(self,
109 + prev_block: Optional[Block],
110 + device: Optional[Device] = None) -> Block:
111 """Allocates a new mutable block, linked to the previous block.
112
113 Args:
@@ -84,20 +119,39 @@ class NaiveBlockAllocator(BlockAllocator):
119 Block: The newly allocated mutable block.
120 """
121 assert device is None
87 - block_id = self._allocate_new_block_id()
88 - return self._create_block(
89 - prev_block=prev_block,
90 - token_ids=[],
91 - block_id=block_id,
92 - block_size=self._block_size,
93 - allocator=self,
94 - )
95 -
96 - def free(self, block: Block) -> None:
97 - assert block.block_id is not None
98 - self._free_block_id(block.block_id)
122 + block_id = self._allocate_block_id()
123 + block = self._block_pool.init_block(prev_block=prev_block,
124 + token_ids=[],
125 + block_size=self._block_size,
126 + physical_block_id=block_id)
127 + return block
128 +
129 + def _allocate_block_id(self) -> BlockId:
130 + if not self._free_block_indices:
131 + raise BlockAllocator.NoFreeBlocksError()
132 +
133 + block_id = self._free_block_indices.popleft()
134 + self._refcounter.incr(block_id)
135 + return block_id
136 +
137 + def _free_block_id(self, block: Block) -> None:
138 + block_id = block.block_id
139 + assert block_id is not None
140 +
141 + refcount = self._refcounter.decr(block_id)
142 + if refcount == 0:
143 + self._free_block_indices.appendleft(block_id)
144 +
145 block.block_id = None
146
147 + def free(self, block: Block, keep_block_object: bool = False) -> None:
148 + # Release the physical block id
149 + self._free_block_id(block)
150 +
151 + # Release the block object
152 + if not keep_block_object:
153 + self._block_pool.free_block(block)
154 +
155 def fork(self, last_block: Block) -> List[Block]:
156 """Creates a new sequence of blocks that shares the same underlying
157 memory as the original sequence.
@@ -120,14 +174,13 @@ class NaiveBlockAllocator(BlockAllocator):
174 refcount = self._refcounter.incr(block.block_id)
175 assert refcount != 1, "can't fork free'd block"
176
123 - forked_blocks.append(
124 - self._create_block(
125 - prev_block=prev_block,
126 - token_ids=block.token_ids,
127 - block_id=block.block_id,
128 - block_size=self._block_size,
129 - allocator=self,
130 - ))
177 + forked_block = self._block_pool.init_block(
178 + prev_block=prev_block,
179 + token_ids=block.token_ids,
180 + block_size=self._block_size,
181 + physical_block_id=block.block_id)
182 +
183 + forked_blocks.append(forked_block)
184 prev_block = forked_blocks[-1]
185
186 return forked_blocks
@@ -138,20 +191,6 @@ class NaiveBlockAllocator(BlockAllocator):
191 def get_num_total_blocks(self) -> int:
192 return len(self._all_block_indices)
193
141 - def _allocate_new_block_id(self) -> BlockId:
142 - if not self._free_block_indices:
143 - raise BlockAllocator.NoFreeBlocksError()
144 -
145 - block_id = next(iter(self._free_block_indices))
146 - self._refcounter.incr(block_id)
147 - self._free_block_indices.remove(block_id)
148 - return block_id
149 -
150 - def _free_block_id(self, block_id: BlockId) -> None:
151 - refcount = self._refcounter.decr(block_id)
152 - if refcount == 0:
153 - self._free_block_indices.add(block_id)
154 -
194 def get_physical_block_id(self, absolute_id: int) -> int:
195 """Returns the zero-offset block id on certain block allocator
196 given the absolute block id.
@@ -173,7 +212,7 @@ class NaiveBlockAllocator(BlockAllocator):
212 def all_block_ids(self) -> FrozenSet[int]:
213 return self._all_block_indices
214
176 - def cow_block_if_not_appendable(self, block: Block) -> Optional[BlockId]:
215 + def cow_block_if_not_appendable(self, block: Block) -> BlockId:
216 """Performs a copy-on-write operation on the given block if it is not
217 appendable.
218
@@ -181,11 +220,22 @@ class NaiveBlockAllocator(BlockAllocator):
220 block (Block): The block to check for copy-on-write.
221
222 Returns:
184 - Optional[BlockId]: The block index of the new block if a copy-on
185 - -write operation was performed, or the original block index if
223 + BlockId: The block index of the new block if a copy-on-write
224 + operation was performed, or the original block index if
225 no copy-on-write was necessary.
226 """
188 - return self._cow_tracker.cow_block_if_not_appendable(block)
227 + src_block_id = block.block_id
228 + assert src_block_id is not None
229 +
230 + if self._cow_tracker.is_appendable(block):
231 + return src_block_id
232 +
233 + self._free_block_id(block)
234 + trg_block_id = self._allocate_block_id()
235 +
236 + self._cow_tracker.record_cow(src_block_id, trg_block_id)
237 +
238 + return trg_block_id
239
240 def clear_copy_on_writes(self) -> List[Tuple[BlockId, BlockId]]:
241 """Returns the copy-on-write source->destination mapping and clears it.
@@ -213,8 +263,15 @@ class NaiveBlockAllocator(BlockAllocator):
263 """
264 pass
265
266 + def get_computed_block_ids(self, prev_computed_block_ids: List[int],
267 + block_ids: List[int],
268 + skip_last_block_id: bool) -> List[int]:
269 + """No prefix caching here => return empty list
270 + """
271 + return []
272 +
273 def get_common_computed_block_ids(
217 - self, seq_block_ids: List[List[int]]) -> List[int]:
274 + self, computed_seq_block_ids: List[List[int]]) -> List[int]:
275 """Determine blocks that can be skipped in prefill.
276
277 Since the naive allocator does not support prefix caching, always return
@@ -223,7 +280,7 @@ class NaiveBlockAllocator(BlockAllocator):
280 return []
281
282 def promote_to_immutable_block(self, block: Block) -> BlockId:
226 - raise NotImplementedError
283 + raise NotImplementedError("There is no promotion for naive blocks")
284
285 def get_num_blocks_touched(self,
286 blocks: List[Block],
@@ -263,17 +320,27 @@ class NaiveBlockAllocator(BlockAllocator):
320
321 def swap_out(self, blocks: List[Block]) -> None:
322 for block in blocks:
266 - self.free(block)
323 + self._free_block_id(block)
324
325 def swap_in(self, blocks: List[Block]) -> None:
326 for block in blocks:
327 + # Here we allocate either immutable or mutable block and then
328 + # extract its block_id. Note that the block object is released
329 + # and the block_id is assigned to "block" to allow reusing the
330 + # existing "block" object
331 if block.is_full:
271 - alloc = self.allocate_immutable(block.prev_block,
272 - block.token_ids)
332 + tmp_block = self.allocate_immutable_block(
333 + prev_block=block.prev_block, token_ids=block.token_ids)
334 else:
274 - alloc = self.allocate_mutable(block.prev_block)
275 - alloc.append_token_ids(block.token_ids)
276 - block.block_id = alloc.block_id
335 + tmp_block = self.allocate_mutable_block(
336 + prev_block=block.prev_block)
337 + tmp_block.append_token_ids(block.token_ids)
338 +
339 + block_id = tmp_block.block_id
340 + tmp_block.block_id = None
341 + self._block_pool.free_block(tmp_block)
342 +
343 + block.block_id = block_id # Assign block_id
344
345
346 class NaiveBlock(Block):
@@ -315,11 +382,12 @@ class NaiveBlock(Block):
382 self._append_token_ids_no_cow(token_ids)
383
384 def append_token_ids(self, token_ids: List[int]) -> None:
318 - """Appends the given token IDs to the block, instructing the allocator
319 - to perform a copy-on-write if necessary.
385 + """Appends the given token IDs to the block and performs a
386 + copy-on-write if necessary.
387
388 Args:
322 - token_ids (List[int]): The token IDs to be appended to the block.
389 + token_ids (Optional[List[int]]): The token IDs to be appended
390 + to the block.
391 """
392 self._append_token_ids_no_cow(token_ids)
393
@@ -328,7 +396,16 @@ class NaiveBlock(Block):
396 self._cow_target))
397
398 def _append_token_ids_no_cow(self, token_ids: List[int]) -> None:
331 - assert self.num_empty_slots >= len(token_ids)
399 + """Appends the given token IDs to the block
400 +
401 + Args:
402 + token_ids (List[int]): The token IDs to be appended to the block.
403 + """
404 + if len(token_ids) == 0:
405 + return
406 +
407 + assert len(token_ids) <= self.num_empty_slots
408 +
409 self._token_ids.extend(token_ids)
410
411 @property
@@ -361,12 +438,17 @@ class NaiveBlock(Block):
438
439 @property
440 def num_empty_slots(self) -> int:
364 - return self._block_size - len(self._token_ids)
441 + return self._block_size - len(self.token_ids)
442
443 @property
444 def token_ids(self) -> List[int]:
445 return self._token_ids
446
447 + @property
448 + def num_tokens_total(self) -> int:
449 + raise NotImplementedError(
450 + "num_tokens_total is not used for naive block")
451 +
452 @property
453 def block_size(self) -> int:
454 return self._block_size
vllm/core/block/prefix_caching_block.py
+479 -214
@@ -1,13 +1,13 @@
1 """Token blocks."""
2
3 -from itertools import takewhile
3 from os.path import commonprefix
4 from typing import Dict, FrozenSet, Iterable, List, Optional, Tuple
5
6 from vllm.core.block.common import (CopyOnWriteTracker,
7 get_all_blocks_recursively)
8 from vllm.core.block.interfaces import Block, BlockAllocator, BlockId, Device
10 -from vllm.core.block.naive_block import NaiveBlock, NaiveBlockAllocator
9 +from vllm.core.block.naive_block import (BlockPool, NaiveBlock,
10 + NaiveBlockAllocator)
11 from vllm.core.evictor_v2 import EvictionPolicy, Evictor, make_evictor
12 from vllm.utils import cdiv
13
@@ -19,6 +19,30 @@ PrefixHash = int
19 _DEFAULT_LAST_ACCESSED_TIME = -1
20
21
22 +class BlockTracker:
23 + """Used to track the status of a block inside the prefix caching allocator
24 + """
25 + __slots__ = ("active", "last_accessed", "computed")
26 +
27 + def reset(self):
28 + self.last_accessed: float = _DEFAULT_LAST_ACCESSED_TIME
29 + self.computed: bool = False
30 +
31 + def __init__(self):
32 + self.active: bool = False
33 + self.reset()
34 +
35 + def enable(self):
36 + assert not self.active
37 + self.active = True
38 + self.reset()
39 +
40 + def disable(self):
41 + assert self.active
42 + self.active = False
43 + self.reset()
44 +
45 +
46 class PrefixCachingBlockAllocator(BlockAllocator):
47 """A block allocator that implements prefix caching.
48
@@ -41,12 +65,26 @@ class PrefixCachingBlockAllocator(BlockAllocator):
65 block_ids: Optional[Iterable[int]] = None,
66 eviction_policy: EvictionPolicy = EvictionPolicy.LRU,
67 ):
68 + if block_ids is None:
69 + block_ids = range(num_blocks)
70 +
71 + self._block_size = block_size
72 +
73 # A mapping of prefix hash to block index. All blocks which have a
74 # prefix hash will be in this dict, even if they have refcount 0.
75 self._cached_blocks: Dict[PrefixHash, BlockId] = {}
76
48 - # A mapping of blockId to Block to track those cached blocks
49 - self._blocks: Dict[BlockId, Block] = {}
77 + # Used to track status of each physical block id
78 + self._block_tracker: Dict[BlockId, BlockTracker] = {}
79 + for block_id in block_ids:
80 + self._block_tracker[block_id] = BlockTracker()
81 +
82 + # Pre-allocate "num_blocks * extra_factor" block objects.
83 + # The "* extra_factor" is a buffer to allow more block objects
84 + # than physical blocks
85 + extra_factor = 4
86 + self._block_pool = BlockPool(self._block_size, self._create_block,
87 + self, num_blocks * extra_factor)
88
89 # An allocator for blocks that do not have prefix hashes.
90 self._hashless_allocator = NaiveBlockAllocator(
@@ -54,10 +92,9 @@ class PrefixCachingBlockAllocator(BlockAllocator):
92 num_blocks=num_blocks,
93 block_size=block_size,
94 block_ids=block_ids,
95 + block_pool=self._block_pool, # Share block pool here
96 )
97
59 - self._block_size = block_size
60 -
98 # Evitor used to maintain how we want to handle those computed blocks
99 # if we find memory pressure is high.
100 self.evictor: Evictor = make_evictor(eviction_policy)
@@ -68,9 +105,7 @@ class PrefixCachingBlockAllocator(BlockAllocator):
105 self._refcounter = self._hashless_allocator.refcounter
106
107 self._cow_tracker = CopyOnWriteTracker(
71 - refcounter=self._refcounter.as_readonly(),
72 - allocator=self,
73 - )
108 + refcounter=self._refcounter.as_readonly())
109
110 # Implements Block.Factory.
111 def _create_block(
@@ -90,14 +125,14 @@ class PrefixCachingBlockAllocator(BlockAllocator):
125 token_ids=token_ids,
126 block_size=block_size,
127 block_id=block_id,
93 - prefix_caching_allocator=allocator,
128 + allocator=allocator,
129 computed=computed,
130 )
131
97 - def allocate_immutable(self,
98 - prev_block: Optional[Block],
99 - token_ids: List[int],
100 - device: Optional[Device] = None) -> Block:
132 + def allocate_immutable_block(self,
133 + prev_block: Optional[Block],
134 + token_ids: List[int],
135 + device: Optional[Device] = None) -> Block:
136 """Allocates an immutable block with the given token IDs, reusing cached
137 blocks if possible.
138
@@ -111,29 +146,41 @@ class PrefixCachingBlockAllocator(BlockAllocator):
146 assert device is None
147 assert_prefix_caching_block_or_none(prev_block)
148
114 - block = self._create_block(
115 - prev_block=prev_block,
116 - token_ids=token_ids,
117 - block_size=self._block_size,
118 - allocator=self,
119 - )
149 + # First, try to create a block that points to cached data
150 + block = self._block_pool.init_block(prev_block=prev_block,
151 + token_ids=token_ids,
152 + block_size=self._block_size,
153 + physical_block_id=None)
154 assert block.content_hash is not None
155
156 cached_block_id = self._cached_blocks.get(block.content_hash, None)
157 if cached_block_id is not None:
158 block.block_id = cached_block_id
125 - self._incr_refcount_cached_block(block, block.block_id)
159 + self._incr_refcount_cached_block(block)
160 return block
161 + self._block_pool.free_block(block)
162
128 - block = self.allocate_mutable(prev_block)
163 + # No cached block => Allocate a new block
164 + block = self.allocate_mutable_block(prev_block)
165 block.append_token_ids(token_ids)
130 - assert block.content_hash is not None
131 -
166 return block
167
134 - def allocate_mutable(self,
135 - prev_block: Optional[Block],
136 - device: Optional[Device] = None) -> Block:
168 + def allocate_immutable_blocks(
169 + self,
170 + prev_block: Optional[Block],
171 + block_token_ids: List[List[int]],
172 + device: Optional[Device] = None) -> List[Block]:
173 + blocks = []
174 + for token_ids in block_token_ids:
175 + prev_block = self.allocate_immutable_block(prev_block=prev_block,
176 + token_ids=token_ids,
177 + device=device)
178 + blocks.append(prev_block)
179 + return blocks
180 +
181 + def allocate_mutable_block(self,
182 + prev_block: Optional[Block],
183 + device: Optional[Device] = None) -> Block:
184 """Allocates a mutable block. If there are no free blocks, this will
185 evict unused cached blocks.
186
@@ -147,116 +194,154 @@ class PrefixCachingBlockAllocator(BlockAllocator):
194 assert device is None
195 assert_prefix_caching_block_or_none(prev_block)
196
150 - try:
151 - block = self._hashless_allocator.allocate_mutable(
152 - prev_block=prev_block)
153 -
154 - assert block.block_id not in self._blocks
155 - assert block.block_id is not None
156 - self._blocks[block.block_id] = block
157 - return block
158 - except BlockAllocator.NoFreeBlocksError:
159 - # We must check the unused cached blocks before raising OOM.
160 - pass
161 -
162 - # If the evictor has blocks available for eviction, evict a block
163 - # and return it.
164 - if self.evictor.num_blocks > 0:
165 - # here we get an evicted block, which is only added
166 - # into evictor if its ref counter is 0
167 - # and since its content would be changed, we need
168 - # to remove it from _cached_blocks's tracking list
169 - block_id, content_hash_to_evict = self.evictor.evict()
170 -
171 - _block_id = self._cached_blocks[content_hash_to_evict]
172 - assert self._refcounter.get(_block_id) == 0
173 - assert _block_id == block_id
174 -
175 - self._cached_blocks.pop(content_hash_to_evict)
176 -
177 - self._refcounter.incr(block_id)
178 -
179 - # Now this block is pop from evictor and ready to write
180 - # with new content which most probably different with
181 - # original content. So need to tell worker to recompute
182 - # its kvcache
183 - block = self._create_block(
184 - prev_block=prev_block,
185 - token_ids=[],
186 - block_size=self._block_size,
187 - allocator=self,
188 - block_id=block_id,
189 - computed=False,
190 - )
191 - assert block.content_hash is None
192 -
193 - assert block.block_id not in self._blocks
194 - assert block.block_id is not None
195 - self._blocks[block.block_id] = block
196 - return block
197 -
198 - # No block available in hashless allocator, nor in unused cache blocks.
199 - raise BlockAllocator.NoFreeBlocksError()
197 + block_id = self._allocate_block_id()
198 + block = self._block_pool.init_block(prev_block=prev_block,
199 + token_ids=[],
200 + block_size=self._block_size,
201 + physical_block_id=block_id)
202 + assert not block.computed
203 + assert block.content_hash is None
204 + return block
205
201 - def _incr_refcount_cached_block(self, block: Block,
202 - block_id: BlockId) -> None:
203 - # now _incr_refcount_cached_block comes from two place
204 - # allocate_immutable/promote_to_immutable_block where hit
205 - # _cached_blocks hash key.
206 - # In both cases, it means that already exists a already
207 - # computed block which shared with block now
206 + def _incr_refcount_cached_block(self, block: Block) -> None:
207 + # Set this block to be "computed" since it is pointing to a
208 + # cached block id (which was already computed)
209 block.computed = True
210
211 + block_id = block.block_id
212 + assert block_id is not None
213 +
214 refcount = self._refcounter.incr(block_id)
215 if refcount == 1:
212 - # if block get referred, then it shall not be in evictor
213 - # and put it into _blocks for tracking
216 + # In case a cached block was evicted, restore its tracking
217 if block_id in self.evictor:
218 self.evictor.remove(block_id)
216 - self._blocks[block_id] = block
219
218 - def free(self, block: Block) -> None:
219 - """Decrement the refcount of the block. If the decremented refcount is
220 - zero, store the block in the freelist.
220 + self._track_block_id(block_id, computed=True)
221
222 - If the block has a content hash (meaning it is immutable), then we will
223 - keep the block around in case future allocations require it.
224 - """
225 - assert (block.block_id
226 - is not None), "freeing unallocated block is undefined"
222 + def _decr_refcount_cached_block(self, block: Block) -> None:
223 + # Ensure this is immutable/cached block
224 + assert block.content_hash is not None
225 +
226 + block_id = block.block_id
227 + assert block_id is not None
228 +
229 + refcount = self._refcounter.decr(block_id)
230 + if refcount > 0:
231 + block.block_id = None
232 + return
233 + else:
234 + assert refcount == 0
235
228 - self._free_block_id_for_block(block.block_id, block)
236 + # No longer used
237 + assert block.content_hash in self._cached_blocks
238 +
239 + # Add the cached block to the evictor
240 + # (This keeps the cached block around so it can be reused)
241 + self.evictor.add(block_id, block.content_hash, block.num_tokens_total,
242 + self._block_tracker[block_id].last_accessed)
243 +
244 + # Stop tracking the block
245 + self._untrack_block_id(block_id)
246
247 block.block_id = None
248
232 - def _free_block_id_for_block(self, block_id: BlockId,
233 - block: Block) -> None:
234 - assert isinstance(block, PrefixCachingBlock)
235 -
236 - # if we comes from promote_to_immutable_block, it means that
237 - # block.content_hash is never None.
238 - # However we need to release the same content block, so that
239 - # physical block could get reused.
240 - if block.block_id != block_id or block.content_hash is None:
241 - refcount = self._refcounter.get(block_id)
242 - # We have fork case where block would get more than one ref,
243 - # so we cannot free it from tracking if ref cnt large than 1
244 - assert block.block_id is not None
245 - refcount = self._refcounter.get(block.block_id)
246 - if refcount == 1:
247 - del self._blocks[block.block_id]
248 -
249 - return self._hashless_allocator.free(block)
249 + def _decr_refcount_hashless_block(self, block: Block) -> None:
250 + block_id = block.block_id
251 + assert block_id is not None
252
251 - refcount = self._refcounter.decr(block_id)
253 + # We may have a fork case where block is shared,
254 + # in which case, we cannot remove it from tracking
255 + refcount = self._refcounter.get(block_id)
256 + if refcount == 1:
257 + self._untrack_block_id(block_id)
258
253 - # If no longer used, add the block to the evictor.
254 - if refcount == 0:
255 - assert block.content_hash in self._cached_blocks
256 - assert block.block_id is not None
257 - del self._blocks[block.block_id]
258 - self.evictor.add(block.block_id, block.content_hash,
259 - block.num_tokens_total, block.last_accessed)
259 + # Decrement refcount of the block_id, but do not free the block object
260 + # itself (will be handled by the caller)
261 + self._hashless_allocator.free(block, keep_block_object=True)
262 +
263 + def _allocate_block_id(self) -> BlockId:
264 + """First tries to allocate a block id from the hashless allocator,
265 + and if there are no blocks, then tries to evict an unused cached block.
266 + """
267 + hashless_block_id = self._maybe_allocate_hashless_block_id()
268 + if hashless_block_id is not None:
269 + return hashless_block_id
270 +
271 + evicted_block_id = self._maybe_allocate_evicted_block_id()
272 + if evicted_block_id is not None:
273 + return evicted_block_id
274 +
275 + # No block available in hashless allocator, nor in unused cache blocks.
276 + raise BlockAllocator.NoFreeBlocksError()
277 +
278 + def _maybe_allocate_hashless_block_id(self) -> Optional[BlockId]:
279 + try:
280 + # Allocate mutable block and extract its block_id
281 + block = self._hashless_allocator.allocate_mutable_block(
282 + prev_block=None)
283 + block_id = block.block_id
284 + self._block_pool.free_block(block)
285 +
286 + self._track_block_id(block_id, computed=False)
287 + return block_id
288 + except BlockAllocator.NoFreeBlocksError:
289 + return None
290 +
291 + def _maybe_allocate_evicted_block_id(self) -> Optional[BlockId]:
292 + if self.evictor.num_blocks == 0:
293 + return None
294 +
295 + # Here we get an evicted block, which is only added
296 + # into evictor if its ref counter is 0
297 + # and since its content would be changed, we need
298 + # to remove it from _cached_blocks's tracking list
299 + block_id, content_hash_to_evict = self.evictor.evict()
300 +
301 + # Sanity checks
302 + assert content_hash_to_evict in self._cached_blocks
303 + _block_id = self._cached_blocks[content_hash_to_evict]
304 + assert self._refcounter.get(_block_id) == 0
305 + assert _block_id == block_id
306 +
307 + self._cached_blocks.pop(content_hash_to_evict)
308 +
309 + self._refcounter.incr(block_id)
310 + self._track_block_id(block_id, computed=False)
311 +
312 + return block_id
313 +
314 + def _free_block_id(self, block: Block) -> None:
315 + """Decrements the refcount of the block. The block may be in two
316 + possible states: (1) immutable/cached or (2) mutable/hashless.
317 + In the first case, the refcount is decremented directly and the block
318 + may be possibly added to the evictor. In other case, hashless
319 + allocator free(..) with keep_block_object=True is called to only free
320 + the block id (since the block object may be reused by the caller)
321 + """
322 + block_id = block.block_id
323 + assert block_id is not None, "Freeing unallocated block is undefined"
324 +
325 + if block.content_hash is not None:
326 + # Immutable: This type of block is always cached, and we want to
327 + # keep it in the evictor for future reuse
328 + self._decr_refcount_cached_block(block)
329 + else:
330 + # Mutable: This type of block is not cached, so we release it
331 + # directly to the hashless allocator
332 + self._decr_refcount_hashless_block(block)
333 +
334 + assert block.block_id is None
335 +
336 + def free(self, block: Block, keep_block_object: bool = False) -> None:
337 + """Release the block (look at free_block_id(..) docs)
338 + """
339 + # Release the physical block index
340 + self._free_block_id(block)
341 +
342 + # Release the block object to the pool
343 + if not keep_block_object:
344 + self._block_pool.free_block(block)
345
346 def fork(self, last_block: Block) -> List[Block]:
347 """Creates a new sequence of blocks that shares the same underlying
@@ -274,17 +359,20 @@ class PrefixCachingBlockAllocator(BlockAllocator):
359 forked_blocks: List[Block] = []
360 prev_block = None
361 for block in source_blocks:
277 - refcount = self._refcounter.incr(block.block_id)
278 - assert refcount != 1, "can't fork free'd block"
279 -
280 - forked_blocks.append(
281 - self._create_block(
282 - prev_block=prev_block,
283 - token_ids=block.token_ids,
284 - block_id=block.block_id,
285 - block_size=self._block_size,
286 - allocator=self,
287 - ))
362 + block_id = block.block_id
363 + assert block_id is not None
364 +
365 + refcount = self._refcounter.incr(block_id)
366 + assert refcount != 1, "can't fork free'd block_id = {}".format(
367 + block_id)
368 +
369 + forked_block = self._block_pool.init_block(
370 + prev_block=prev_block,
371 + token_ids=block.token_ids,
372 + block_size=self._block_size,
373 + physical_block_id=block_id)
374 +
375 + forked_blocks.append(forked_block)
376 prev_block = forked_blocks[-1]
377
378 return forked_blocks
@@ -329,7 +417,7 @@ class PrefixCachingBlockAllocator(BlockAllocator):
417
418 Note that if we already have a cached block with the same content, we
419 will replace the newly-promoted block's mapping with the existing cached
332 - block.
420 + block id.
421
422 Args:
423 block: The mutable block to be promoted.
@@ -338,23 +426,30 @@ class PrefixCachingBlockAllocator(BlockAllocator):
426 BlockId: Either the original block index, or the block index of
427 the previously cached block matching the same content.
428 """
429 + # Ensure block can be promoted
430 assert block.content_hash is not None
431 assert block.block_id is not None
432 assert self._refcounter.get(block.block_id) > 0
433
345 - # If the content hash does not have a corresponding cached block,
346 - # set this block as the cached block.
434 if block.content_hash not in self._cached_blocks:
435 + # No cached content hash => Set this block as cached
436 + # (Note that this block is not computed yet =>
437 + # Will be computed after free())
438 self._cached_blocks[block.content_hash] = block.block_id
349 - else:
350 - self._free_block_id_for_block(
351 - self._cached_blocks[block.content_hash], block)
352 - self._incr_refcount_cached_block(
353 - block, self._cached_blocks[block.content_hash])
439 + return block.block_id
440
355 - return self._cached_blocks[block.content_hash]
441 + # Reuse the cached content hash
442 + self._decr_refcount_hashless_block(block)
443 + block.block_id = self._cached_blocks[block.content_hash]
444
357 - def cow_block_if_not_appendable(self, block: Block) -> Optional[BlockId]:
445 + # Increment refcount of the cached block and (possibly) restore
446 + # it from the evictor.
447 + # Note that in this case, the block is marked as computed
448 + self._incr_refcount_cached_block(block)
449 +
450 + return block.block_id
451 +
452 + def cow_block_if_not_appendable(self, block: Block) -> BlockId:
453 """Performs a copy-on-write operation on the given block if it is not
454 appendable.
455
@@ -362,11 +457,22 @@ class PrefixCachingBlockAllocator(BlockAllocator):
457 block (Block): The block to check for copy-on-write.
458
459 Returns:
365 - Optional[BlockId]: The block index of the new block if a copy-on
366 - -write operation was performed, or the original block index if
460 + BlockId: The block index of the new block if a copy-on-write
461 + operation was performed, or the original block index if
462 no copy-on-write was necessary.
463 """
369 - return self._cow_tracker.cow_block_if_not_appendable(block)
464 + src_block_id = block.block_id
465 + assert src_block_id is not None
466 +
467 + if self._cow_tracker.is_appendable(block):
468 + return src_block_id
469 +
470 + self._free_block_id(block)
471 + trg_block_id = self._allocate_block_id()
472 +
473 + self._cow_tracker.record_cow(src_block_id, trg_block_id)
474 +
475 + return trg_block_id
476
477 def clear_copy_on_writes(self) -> List[Tuple[BlockId, BlockId]]:
478 """Returns the copy-on-write source->destination mapping and clears it.
@@ -386,8 +492,8 @@ class PrefixCachingBlockAllocator(BlockAllocator):
492 """
493
494 for block_id in block_ids:
389 - if block_id in self._blocks:
390 - self._blocks[block_id].last_accessed = now
495 + if self._block_tracker[block_id].active:
496 + self._block_tracker[block_id].last_accessed = now
497 elif block_id in self.evictor:
498 self.evictor.update(block_id, now)
499 else:
@@ -395,25 +501,46 @@ class PrefixCachingBlockAllocator(BlockAllocator):
501 "Mark block as accessed which is not belonged to GPU")
502
503 def mark_blocks_as_computed(self, block_ids: List[int]) -> None:
398 - """Mark blocks as computed, used in prefix caching."""
504 + raise NotImplementedError("Marking as computed is incremental")
505
400 - for block_id in block_ids:
401 - if block_id in self._blocks:
402 - # only those full block is valid for prefix caching
403 - if self._blocks[block_id].is_full:
404 - self._blocks[block_id].computed = True
405 - elif block_id not in self.evictor:
406 - raise ValueError(f"Mark {block_id=} as computed which "
407 - "is not belonged to GPU")
506 + def _track_block_id(self, block_id: Optional[BlockId],
507 + computed: bool) -> None:
508 + assert block_id is not None
509 + self._block_tracker[block_id].enable()
510 + self._block_tracker[block_id].computed = computed
511 +
512 + def _untrack_block_id(self, block_id: Optional[BlockId]) -> None:
513 + assert block_id is not None
514 + self._block_tracker[block_id].disable()
515
516 def block_is_computed(self, block_id: int) -> bool:
410 - if block_id in self._blocks:
411 - return self._blocks[block_id].computed
517 + if self._block_tracker[block_id].active:
518 + return self._block_tracker[block_id].computed
519 else:
520 return block_id in self.evictor
521
522 + def get_computed_block_ids(self,
523 + prev_computed_block_ids: List[int],
524 + block_ids: List[int],
525 + skip_last_block_id: bool = True) -> List[int]:
526 + prev_prefix_size = len(prev_computed_block_ids)
527 + cur_size = len(block_ids)
528 + if skip_last_block_id:
529 + cur_size -= 1
530 +
531 + # Sanity checks
532 + assert cur_size >= 0
533 + assert prev_prefix_size <= cur_size
534 +
535 + ret = prev_computed_block_ids
536 + for i in range(prev_prefix_size, cur_size):
537 + block_id = block_ids[i]
538 + if self.block_is_computed(block_id):
539 + ret.append(block_id)
540 + return ret
541 +
542 def get_common_computed_block_ids(
416 - self, seq_block_ids: List[List[int]]) -> List[int]:
543 + self, computed_seq_block_ids: List[List[int]]) -> List[int]:
544 """Return the block ids that are common for a given sequence group.
545
546 Only those blocks that are immutable and already be marked
@@ -424,14 +551,9 @@ class PrefixCachingBlockAllocator(BlockAllocator):
551 # prompt is cached. This would cause erroneous behavior in model
552 # runner.
553
427 - ids_list = [
428 - list(
429 - takewhile(lambda block_id: self.block_is_computed(block_id),
430 - seq[:-1])) for seq in seq_block_ids
431 - ]
554 # It returns a list of int although type annotation says list of string.
555 return commonprefix([
434 - ids for ids in ids_list # type: ignore
556 + ids for ids in computed_seq_block_ids # type: ignore
557 if ids != []
558 ])
559
@@ -473,10 +595,10 @@ class PrefixCachingBlockAllocator(BlockAllocator):
595 blocks: List of blocks to be swapped out.
596 """
597 for block in blocks:
476 - self.free(block)
598 + self._free_block_id(block)
599
600 def swap_in(self, blocks: List[Block]) -> None:
479 - """Execute the swap int actions. Change the block id from
601 + """Execute the swap in actions. Change the block id from
602 old allocator to current allocator for each block to finish
603 the block table update.
604
@@ -484,13 +606,22 @@ class PrefixCachingBlockAllocator(BlockAllocator):
606 blocks: List of blocks to be swapped in.
607 """
608 for block in blocks:
609 + # Here we allocate either immutable or mutable block and then
610 + # extract its block_id. Note that the block object is released
611 + # and the block_id is assigned to "block" to allow reusing the
612 + # existing "block" object
613 if block.is_full:
488 - alloc = self.allocate_immutable(block.prev_block,
489 - block.token_ids)
614 + tmp_block = self.allocate_immutable_block(
615 + prev_block=block.prev_block, token_ids=block.token_ids)
616 else:
491 - alloc = self.allocate_mutable(block.prev_block)
492 - alloc.append_token_ids(block.token_ids)
493 - block.block_id = alloc.block_id
617 + tmp_block = self.allocate_mutable_block(
618 + prev_block=block.prev_block)
619 + tmp_block.append_token_ids(block.token_ids)
620 +
621 + block_id = tmp_block.block_id
622 + self._block_pool.free_block(tmp_block)
623 +
624 + block.block_id = block_id # Assign block_id
625
626
627 class PrefixCachingBlock(Block):
@@ -507,7 +638,7 @@ class PrefixCachingBlock(Block):
638 token_ids (List[int]): The initial token IDs to be stored in the block.
639 block_size (int): The maximum number of token IDs that can be stored in
640 the block.
510 - prefix_caching_allocator (BlockAllocator): The prefix
641 + allocator (BlockAllocator): The prefix
642 caching block allocator associated with this block.
643 block_id (Optional[int], optional): The physical block index
644 of this block. Defaults to None.
@@ -518,31 +649,55 @@ class PrefixCachingBlock(Block):
649 prev_block: Optional[Block],
650 token_ids: List[int],
651 block_size: int,
521 - prefix_caching_allocator: BlockAllocator,
652 + allocator: BlockAllocator,
653 block_id: Optional[int] = None,
654 computed: bool = False,
655 ):
525 - assert isinstance(prefix_caching_allocator,
526 - PrefixCachingBlockAllocator), (
527 - "Currently this class is only tested with "
528 - "PrefixCachingBlockAllocator.")
656 + assert isinstance(allocator, PrefixCachingBlockAllocator), (
657 + "Currently this class is only tested with "
658 + "PrefixCachingBlockAllocator. Got instead allocator = {}".format(
659 + allocator))
660 assert_prefix_caching_block_or_none(prev_block)
661
662 self._prev_block = prev_block
663 self._cached_content_hash: Optional[int] = None
533 - self._cached_num_tokens_total: Optional[int] = None
534 - self._prefix_caching_allocator = prefix_caching_allocator
664 + self._cached_num_tokens_total: int = 0
665 + self._allocator = allocator
666 self._last_accessed: float = _DEFAULT_LAST_ACCESSED_TIME
667 self._computed = computed
668
538 - self._block = NaiveBlock(
539 - prev_block=prev_block,
540 - token_ids=token_ids,
541 - block_size=block_size,
542 - block_id=block_id,
543 - allocator=prefix_caching_allocator,
544 - _cow_target=self,
545 - )
669 + # On the first time, we create the block object, and next we only
670 + # reinitialize it
671 + if hasattr(self, "_block"):
672 + self._block.__init__( # type: ignore[has-type]
673 + prev_block=prev_block,
674 + token_ids=token_ids,
675 + block_size=block_size,
676 + block_id=block_id,
677 + allocator=self._allocator)
678 + else:
679 + self._block = NaiveBlock(prev_block=prev_block,
680 + token_ids=token_ids,
681 + block_size=block_size,
682 + block_id=block_id,
683 + allocator=self._allocator)
684 +
685 + self._update_num_tokens_total()
686 +
687 + def _update_num_tokens_total(self):
688 + """Incrementally computes the number of tokens that there is
689 + till the current block (included)
690 + """
691 + res = 0
692 +
693 + # Add all previous blocks
694 + if self._prev_block is not None:
695 + res += self._prev_block.num_tokens_total
696 +
697 + # Add current block
698 + res += len(self.token_ids)
699 +
700 + self._cached_num_tokens_total = res
701
702 @property
703 def computed(self) -> bool:
@@ -564,22 +719,28 @@ class PrefixCachingBlock(Block):
719 """Appends the given token IDs to the block and registers the block as
720 immutable if the block becomes full.
721
567 - Internally, the naive block handles CoW.
568 -
722 Args:
723 token_ids (List[int]): The token IDs to be appended to the block.
724 """
572 - assert token_ids
725 + # Ensure this is mutable block (not promoted)
726 + assert self.content_hash is None
727 + assert not self.computed
728 +
729 + if len(token_ids) == 0:
730 + return
731
574 - # naive block handles CoW.
732 + # Ensure there are input tokens
733 + assert token_ids, "Got token_ids = {}".format(token_ids)
734 +
735 + # Naive block handles CoW.
736 self._block.append_token_ids(token_ids)
737 + self._update_num_tokens_total()
738
739 # If the content hash is present, then the block can be made immutable.
740 # Register ourselves with the allocator, potentially replacing the
741 # physical block index.
742 if self.content_hash is not None:
581 - self.block_id = (self._prefix_caching_allocator.
582 - promote_to_immutable_block(self))
743 + self.block_id = self._allocator.promote_to_immutable_block(self)
744
745 @property
746 def block_id(self) -> Optional[int]:
@@ -599,23 +760,6 @@ class PrefixCachingBlock(Block):
760
761 @property
762 def num_tokens_total(self) -> int:
602 - """return the total tokens so far.
603 -
604 - Here we iterate the block chain till to the first block, while
605 - cache the result in local to prevent repeated computations.
606 - """
607 - if self._cached_num_tokens_total is not None:
608 - return self._cached_num_tokens_total
609 -
610 - _block: Optional[Block] = self
611 - self._cached_num_tokens_total = 0
612 -
613 - # TODO: current implement here take O(N^2), we expect future
614 - # we have O(1) here
615 - while _block is not None:
616 - self._cached_num_tokens_total += len(_block.token_ids)
617 - _block = _block.prev_block
618 -
763 return self._cached_num_tokens_total
764
765 @property
@@ -638,7 +782,6 @@ class PrefixCachingBlock(Block):
782 For the content-based hash to be defined, the current block must be
783 full.
784 """
641 -
785 # If the hash is already computed, return it.
786 if self._cached_content_hash is not None:
787 return self._cached_content_hash
@@ -688,7 +831,129 @@ class PrefixCachingBlock(Block):
831 return hash((is_first_block, prev_block_hash, *cur_block_token_ids))
832
833
834 +class ComputedBlocksTracker:
835 + """Handles caching of per-sequence computed block ids.
836 + When a sequence appears for the first time, it traverses all of the
837 + blocks and detects the prefix of blocks that is computed. On the
838 + subsequent times, it only traverses the new blocks that were added
839 + and updates the already recorded prefix of blocks with the newly
840 + computed blocks.
841 +
842 + To avoid redundant traversals, the algorithm also detects when there
843 + is a "gap" in the computed prefix. For example, if we have blocks =
844 + [1,2,3,4,5], and we have detected [1,2,3] as the computed prefix, then
845 + we won't try to add more computed blocks to [1,2,3] in this sequence
846 + iteration, and will add more computed blocks only after the sequence is
847 + freed and reused again.
848 +
849 + Note that currently, for a given sequence, we also skip the last
850 + block id for caching purposes, to avoid caching of a full sequence
851 + """
852 +
853 + def __init__(self, allocator):
854 + self._allocator = allocator
855 + self._cached_computed_seq_blocks: Dict[int, Tuple[List[int],
856 + bool]] = {}
857 +
858 + def add_seq(self, seq_id: int) -> None:
859 + """Start tracking seq_id
860 + """
861 + assert seq_id not in self._cached_computed_seq_blocks
862 + self._cached_computed_seq_blocks[seq_id] = ([], False)
863 +
864 + def remove_seq(self, seq_id: int) -> None:
865 + """Stop tracking seq_id
866 + """
867 + assert seq_id in self._cached_computed_seq_blocks
868 + del self._cached_computed_seq_blocks[seq_id]
869 +
870 + def get_cached_computed_blocks_and_update(
871 + self, seq_id: int, block_ids: List[int]) -> List[int]:
872 + """ Look at the class documentation for details
873 + """
874 + # Ensure seq_id is already tracked
875 + assert seq_id in self._cached_computed_seq_blocks
876 +
877 + # Get cached data (may be empty on the first time)
878 + prev_computed_block_ids, has_gap = self._cached_computed_seq_blocks[
879 + seq_id]
880 +
881 + if has_gap:
882 + # When gap is detected, we do not add more computed blocks at this
883 + # sequence iteration
884 + return prev_computed_block_ids
885 +
886 + # We do not consider the last block id for caching purposes.
887 + num_cur_blocks = len(block_ids) - 1
888 + assert num_cur_blocks >= 0
889 +
890 + if len(prev_computed_block_ids) >= num_cur_blocks:
891 + # Cache HIT
892 + assert len(prev_computed_block_ids) == num_cur_blocks
893 + return prev_computed_block_ids
894 +
895 + # If here, then we may possibly add more computed blocks. As a result,
896 + # traverse the additional blocks after prev_computed_block_ids to
897 + # detect more computed blocks and add them.
898 +
899 + # Incremental init for seq_id => Look only at the new blocks
900 + computed_block_ids = self._allocator.get_computed_block_ids( # noqa: E501
901 + prev_computed_block_ids,
902 + block_ids,
903 + skip_last_block_id=
904 + True, # We skip last block id to avoid caching of full seq
905 + )
906 +
907 + # Detect if there is a "gap"
908 + has_gap = len(computed_block_ids) < num_cur_blocks
909 +
910 + # Record
911 + self._cached_computed_seq_blocks[seq_id] = (computed_block_ids,
912 + has_gap)
913 +
914 + return computed_block_ids
915 +
916 +
917 +class LastAccessBlocksTracker:
918 + """Manages the last access time of the tracked sequences, in order to allow
919 + an efficient update of allocator's block last access times
920 + """
921 +
922 + def __init__(self, allocator):
923 + self._allocator = allocator
924 + self._seq_last_access: Dict[int, Optional[float]] = {}
925 +
926 + def add_seq(self, seq_id: int) -> None:
927 + """Start tracking seq_id
928 + """
929 + assert seq_id not in self._seq_last_access
930 + self._seq_last_access[seq_id] = None
931 +
932 + def remove_seq(self, seq_id: int) -> None:
933 + """Stop tracking seq_id
934 + """
935 + assert seq_id in self._seq_last_access
936 + del self._seq_last_access[seq_id]
937 +
938 + def update_last_access(self, seq_id: int, time: float) -> None:
939 + assert seq_id in self._seq_last_access
940 + self._seq_last_access[seq_id] = time
941 +
942 + def update_seq_blocks_last_access(self, seq_id: int,
943 + block_ids: List[int]) -> None:
944 + assert seq_id in self._seq_last_access
945 +
946 + ts = self._seq_last_access[seq_id]
947 +
948 + if ts is None:
949 + # No last access was recorded, no need to update.
950 + return
951 +
952 + self._allocator.mark_blocks_as_accessed(block_ids, ts)
953 +
954 +
955 def assert_prefix_caching_block_or_none(block: Optional[Block]):
956 if block is None:
957 return
694 - assert isinstance(block, PrefixCachingBlock)
958 + assert isinstance(block,
959 + PrefixCachingBlock), "Got block = {}".format(block)
vllm/core/block_manager_v2.py
+101 -49
@@ -7,6 +7,8 @@ from typing import Tuple
7 from vllm.core.block.block_table import BlockTable
8 from vllm.core.block.cpu_gpu_block_allocator import CpuGpuBlockAllocator
9 from vllm.core.block.interfaces import Block
10 +from vllm.core.block.prefix_caching_block import (ComputedBlocksTracker,
11 + LastAccessBlocksTracker)
12 from vllm.core.block.utils import check_no_caching_or_swa_for_blockmgr_encdec
13 from vllm.core.interfaces import AllocStatus, BlockSpaceManager
14 from vllm.sequence import Sequence, SequenceGroup, SequenceStatus
@@ -100,6 +102,11 @@ class BlockSpaceManagerV2(BlockSpaceManager):
102 self.block_tables: Dict[SeqId, BlockTable] = {}
103 self.cross_block_tables: Dict[EncoderSeqId, BlockTable] = {}
104
105 + self._computed_blocks_tracker = ComputedBlocksTracker(
106 + self.block_allocator)
107 + self._last_access_blocks_tracker = LastAccessBlocksTracker(
108 + self.block_allocator)
109 +
110 def can_allocate(self, seq_group: SequenceGroup) -> AllocStatus:
111 # FIXME(woosuk): Here we assume that all sequences in the group share
112 # the same prompt. This may not be true for preempted sequences.
@@ -157,10 +164,18 @@ class BlockSpaceManagerV2(BlockSpaceManager):
164 block_table: BlockTable = self._allocate_sequence(seq)
165 self.block_tables[seq.seq_id] = block_table
166
167 + # Track seq
168 + self._computed_blocks_tracker.add_seq(seq.seq_id)
169 + self._last_access_blocks_tracker.add_seq(seq.seq_id)
170 +
171 # Assign the block table for each sequence.
172 for seq in waiting_seqs[1:]:
173 self.block_tables[seq.seq_id] = block_table.fork()
174
175 + # Track seq
176 + self._computed_blocks_tracker.add_seq(seq.seq_id)
177 + self._last_access_blocks_tracker.add_seq(seq.seq_id)
178 +
179 # Allocate cross-attention block table for encoder sequence
180 #
181 # NOTE: Here we assume that all sequences in the group have the same
@@ -224,11 +239,23 @@ class BlockSpaceManagerV2(BlockSpaceManager):
239 return new_cows
240
241 def free(self, seq: Sequence) -> None:
227 - if seq.seq_id not in self.block_tables:
242 + seq_id = seq.seq_id
243 +
244 + if seq_id not in self.block_tables:
245 # Already freed or haven't been scheduled yet.
246 return
230 - self.block_tables[seq.seq_id].free()
231 - del self.block_tables[seq.seq_id]
247 +
248 + # Update seq block ids with the latest access time
249 + self._last_access_blocks_tracker.update_seq_blocks_last_access(
250 + seq_id, self.block_tables[seq.seq_id].physical_block_ids)
251 +
252 + # Untrack seq
253 + self._last_access_blocks_tracker.remove_seq(seq_id)
254 + self._computed_blocks_tracker.remove_seq(seq_id)
255 +
256 + # Free table/blocks
257 + self.block_tables[seq_id].free()
258 + del self.block_tables[seq_id]
259
260 def free_cross(self, seq_group: SequenceGroup) -> None:
261 request_id = seq_group.request_id
@@ -239,9 +266,7 @@ class BlockSpaceManagerV2(BlockSpaceManager):
266 del self.cross_block_tables[request_id]
267
268 def get_block_table(self, seq: Sequence) -> List[int]:
242 - assert seq.seq_id in self.block_tables
269 block_ids = self.block_tables[seq.seq_id].physical_block_ids
244 - assert all(b is not None for b in block_ids)
270 return block_ids # type: ignore
271
272 def get_cross_block_table(self, seq_group: SequenceGroup) -> List[int]:
@@ -252,20 +277,14 @@ class BlockSpaceManagerV2(BlockSpaceManager):
277 return block_ids # type: ignore
278
279 def access_all_blocks_in_seq(self, seq: Sequence, now: float):
255 - # Update the last accessed time of all the blocks accessed
256 - # in this step.
257 - # And the accessed time is only useful for prefix caching now,
258 - # as it support internal evictor policy for which cached
259 - # block could be refilled, to keep cached content could be reused
260 - # at max extend.
280 if self.enable_caching:
262 - block_table = self.block_tables[seq.seq_id]
263 - block_ids: List[Optional[int]] = []
264 - for block_id in block_table.physical_block_ids:
265 - block_ids.append(block_id)
266 - self.block_allocator.mark_blocks_as_accessed(
267 - block_ids, # type: ignore
268 - now)
281 + # Record the latest access time for the sequence. The actual update
282 + # of the block ids is deferred to the sequence free(..) call, since
283 + # only during freeing of block ids, the blocks are actually added to
284 + # the evictor (which is when the most updated time is required)
285 + # (This avoids expensive calls to mark_blocks_as_accessed(..))
286 + self._last_access_blocks_tracker.update_last_access(
287 + seq.seq_id, now)
288
289 def mark_blocks_as_computed(self, seq_group: SequenceGroup):
290 # The only need for mark block as computed is for prefix caching,
@@ -285,17 +304,26 @@ class BlockSpaceManagerV2(BlockSpaceManager):
304 This method determines which blocks can be safely skipped for all
305 sequences in the sequence group.
306 """
288 - seq_block_ids = [
289 - self.block_tables[seq.seq_id].physical_block_ids for seq in seqs
290 - ]
307 + computed_seq_block_ids = []
308 + for seq in seqs:
309 + computed_seq_block_ids.append(
310 + self._computed_blocks_tracker.
311 + get_cached_computed_blocks_and_update(
312 + seq.seq_id,
313 + self.block_tables[seq.seq_id].physical_block_ids))
314 +
315 # NOTE(sang): This assumes seq_block_ids doesn't contain any None.
316 return self.block_allocator.get_common_computed_block_ids(
293 - seq_block_ids) # type: ignore
317 + computed_seq_block_ids) # type: ignore
318
319 def fork(self, parent_seq: Sequence, child_seq: Sequence) -> None:
320 src_block_table = self.block_tables[parent_seq.seq_id]
321 self.block_tables[child_seq.seq_id] = src_block_table.fork()
322
323 + # Track child seq
324 + self._computed_blocks_tracker.add_seq(child_seq.seq_id)
325 + self._last_access_blocks_tracker.add_seq(child_seq.seq_id)
326 +
327 def can_swap_in(self, seq_group: SequenceGroup,
328 num_lookahead_slots: int) -> AllocStatus:
329 """Returns the AllocStatus for the given sequence_group
@@ -323,19 +351,31 @@ class BlockSpaceManagerV2(BlockSpaceManager):
351 List[Tuple[int, int]]: The mapping of swapping block from CPU
352 to GPU.
353 """
326 - blocks = self._get_blocks_for_swap(seq_group, SequenceStatus.SWAPPED)
327 - current_swap_mapping = self.block_allocator.swap(
328 - blocks=blocks, source_device=Device.CPU, dest_device=Device.GPU)
329 -
330 - block_number_mapping = {
331 - self.block_allocator.get_physical_block_id(Device.CPU,
332 - cpu_block_id):
333 - self.block_allocator.get_physical_block_id(Device.GPU,
334 - gpu_block_id)
335 - for cpu_block_id, gpu_block_id in current_swap_mapping.items()
336 - }
337 - # convert to list of tuples once here
338 - return list(block_number_mapping.items())
354 + physical_block_id_mapping = []
355 + for seq in seq_group.get_seqs(status=SequenceStatus.SWAPPED):
356 + blocks = self.block_tables[seq.seq_id].blocks
357 + if len(blocks) == 0:
358 + continue
359 +
360 + seq_swap_mapping = self.block_allocator.swap(blocks=blocks,
361 + src_device=Device.CPU,
362 + dst_device=Device.GPU)
363 +
364 + # Refresh the block ids of the table (post-swap)
365 + self.block_tables[seq.seq_id].update(blocks)
366 +
367 + seq_physical_block_id_mapping = {
368 + self.block_allocator.get_physical_block_id(
369 + Device.CPU, cpu_block_id):
370 + self.block_allocator.get_physical_block_id(
371 + Device.GPU, gpu_block_id)
372 + for cpu_block_id, gpu_block_id in seq_swap_mapping.items()
373 + }
374 +
375 + physical_block_id_mapping.extend(
376 + list(seq_physical_block_id_mapping.items()))
377 +
378 + return physical_block_id_mapping
379
380 def can_swap_out(self, seq_group: SequenceGroup) -> bool:
381 """Returns whether we can swap out the given sequence_group
@@ -355,7 +395,7 @@ class BlockSpaceManagerV2(BlockSpaceManager):
395 return True
396 return False
397
358 - def swap_out(self, sequence_group: SequenceGroup) -> List[Tuple[int, int]]:
398 + def swap_out(self, seq_group: SequenceGroup) -> List[Tuple[int, int]]:
399 """Returns the block id mapping (from GPU to CPU) generated by
400 swapping out the given sequence_group with num_lookahead_slots.
401
@@ -366,19 +406,31 @@ class BlockSpaceManagerV2(BlockSpaceManager):
406 List[Tuple[int, int]]: The mapping of swapping block from
407 GPU to CPU.
408 """
369 - blocks = self._get_blocks_for_swap(sequence_group,
370 - SequenceStatus.RUNNING)
371 - current_swap_mapping = self.block_allocator.swap(
372 - blocks=blocks, source_device=Device.GPU, dest_device=Device.CPU)
373 - block_number_mapping = {
374 - self.block_allocator.get_physical_block_id(Device.GPU,
375 - gpu_block_id):
376 - self.block_allocator.get_physical_block_id(Device.CPU,
377 - cpu_block_id)
378 - for gpu_block_id, cpu_block_id in current_swap_mapping.items()
379 - }
380 - # convert to list of tuples once here
381 - return list(block_number_mapping.items())
409 + physical_block_id_mapping = []
410 + for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
411 + blocks = self.block_tables[seq.seq_id].blocks
412 + if len(blocks) == 0:
413 + continue
414 +
415 + seq_swap_mapping = self.block_allocator.swap(blocks=blocks,
416 + src_device=Device.GPU,
417 + dst_device=Device.CPU)
418 +
419 + # Refresh the block ids of the table (post-swap)
420 + self.block_tables[seq.seq_id].update(blocks)
421 +
422 + seq_physical_block_id_mapping = {
423 + self.block_allocator.get_physical_block_id(
424 + Device.GPU, gpu_block_id):
425 + self.block_allocator.get_physical_block_id(
426 + Device.CPU, cpu_block_id)
427 + for gpu_block_id, cpu_block_id in seq_swap_mapping.items()
428 + }
429 +
430 + physical_block_id_mapping.extend(
431 + list(seq_physical_block_id_mapping.items()))
432 +
433 + return physical_block_id_mapping
434
435 def get_num_free_gpu_blocks(self) -> int:
436 return self.block_allocator.get_num_free_blocks(Device.GPU)
vllm/engine/llm_engine.py
+4 -1
@@ -177,7 +177,8 @@ class LLMEngine:
177 "enforce_eager=%s, kv_cache_dtype=%s, "
178 "quantization_param_path=%s, device_config=%s, "
179 "decoding_config=%r, observability_config=%r, "
180 - "seed=%d, served_model_name=%s)",
180 + "seed=%d, served_model_name=%s, use_v2_block_manager=%s, "
181 + "enable_prefix_caching=%s)",
182 VLLM_VERSION,
183 model_config.model,
184 speculative_config,
@@ -204,6 +205,8 @@ class LLMEngine:
205 observability_config,
206 model_config.seed,
207 model_config.served_model_name,
208 + scheduler_config.use_v2_block_manager,
209 + cache_config.enable_prefix_caching,
210 )
211 # TODO(woosuk): Print more configs in debug mode.
212
vllm/entrypoints/openai/serving_completion.py
+1 -1
@@ -345,7 +345,7 @@ class OpenAIServingCompletion(OpenAIServing):
345 out_logprobs = prompt_logprobs
346 output_text = prompt_text
347 elif request.echo and request.max_tokens > 0:
348 - token_ids = prompt_token_ids + output.token_ids
348 + token_ids = prompt_token_ids + list(output.token_ids)
349 out_logprobs = (prompt_logprobs + output.logprobs
350 if request.logprobs is not None else None)
351 output_text = prompt_text + output.text
vllm/model_executor/sampling_metadata.py
+2 -2
@@ -427,8 +427,8 @@ class SamplingTensors:
427 if seq_group.do_sample:
428 for seq_id in seq_ids:
429 seq_data = seq_group.seq_data[seq_id]
430 - prompt_tokens.append(seq_data.prompt_token_ids)
431 - output_tokens.append(seq_data.output_token_ids)
430 + prompt_tokens.append(list(seq_data.prompt_token_ids))
431 + output_tokens.append(list(seq_data.output_token_ids))
432
433 sampling_tensors = SamplingTensors.from_lists(
434 temperatures, top_ps, top_ks, min_ps, presence_penalties,
vllm/outputs.py
+2 -2
@@ -1,6 +1,6 @@
1 import time
2 from dataclasses import dataclass
3 -from typing import List, Optional, Union
3 +from typing import List, Optional, Tuple, Union
4
5 from vllm.lora.request import LoRARequest
6 from vllm.sequence import (PromptLogprobs, RequestMetrics, SampleLogprobs,
@@ -28,7 +28,7 @@ class CompletionOutput:
28
29 index: int
30 text: str
31 - token_ids: List[int]
31 + token_ids: Tuple[int, ...]
32 cumulative_logprob: float
33 logprobs: Optional[SampleLogprobs]
34 finish_reason: Optional[str] = None
vllm/sequence.py
+47 -22
@@ -116,41 +116,66 @@ class SequenceData:
116 prompt_token_ids: List[int],
117 output_token_ids: Optional[List[int]] = None,
118 ) -> None:
119 - if output_token_ids is None:
120 - output_token_ids = []
119 + self._prompt_token_ids: List[int] = list(prompt_token_ids)
120 + self._prompt_token_ids_tuple: Tuple[int, ...] = tuple(prompt_token_ids)
121 + self._output_token_ids: List[int] = (
122 + list(output_token_ids) if output_token_ids is not None else [])
123
122 - self.prompt_token_ids = prompt_token_ids
123 - self._prompt_token_ids_tuple = tuple(prompt_token_ids)
124 - self.output_token_ids = output_token_ids
124 self.cumulative_logprob = 0.0
125 # The number of tokens that are computed (that run against the model).
126 self._num_computed_tokens = 0
127 self._stage: SequenceStage = SequenceStage.PREFILL
128
129 + self._update_cached_all_tokens()
130 +
131 + def _update_cached_all_tokens(self):
132 + self._cached_all_token_ids: List[int] = (self._prompt_token_ids +
133 + self._output_token_ids)
134 +
135 + @property
136 + def prompt_token_ids(self) -> Tuple[int, ...]:
137 + return self._prompt_token_ids_tuple
138 +
139 + @prompt_token_ids.setter
140 + def prompt_token_ids(self, new_prompt_token_ids) -> None:
141 + self._prompt_token_ids = list(new_prompt_token_ids)
142 + self._prompt_token_ids_tuple = tuple(new_prompt_token_ids)
143 + self._update_cached_all_tokens()
144 +
145 + @property
146 + def output_token_ids(self) -> Tuple[int, ...]:
147 + return tuple(self._output_token_ids)
148 +
149 + @output_token_ids.setter
150 + def output_token_ids(self, new_output_token_ids) -> None:
151 + self._output_token_ids = list(new_output_token_ids)
152 + self._update_cached_all_tokens()
153 +
154 def append_token_id(self, token_id: int, logprob: float) -> None:
131 - self.output_token_ids.append(token_id)
155 + self._output_token_ids.append(token_id)
156 + self._cached_all_token_ids.append(token_id)
157 self.cumulative_logprob += logprob
158
159 def get_len(self) -> int:
135 - return len(self.output_token_ids) + len(self.prompt_token_ids)
160 + return len(self._output_token_ids) + len(self._prompt_token_ids)
161
162 def get_prompt_len(self) -> int:
138 - return len(self.prompt_token_ids)
163 + return len(self._prompt_token_ids)
164
165 def get_output_len(self) -> int:
141 - return len(self.output_token_ids)
166 + return len(self._output_token_ids)
167
168 def get_token_ids(self) -> List[int]:
144 - return self.prompt_token_ids + self.output_token_ids
169 + return self._cached_all_token_ids
170
171 def get_prefix_token_ids(
172 self, num_tokens: int
173 ) -> Tuple[Tuple[int, ...], Optional[Tuple[int, ...]]]:
174 """Get prefix tokens, and make the return value hashable"""
150 - prompt_length = len(self.prompt_token_ids)
175 + prompt_length = self.get_prompt_len()
176 if num_tokens > prompt_length:
177 return (self._prompt_token_ids_tuple,
153 - tuple(self.output_token_ids[:num_tokens - prompt_length]))
178 + tuple(self._output_token_ids[:num_tokens - prompt_length]))
179 else:
180 return (self._prompt_token_ids_tuple[:num_tokens], None)
181
@@ -183,14 +208,14 @@ class SequenceData:
208 return self.get_len() - self.get_num_computed_tokens()
209
210 def get_last_token_id(self) -> int:
186 - if not self.output_token_ids:
187 - return self.prompt_token_ids[-1]
188 - return self.output_token_ids[-1]
211 + if not self._output_token_ids:
212 + return self._prompt_token_ids[-1]
213 + return self._output_token_ids[-1]
214
190 - def get_prompt_token_ids(self) -> List[int]:
215 + def get_prompt_token_ids(self) -> Tuple[int, ...]:
216 return self.prompt_token_ids
217
193 - def get_output_token_ids(self) -> List[int]:
218 + def get_output_token_ids(self) -> Tuple[int, ...]:
219 return self.output_token_ids
220
221 @property
@@ -199,8 +224,8 @@ class SequenceData:
224
225 def __repr__(self) -> str:
226 return (f"SequenceData("
202 - f"prompt_token_ids={self.prompt_token_ids}, "
203 - f"output_token_ids={self.output_token_ids}, "
227 + f"prompt_token_ids={self._prompt_token_ids}, "
228 + f"output_token_ids={self._output_token_ids}, "
229 f"cumulative_logprob={self.cumulative_logprob})")
230
231
@@ -306,14 +331,14 @@ class Sequence:
331 def get_token_ids(self) -> List[int]:
332 return self.data.get_token_ids()
333
309 - def get_prompt_token_ids(self) -> List[int]:
334 + def get_prompt_token_ids(self) -> Tuple[int, ...]:
335 return self.data.get_prompt_token_ids()
336
337 def get_last_token_id(self) -> int:
338 return self.data.get_last_token_id()
339
315 - def get_output_token_ids(self) -> List[int]:
316 - return self.data.output_token_ids
340 + def get_output_token_ids(self) -> Tuple[int, ...]:
341 + return self.data.get_output_token_ids()
342
343 def get_cumulative_logprob(self) -> float:
344 return self.data.cumulative_logprob