A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[Core] Optimize block_manager_v2 vs block_manager_v1 (to make V2 default) (#5602)
Alexander Matveev committed
Jul 1, 2024 at 23:10 UTC
3476ed0809ec91a3457da0cb90543133a4f4b519
19 files changed
+1189
-532
benchmarks/benchmark_latency.py
+4
@@ -46,6 +46,7 @@ def main(args: argparse.Namespace):
46
load_format=args.load_format,
47
distributed_executor_backend=args.distributed_executor_backend,
48
otlp_traces_endpoint=args.otlp_traces_endpoint,
49
+ enable_prefix_caching=args.enable_prefix_caching,
50
)
51
52
sampling_params = SamplingParams(
@@ -220,6 +221,9 @@ if __name__ == '__main__':
221
action='store_true',
222
help='If True, the prefill requests can be chunked based on the '
223
'max_num_batched_tokens')
224
+ parser.add_argument("--enable-prefix-caching",
225
+ action='store_true',
226
+ help="Enable automatic prefix caching")
227
parser.add_argument('--use-v2-block-manager', action='store_true')
228
parser.add_argument(
229
"--ray-workers-use-nsight",
tests/conftest.py
+1
-1
@@ -474,7 +474,7 @@ class VllmRunner:
474
req_sample_output_strs: List[str] = []
475
for sample in req_output.outputs:
476
output_str = sample.text
477
- output_ids = sample.token_ids
477
+ output_ids = list(sample.token_ids)
478
req_sample_output_ids.append(prompt_ids + output_ids)
479
req_sample_output_strs.append(prompt_str + output_str)
480
outputs.append((req_sample_output_ids, req_sample_output_strs))
tests/core/block/test_block_table.py
+3
-2
@@ -373,8 +373,9 @@ def test_cow(block_size: int, sequence_len: int, append_len: int,
373
block_size) - (sequence_len // block_size)
374
375
original_block_table.allocate(token_ids=token_ids, device=Device.GPU)
376
- original_block_ids = original_block_table.physical_block_ids
376
+ original_block_ids = original_block_table.physical_block_ids[:]
377
378
+ print("original_block_ids = {}".format(original_block_ids))
379
forked_block_table = original_block_table.fork()
380
381
# Expect no additional allocation (copy on _write_).
@@ -457,7 +458,7 @@ def test_cow_lookahead_simple(block_size: int, sequence_len: int,
458
459
# Allocate lookahead slots.
460
original_block_table.ensure_num_empty_slots(lookahead_slots)
460
- original_block_ids = original_block_table.physical_block_ids
461
+ original_block_ids = original_block_table.physical_block_ids[:]
462
463
forked_block_table = original_block_table.fork()
464
tests/core/block/test_cpu_gpu_block_allocator.py
+12
-12
@@ -8,8 +8,8 @@ from vllm.utils import Device, chunk_list
8
@pytest.mark.parametrize("num_gpu_blocks", [1024])
9
@pytest.mark.parametrize("block_size", [16])
10
@pytest.mark.parametrize("allocator_type", ["naive", "prefix_caching"])
11
-def test_allocate_mutable(num_cpu_blocks: int, num_gpu_blocks: int,
12
- block_size: int, allocator_type: str):
11
+def test_allocate_mutable_block(num_cpu_blocks: int, num_gpu_blocks: int,
12
+ block_size: int, allocator_type: str):
13
allocator = CpuGpuBlockAllocator.create(
14
allocator_type=allocator_type,
15
num_gpu_blocks=num_gpu_blocks,
@@ -21,14 +21,14 @@ def test_allocate_mutable(num_cpu_blocks: int, num_gpu_blocks: int,
21
assert allocator.get_num_free_blocks(Device.GPU) == num_gpu_blocks
22
23
cpu_blocks = [
24
- allocator.allocate_mutable(prev_block=None, device=Device.CPU)
24
+ allocator.allocate_mutable_block(prev_block=None, device=Device.CPU)
25
for _ in range(num_cpu_blocks)
26
]
27
assert allocator.get_num_free_blocks(Device.CPU) == 0
28
assert allocator.get_num_free_blocks(Device.GPU) == num_gpu_blocks
29
30
gpu_blocks = [
31
- allocator.allocate_mutable(prev_block=None, device=Device.GPU)
31
+ allocator.allocate_mutable_block(prev_block=None, device=Device.GPU)
32
for _ in range(num_gpu_blocks)
33
]
34
assert allocator.get_num_free_blocks(Device.CPU) == 0
@@ -47,8 +47,8 @@ def test_allocate_mutable(num_cpu_blocks: int, num_gpu_blocks: int,
47
@pytest.mark.parametrize("num_gpu_blocks", [1024])
48
@pytest.mark.parametrize("block_size", [2])
49
@pytest.mark.parametrize("allocator_type", ["naive", "prefix_caching"])
50
-def test_allocate_immutable(num_cpu_blocks: int, num_gpu_blocks: int,
51
- block_size: int, allocator_type: str):
50
+def test_allocate_immutable_block(num_cpu_blocks: int, num_gpu_blocks: int,
51
+ block_size: int, allocator_type: str):
52
allocator = CpuGpuBlockAllocator.create(
53
allocator_type=allocator_type,
54
num_gpu_blocks=num_gpu_blocks,
@@ -67,18 +67,18 @@ def test_allocate_immutable(num_cpu_blocks: int, num_gpu_blocks: int,
67
assert allocator.get_num_free_blocks(Device.GPU) == num_gpu_blocks
68
69
cpu_blocks = [
70
- allocator.allocate_immutable(prev_block=None,
71
- token_ids=token_ids,
72
- device=Device.CPU)
70
+ allocator.allocate_immutable_block(prev_block=None,
71
+ token_ids=token_ids,
72
+ device=Device.CPU)
73
for token_ids in cpu_token_ids
74
]
75
assert allocator.get_num_free_blocks(Device.CPU) == 0
76
assert allocator.get_num_free_blocks(Device.GPU) == num_gpu_blocks
77
78
gpu_blocks = [
79
- allocator.allocate_immutable(prev_block=None,
80
- token_ids=token_ids,
81
- device=Device.GPU)
79
+ allocator.allocate_immutable_block(prev_block=None,
80
+ token_ids=token_ids,
81
+ device=Device.GPU)
82
for token_ids in gpu_token_ids
83
]
84
assert allocator.get_num_free_blocks(Device.CPU) == 0
tests/core/block/test_naive_block.py
+3
-3
@@ -14,11 +14,11 @@ class TestNaiveBlockAllocator:
14
prev_block: Optional[Block],
15
token_ids: List[int]):
16
if allocate_type == "immutable":
17
- allocate_block = lambda: allocator.allocate_immutable(
17
+ allocate_block = lambda: allocator.allocate_immutable_block(
18
prev_block=prev_block, token_ids=token_ids)
19
elif allocate_type == "mutable":
20
- allocate_block = lambda: allocator.allocate_mutable(prev_block=
21
- prev_block)
20
+ allocate_block = lambda: allocator.allocate_mutable_block(
21
+ prev_block=prev_block)
22
else:
23
raise ValueError()
24
tests/core/block/test_prefix_caching_block.py
+67
-39
@@ -26,11 +26,10 @@ class TestPrefixCachingBlock:
26
token_ids = list(range(num_to_fill))
27
mock_allocator = MagicMock(spec=PrefixCachingBlockAllocator)
28
29
- block_with_prev = PrefixCachingBlock(
30
- prev_block=None,
31
- token_ids=token_ids,
32
- block_size=block_size,
33
- prefix_caching_allocator=mock_allocator)
29
+ block_with_prev = PrefixCachingBlock(prev_block=None,
30
+ token_ids=token_ids,
31
+ block_size=block_size,
32
+ allocator=mock_allocator)
33
34
if is_curr_block_full:
35
# Expect hash since block is full.
@@ -71,7 +70,7 @@ class TestPrefixCachingBlock:
70
prev_block=previous_block,
71
token_ids=token_ids,
72
block_size=block_size,
74
- prefix_caching_allocator=mock_allocator,
73
+ allocator=mock_allocator,
74
)
75
76
if is_curr_block_full and prev_block_has_hash:
@@ -138,7 +137,7 @@ class TestPrefixCachingBlock:
137
prev_block=prev_block,
138
token_ids=[],
139
block_size=block_size,
141
- prefix_caching_allocator=allocator,
140
+ allocator=allocator,
141
)
142
143
tokens_to_append = token_ids[block_number *
@@ -159,11 +158,11 @@ class TestPrefixCachingBlockAllocator:
158
prev_block: Optional[Block],
159
token_ids: List[int]):
160
if allocate_type == "immutable":
162
- allocate_block = lambda: allocator.allocate_immutable(
161
+ allocate_block = lambda: allocator.allocate_immutable_block(
162
prev_block=prev_block, token_ids=token_ids)
163
elif allocate_type == "mutable":
165
- allocate_block = lambda: allocator.allocate_mutable(prev_block=
166
- prev_block)
164
+ allocate_block = lambda: allocator.allocate_mutable_block(
165
+ prev_block=prev_block)
166
else:
167
raise ValueError()
168
@@ -233,12 +232,13 @@ class TestPrefixCachingBlockAllocator:
232
233
# Expect allocation with unseen hash to fail.
234
with pytest.raises(BlockAllocator.NoFreeBlocksError):
236
- allocator.allocate_immutable(prev_block=chain[-1],
237
- token_ids=list(range(block_size)))
235
+ allocator.allocate_immutable_block(prev_block=chain[-1],
236
+ token_ids=list(
237
+ range(block_size)))
238
239
# Expect mutable allocation to fail.
240
with pytest.raises(BlockAllocator.NoFreeBlocksError):
241
- allocator.allocate_mutable(prev_block=chain[-1])
241
+ allocator.allocate_mutable_block(prev_block=chain[-1])
242
243
# Expect allocation of exact same chain to pass.
244
second_chain = TestPrefixCachingBlockAllocator.create_immutable_chain(
@@ -270,7 +270,7 @@ class TestPrefixCachingBlockAllocator:
270
271
# Expect mutable allocation to fail.
272
with pytest.raises(BlockAllocator.NoFreeBlocksError):
273
- allocator.allocate_mutable(prev_block=None)
273
+ allocator.allocate_mutable_block(prev_block=None)
274
275
block_to_free = chain[-1]
276
@@ -280,11 +280,11 @@ class TestPrefixCachingBlockAllocator:
280
allocator.free(block_to_free)
281
assert block_to_free.block_id is None, i
282
283
- new_block = allocator.allocate_mutable(prev_block=None)
283
+ new_block = allocator.allocate_mutable_block(prev_block=None)
284
assert new_block.block_id == block_id, i
285
286
with pytest.raises(BlockAllocator.NoFreeBlocksError):
287
- allocator.allocate_mutable(prev_block=None)
287
+ allocator.allocate_mutable_block(prev_block=None)
288
289
block_to_free = new_block
290
@@ -376,7 +376,6 @@ class TestPrefixCachingBlockAllocator:
376
377
# Create token ids that will exhaust all blocks.
378
token_ids = list(range(num_blocks_to_consume * block_size))
379
- blocks = list(range(num_blocks_to_consume))
379
380
first_chain = TestPrefixCachingBlockAllocator.create_immutable_chain(
381
block_size=block_size,
@@ -384,9 +383,6 @@ class TestPrefixCachingBlockAllocator:
383
allocator=allocator,
384
)
385
387
- # mark all blocks in first chain as computed
388
- allocator.mark_blocks_as_computed(blocks)
389
-
386
# After zero_point, second_chain's token_ids would be set -1, which
387
# make it different from here comparing with first_chain
388
zero_point = random.randint(1, len(token_ids) - 1)
@@ -424,15 +420,16 @@ class TestPrefixCachingBlockAllocator:
420
block_size=block_size)
421
token_ids = list(range(block_size))
422
427
- block = allocator.allocate_immutable(prev_block=None,
428
- token_ids=token_ids)
423
+ block = allocator.allocate_immutable_block(prev_block=None,
424
+ token_ids=token_ids)
425
426
assert allocator._refcounter.get(block.block_id) == 1
431
- m = allocator.allocate_mutable(prev_block=None)
427
+ m = allocator.allocate_mutable_block(prev_block=None)
428
429
block_id = m.block_id
430
for i in range(block_size):
431
m.append_token_ids([i])
432
+
433
# After block get promoted to immutable from mutable, if there is
434
# already same content hash block, then it shall be released into
435
# hashless_allocator
@@ -452,48 +449,79 @@ class TestPrefixCachingBlockAllocator:
449
450
all_blocks_list = [i for i in range(num_blocks)]
451
zero_ref = {i: 0 for i in range(num_blocks)}
452
+ one_ref = {i: 1 for i in range(num_blocks)}
453
allocator = PrefixCachingBlockAllocator(num_blocks=num_blocks,
454
block_size=block_size)
455
token_ids = list(range(num_blocks * block_size))
456
459
- # now we have num_blocks free blocks in hashless allocator
460
- # with internal tracking list _blocks _cached_blocks and evictor
461
- # empty and block's ref shall be 0
457
+ # Verify initial/pre-alloc state
458
+
459
+ # Ensure all blocks are free inside hashless allocator
460
assert list(allocator._hashless_allocator._free_block_indices
461
) == all_blocks_list
464
- assert len(allocator._blocks.keys()) == 0
462
+ # Ensure no tracked blocks
463
+ assert len(allocator._block_tracker.keys()) == num_blocks
464
+ for block_id in range(num_blocks):
465
+ assert not allocator._block_tracker[block_id].active
466
+ # Ensure no cached blocks
467
assert len(allocator._cached_blocks.values()) == 0
468
+ # Ensure no evicted blocks
469
assert len(allocator.evictor.free_table.keys()) == 0
470
+ # Ensure 0s ref counts for all blocks
471
assert allocator._refcounter._refcounts == zero_ref
472
473
# Allocate immutable chains with only one block residuled in
474
new_block = []
475
for i in range(num_blocks):
472
- block = allocator.allocate_immutable(
476
+ block = allocator.allocate_immutable_block(
477
prev_block=None,
478
token_ids=token_ids[block_size * i:block_size * (i + 1)])
479
new_block.append(block)
480
481
+ # Verify post-alloc state
482
+
483
+ # Ensure no blocks are free inside hashless allocator
484
+ assert (len(allocator._hashless_allocator._free_block_indices) == 0)
485
+ # Ensure all blocks are tracked
486
+ assert len(allocator._block_tracker.keys()) == num_blocks
487
+ for block_id in range(num_blocks):
488
+ assert allocator._block_tracker[block_id].active
489
+ # Ensure all blocks are cached (all promoted)
490
+ assert len(allocator._cached_blocks.values()) == num_blocks
491
+ # Ensure no evicted blocks
492
+ assert len(allocator.evictor.free_table.keys()) == 0
493
+ # Ensure 1s ref counts for all blocks
494
+ assert allocator._refcounter._refcounts == one_ref
495
+
496
# Free all blocks, and now all blocks shall be in the evictor
478
- # there shall be no tracking data left in _blocks
497
+ # there shall be no tracking data left in _block_tracker
498
# all blocks shall be tracked in _cached_blocks
499
# all blocks' ref shall be zero
500
for block in new_block:
501
allocator.free(block)
502
484
- assert len(allocator._blocks.keys()) == 0
503
+ # Verify post-free state
504
+
505
+ # Ensure no tracked blocks
506
+ assert len(allocator._block_tracker.keys()) == num_blocks
507
+ for block_id in range(num_blocks):
508
+ assert not allocator._block_tracker[block_id].active
509
+ # Ensure no blocks in hashless allocator (all promoted)
510
assert len(allocator._hashless_allocator._free_block_indices) == 0
511
+ # Ensure all blocks are cached
512
assert list(allocator._cached_blocks.values()) == all_blocks_list
513
+ # Ensure all blocks are inside the evictor
514
assert list(allocator.evictor.free_table.keys()) == all_blocks_list
515
+ # Ensure 0s refcounts
516
assert allocator._refcounter._refcounts == zero_ref
517
518
# Allocate a mutable block, and the first block shall be evicted
519
# and set its content hash into None, ref to 1
492
- mutable = allocator.allocate_mutable(prev_block=None)
520
+ mutable = allocator.allocate_mutable_block(prev_block=None)
521
522
assert mutable.block_id == 0
523
assert mutable.content_hash is None
496
- assert 0 in allocator._blocks
524
+ assert allocator._block_tracker[0].active
525
assert allocator._refcounter.get(0) == 1
526
assert 0 not in allocator._cached_blocks
527
assert 0 not in allocator.evictor
@@ -502,27 +530,27 @@ class TestPrefixCachingBlockAllocator:
530
# hashless allocator
531
allocator.free(mutable)
532
505
- assert len(allocator._blocks.keys()) == 0
533
+ assert not allocator._block_tracker[0].active
534
assert allocator._refcounter._refcounts == zero_ref
535
assert 0 not in allocator._cached_blocks
536
assert 0 not in allocator.evictor
537
assert 0 in allocator._hashless_allocator._free_block_indices
538
511
- # when allocate immutable with first block_size tokens, we
539
+ # When allocate immutable with first block_size tokens, we
540
# shall get free block from hashless allocator, thus no block left
541
# in hashless
514
- block = allocator.allocate_immutable(prev_block=None,
515
- token_ids=token_ids[:block_size])
542
+ block = allocator.allocate_immutable_block(
543
+ prev_block=None, token_ids=token_ids[:block_size])
544
545
assert block.block_id == 0
546
assert len(allocator._hashless_allocator._free_block_indices) == 0
519
- assert 0 in allocator._blocks
547
+ assert allocator._block_tracker[0].active
548
assert 0 in allocator._cached_blocks.values()
549
assert allocator._refcounter.get(0) == 1
550
assert 0 not in allocator.evictor
551
552
# allocate mutable block again, it shall be popped from evictor
525
- mutable = allocator.allocate_mutable(prev_block=None)
553
+ mutable = allocator.allocate_mutable_block(prev_block=None)
554
assert len(allocator._hashless_allocator._free_block_indices) == 0
555
assert mutable.block_id not in allocator.evictor.free_table
556
assert allocator._refcounter.get(mutable.block_id) == 1
@@ -619,7 +647,7 @@ class TestPrefixCachingBlockAllocator:
647
block_token_ids = token_ids[block_number *
648
block_size:(block_number + 1) *
649
block_size]
622
- prev_block = allocator.allocate_immutable(
650
+ prev_block = allocator.allocate_immutable_block(
651
prev_block=prev_block, token_ids=block_token_ids)
652
blocks.append(prev_block)
653
tests/spec_decode/test_batch_expansion.py
+4
-4
@@ -90,10 +90,10 @@ def test_create_single_target_seq_group_metadata(k: int):
90
91
assert output.request_id == input_seq_group_metadata.request_id
92
assert len(output.seq_data) == 1
93
- assert output.seq_data[target_seq_id].get_prompt_token_ids(
94
- ) == prompt_tokens
95
- assert output.seq_data[target_seq_id].get_output_token_ids(
96
- ) == prev_output_tokens + token_ids
93
+ assert output.seq_data[target_seq_id].get_prompt_token_ids() == tuple(
94
+ prompt_tokens)
95
+ assert output.seq_data[target_seq_id].get_output_token_ids() == tuple(
96
+ prev_output_tokens + token_ids)
97
98
assert len(output.block_tables) == 1
99
assert output.block_tables[
vllm/core/block/block_table.py
+56
-29
@@ -1,5 +1,6 @@
1
from typing import List, Optional
2
3
+from vllm.core.block.common import BlockList
4
from vllm.core.block.interfaces import Block, DeviceAwareBlockAllocator
5
from vllm.utils import Device, cdiv, chunk_list
6
@@ -47,12 +48,10 @@ class BlockTable:
48
self._allocator = block_allocator
49
if _blocks is None:
50
_blocks = []
50
- self._blocks: List[Block] = _blocks
51
+ self._blocks: BlockList = BlockList(_blocks)
52
53
self._max_block_sliding_window = max_block_sliding_window
53
- # Use helper method instead of directly calculating, as blocks
54
- # may not be allocated.
55
- self._num_full_slots = len(self._get_all_token_ids())
54
+ self._num_full_slots = self._get_num_token_ids()
55
56
@staticmethod
57
def get_num_required_blocks(token_ids: List[int], block_size: int) -> int:
@@ -88,11 +87,18 @@ class BlockTable:
87
"""
88
assert not self._is_allocated
89
assert token_ids
91
- self._blocks = self._allocate_blocks_for_token_ids(prev_block=None,
92
- token_ids=token_ids,
93
- device=device)
90
+ blocks = self._allocate_blocks_for_token_ids(prev_block=None,
91
+ token_ids=token_ids,
92
+ device=device)
93
+ self.update(blocks)
94
self._num_full_slots = len(token_ids)
95
96
+ def update(self, blocks: List[Block]) -> None:
97
+ """Resets the table to the newly provided blocks
98
+ (with their corresponding block ids)
99
+ """
100
+ self._blocks.update(blocks)
101
+
102
def append_token_ids(self,
103
token_ids: List[int],
104
num_lookahead_slots: int = 0,
@@ -140,11 +146,11 @@ class BlockTable:
146
num_lookahead_slots)
147
148
# Update the blocks with the new tokens
143
- blocks = self._blocks[self._num_full_slots // self._block_size:]
149
+ first_block_idx = self._num_full_slots // self._block_size
150
token_blocks = self._chunk_token_blocks_for_append(token_ids)
151
146
- for block, token_block in zip(blocks, token_blocks):
147
- block.append_token_ids(token_block)
152
+ for i, token_block in enumerate(token_blocks):
153
+ self._blocks.append_token_ids(first_block_idx + i, token_block)
154
155
self._num_full_slots += len(token_ids)
156
@@ -174,8 +180,8 @@ class BlockTable:
180
for _ in range(blocks_to_allocate):
181
assert len(self._blocks) > 0
182
self._blocks.append(
177
- self._allocator.allocate_mutable(prev_block=self._blocks[-1],
178
- device=device))
183
+ self._allocator.allocate_mutable_block(
184
+ prev_block=self._blocks[-1], device=device))
185
186
def fork(self) -> "BlockTable":
187
"""Creates a new BlockTable instance with a copy of the blocks from the
@@ -209,12 +215,12 @@ class BlockTable:
215
is set to `None`.
216
"""
217
assert self._is_allocated
212
- for block in self._blocks:
218
+ for block in self.blocks:
219
self._allocator.free(block)
214
- self._blocks = []
220
+ self._blocks.reset()
221
222
@property
217
- def physical_block_ids(self) -> List[Optional[int]]:
223
+ def physical_block_ids(self) -> List[int]:
224
"""Returns a list of physical block indices for the blocks in the
225
BlockTable.
226
@@ -228,7 +234,7 @@ class BlockTable:
234
BlockTable.
235
"""
236
assert self._is_allocated
231
- return [block.block_id for block in self._blocks]
237
+ return self._blocks.ids()
238
239
def get_unseen_token_ids(self, sequence_token_ids: List[int]) -> List[int]:
240
"""Get the number of "unseen" tokens in the sequence.
@@ -253,17 +259,31 @@ class BlockTable:
259
token_ids: List[int],
260
device: Device) -> List[Block]:
261
blocks: List[Block] = []
256
- for block_token_ids in chunk_list(token_ids, self._block_size):
257
- if len(block_token_ids) == self._block_size:
258
- # If the block is full, create an immutable block.
259
- prev_block = self._allocator.allocate_immutable(
260
- prev_block, token_ids=block_token_ids, device=device)
262
+
263
+ block_token_ids = []
264
+ tail_token_ids = []
265
+ for cur_token_ids in chunk_list(token_ids, self._block_size):
266
+ if len(cur_token_ids) == self._block_size:
267
+ block_token_ids.append(cur_token_ids)
268
else:
262
- # Else, partially fill a mutable block with token ids.
263
- prev_block = self._allocator.allocate_mutable(
264
- prev_block=prev_block, device=device)
265
- prev_block.append_token_ids(block_token_ids)
266
- blocks.append(prev_block)
269
+ tail_token_ids.append(cur_token_ids)
270
+
271
+ if block_token_ids:
272
+ blocks.extend(
273
+ self._allocator.allocate_immutable_blocks(
274
+ prev_block, block_token_ids=block_token_ids,
275
+ device=device))
276
+ prev_block = blocks[-1]
277
+
278
+ if tail_token_ids:
279
+ assert len(tail_token_ids) == 1
280
+ cur_token_ids = tail_token_ids[0]
281
+
282
+ block = self._allocator.allocate_mutable_block(
283
+ prev_block=prev_block, device=device)
284
+ block.append_token_ids(cur_token_ids)
285
+
286
+ blocks.append(block)
287
288
return blocks
289
@@ -274,18 +294,25 @@ class BlockTable:
294
if not self._is_allocated:
295
return token_ids
296
277
- for block in self._blocks:
297
+ for block in self.blocks:
298
token_ids.extend(block.token_ids)
299
300
return token_ids
301
302
+ def _get_num_token_ids(self) -> int:
303
+ res = 0
304
+ for block in self.blocks:
305
+ res += len(block.token_ids)
306
+
307
+ return res
308
+
309
@property
310
def _is_allocated(self) -> bool:
311
return len(self._blocks) > 0
312
313
@property
287
- def blocks(self) -> Optional[List[Block]]:
288
- return self._blocks
314
+ def blocks(self) -> List[Block]:
315
+ return self._blocks.list()
316
317
@property
318
def _num_empty_slots(self) -> int:
vllm/core/block/common.py
+154
-44
@@ -1,4 +1,5 @@
1
-from typing import Dict, Iterable, List, Optional, Protocol, Tuple
1
+from collections import deque
2
+from typing import Deque, Dict, Iterable, List, Optional, Protocol, Tuple
3
4
from vllm.core.block.interfaces import Block, BlockAllocator
5
@@ -95,64 +96,40 @@ class CopyOnWriteTracker:
96
97
The CopyOnWriteTracker class maintains a mapping of source block indices to
98
their corresponding copy-on-write destination block indices. It works in
98
- conjunction with a RefCounter and a BlockAllocator to handle reference
99
- counting and block allocation.
99
+ conjunction with a RefCounter.
100
101
Args:
102
refcounter (RefCounter): The reference counter used to track block
103
reference counts.
104
- allocator (BlockAllocator): The block allocator used to allocate and
105
- free blocks.
104
"""
105
108
- def __init__(
109
- self,
110
- refcounter: RefCounterProtocol,
111
- allocator: BlockAllocator,
112
- ):
106
+ def __init__(self, refcounter: RefCounterProtocol):
107
self._copy_on_writes: List[Tuple[BlockId, BlockId]] = []
108
self._refcounter = refcounter
115
- self._allocator = allocator
116
-
117
- def cow_block_if_not_appendable(self, block: Block) -> Optional[BlockId]:
118
- """Performs a copy-on-write operation on the given block if it is not
119
- appendable.
120
-
121
- This method checks the reference count of the given block. If the
122
- reference count is greater than 1, indicating that the block is shared,
123
- a copy-on-write operation is performed. The original block is freed,
124
- and a new block is allocated with the same content. The new block index
125
- is returned.
126
-
127
- Args:
128
- block (Block): The block to check for copy-on-write.
109
130
- Returns:
131
- Optional[BlockId]: The block index of the new block if a copy-on
132
- -write operation was performed, or the original block index if
133
- no copy-on-write was necessary.
110
+ def is_appendable(self, block: Block) -> bool:
111
+ """Checks if the block is shared or not. If shared, then it cannot
112
+ be appended and needs to be duplicated via copy-on-write
113
"""
114
block_id = block.block_id
115
if block_id is None:
137
- return block_id
116
+ return True
117
118
refcount = self._refcounter.get(block_id)
140
- assert refcount != 0
141
- if refcount > 1:
142
- src_block_id = block_id
143
- # Decrement refcount of the old block.
144
- self._allocator.free(block)
145
-
146
- # Allocate a fresh new block.
147
- block_id = self._allocator.allocate_mutable(
148
- prev_block=block.prev_block).block_id
119
+ return refcount <= 1
120
150
- # Track src/dst copy.
151
- assert src_block_id is not None
152
- assert block_id is not None
153
- self._copy_on_writes.append((src_block_id, block_id))
154
-
155
- return block_id
121
+ def record_cow(self, src_block_id: Optional[BlockId],
122
+ trg_block_id: Optional[BlockId]) -> None:
123
+ """Records a copy-on-write operation from source to target block id
124
+ Args:
125
+ src_block_id (BlockId): The source block id from which to copy
126
+ the data
127
+ trg_block_id (BlockId): The target block id to which the data
128
+ is copied
129
+ """
130
+ assert src_block_id is not None
131
+ assert trg_block_id is not None
132
+ self._copy_on_writes.append((src_block_id, trg_block_id))
133
134
def clear_cows(self) -> List[Tuple[BlockId, BlockId]]:
135
"""Clears the copy-on-write tracking information and returns the current
@@ -172,6 +149,139 @@ class CopyOnWriteTracker:
149
return cows
150
151
152
+class BlockPool:
153
+ """Used to pre-allocate block objects, in order to avoid excessive python
154
+ object allocations/deallocations.
155
+ The pool starts from "pool_size" objects and will increase to more objects
156
+ if necessary
157
+
158
+ Note that multiple block objects may point to the same physical block id,
159
+ which is why this pool is needed, so that it will be easier to support
160
+ prefix caching and more complicated sharing of physical blocks.
161
+ """
162
+
163
+ def __init__(self, block_size: int, create_block: Block.Factory,
164
+ allocator: BlockAllocator, pool_size: int):
165
+ self._block_size = block_size
166
+ self._create_block = create_block
167
+ self._allocator = allocator
168
+ self._pool_size = pool_size
169
+ assert self._pool_size >= 0
170
+
171
+ self._free_ids: Deque[int] = deque(range(self._pool_size))
172
+ self._pool = []
173
+ for i in range(self._pool_size):
174
+ self._pool.append(
175
+ self._create_block(prev_block=None,
176
+ token_ids=[],
177
+ block_size=self._block_size,
178
+ allocator=self._allocator,
179
+ block_id=None))
180
+
181
+ def increase_pool(self):
182
+ """Doubles the internal pool size
183
+ """
184
+ cur_pool_size = self._pool_size
185
+ new_pool_size = cur_pool_size * 2
186
+ self._pool_size = new_pool_size
187
+
188
+ self._free_ids += deque(range(cur_pool_size, new_pool_size))
189
+
190
+ for i in range(cur_pool_size, new_pool_size):
191
+ self._pool.append(
192
+ self._create_block(prev_block=None,
193
+ token_ids=[],
194
+ block_size=self._block_size,
195
+ allocator=self._allocator,
196
+ block_id=None))
197
+
198
+ def init_block(self, prev_block: Optional[Block], token_ids: List[int],
199
+ block_size: int, physical_block_id: Optional[int]) -> Block:
200
+ if len(self._free_ids) == 0:
201
+ self.increase_pool()
202
+ assert len(self._free_ids) > 0
203
+
204
+ pool_id = self._free_ids.popleft()
205
+
206
+ block = self._pool[pool_id]
207
+ block.__init__( # type: ignore[misc]
208
+ prev_block=prev_block,
209
+ token_ids=token_ids,
210
+ block_size=block_size,
211
+ allocator=block._allocator, # type: ignore[attr-defined]
212
+ block_id=physical_block_id)
213
+ block.pool_id = pool_id # type: ignore[attr-defined]
214
+ return block
215
+
216
+ def free_block(self, block: Block) -> None:
217
+ self._free_ids.appendleft(block.pool_id) # type: ignore[attr-defined]
218
+
219
+
220
+class BlockList:
221
+ """This class is an optimization to allow fast-access to physical
222
+ block ids. It maintains a block id list that is updated with the
223
+ block list and this avoids the need to reconstruct the block id
224
+ list on every iteration of the block manager
225
+ """
226
+
227
+ def __init__(self, blocks: List[Block]):
228
+ self._blocks: List[Block] = []
229
+ self._block_ids: List[int] = []
230
+
231
+ self.update(blocks)
232
+
233
+ def _add_block_id(self, block_id: Optional[BlockId]) -> None:
234
+ assert block_id is not None
235
+ self._block_ids.append(block_id)
236
+
237
+ def _update_block_id(self, block_index: int,
238
+ new_block_id: Optional[BlockId]) -> None:
239
+ assert new_block_id is not None
240
+ self._block_ids[block_index] = new_block_id
241
+
242
+ def update(self, blocks: List[Block]):
243
+ self._blocks = blocks
244
+
245
+ # Cache block ids for fast query
246
+ self._block_ids = []
247
+ for block in self._blocks:
248
+ self._add_block_id(block.block_id)
249
+
250
+ def append_token_ids(self, block_index: int, token_ids: List[int]) -> None:
251
+ block = self._blocks[block_index]
252
+ prev_block_id = block.block_id
253
+
254
+ block.append_token_ids(token_ids)
255
+
256
+ # CoW or promotion may update the internal block_id
257
+ if prev_block_id != block.block_id:
258
+ self._update_block_id(block_index, block.block_id)
259
+
260
+ def append(self, new_block: Block):
261
+ self._blocks.append(new_block)
262
+ self._add_block_id(new_block.block_id)
263
+
264
+ def __len__(self) -> int:
265
+ return len(self._blocks)
266
+
267
+ def __getitem__(self, block_index: int) -> Block:
268
+ return self._blocks[block_index]
269
+
270
+ def __setitem__(self, block_index: int, new_block: Block) -> None:
271
+ self._blocks[block_index] = new_block
272
+ self._update_block_id(block_index, new_block.block_id)
273
+
274
+ def reset(self):
275
+ self._blocks = []
276
+ self._block_ids = []
277
+
278
+ def list(self) -> List[Block]:
279
+ return self._blocks
280
+
281
+ def ids(self) -> List[int]:
282
+ return self._block_ids
283
+
284
+
285
def get_all_blocks_recursively(last_block: Block) -> List[Block]:
286
"""Retrieves all the blocks in a sequence starting from the last block.
287
vllm/core/block/cpu_gpu_block_allocator.py
+56
-28
@@ -113,11 +113,11 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
113
def allocate_or_get_null_block(self) -> Block:
114
if self._null_block is None:
115
self._null_block = NullBlock(
116
- self.allocate_mutable(None, Device.GPU))
116
+ self.allocate_mutable_block(None, Device.GPU))
117
return self._null_block
118
119
- def allocate_mutable(self, prev_block: Optional[Block],
120
- device: Device) -> Block:
119
+ def allocate_mutable_block(self, prev_block: Optional[Block],
120
+ device: Device) -> Block:
121
"""Allocates a new mutable block on the specified device.
122
123
Args:
@@ -128,10 +128,31 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
128
Returns:
129
Block: The newly allocated mutable block.
130
"""
131
- return self._allocators[device].allocate_mutable(prev_block)
131
+ return self._allocators[device].allocate_mutable_block(prev_block)
132
133
- def allocate_immutable(self, prev_block: Optional[Block],
134
- token_ids: List[int], device: Device) -> Block:
133
+ def allocate_immutable_blocks(self, prev_block: Optional[Block],
134
+ block_token_ids: List[List[int]],
135
+ device: Optional[Device]) -> List[Block]:
136
+ """Allocates a new group of immutable blocks with the provided block
137
+ token IDs on the specified device.
138
+
139
+ Args:
140
+ prev_block (Optional[Block]): The previous block in the sequence.
141
+ Used for prefix hashing.
142
+ block_token_ids (List[int]): The list of block token IDs to be
143
+ stored in the new blocks.
144
+ device (Device): The device on which to allocate the new block.
145
+
146
+ Returns:
147
+ List[Block]: The newly allocated list of immutable blocks
148
+ containing the provided block token IDs.
149
+ """
150
+ return self._allocators[device].allocate_immutable_blocks(
151
+ prev_block, block_token_ids)
152
+
153
+ def allocate_immutable_block(self, prev_block: Optional[Block],
154
+ token_ids: List[int],
155
+ device: Device) -> Block:
156
"""Allocates a new immutable block with the provided token IDs on the
157
specified device.
158
@@ -146,7 +167,7 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
167
Block: The newly allocated immutable block containing the provided
168
token IDs.
169
"""
149
- return self._allocators[device].allocate_immutable(
170
+ return self._allocators[device].allocate_immutable_block(
171
prev_block, token_ids)
172
173
def free(self, block: Block) -> None:
@@ -161,7 +182,7 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
182
block_id = block.block_id
183
assert block_id is not None
184
allocator = self._block_ids_to_allocator[block_id]
164
- return allocator.free(block)
185
+ allocator.free(block)
186
187
def fork(self, last_block: Block) -> List[Block]:
188
"""Creates a new sequence of blocks that shares the same underlying
@@ -210,8 +231,8 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
231
"""
232
return self._allocators[device].get_physical_block_id(absolute_id)
233
213
- def swap(self, blocks: List[Block], source_device: Device,
214
- dest_device: Device) -> Dict[int, int]:
234
+ def swap(self, blocks: List[Block], src_device: Device,
235
+ dst_device: Device) -> Dict[int, int]:
236
"""Execute the swap for the given blocks from source_device
237
on to dest_device, save the current swap mapping and append
238
them to the accumulated `self._swap_mapping` for each
@@ -219,23 +240,23 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
240
241
Args:
242
blocks: List of blocks to be swapped.
222
- source_device (Device): Device to swap the 'blocks' from.
223
- dest_device (Device): Device to swap the 'blocks' to.
243
+ src_device (Device): Device to swap the 'blocks' from.
244
+ dst_device (Device): Device to swap the 'blocks' to.
245
246
Returns:
247
Dict[int, int]: Swap mapping from source_device
248
on to dest_device.
249
"""
229
- source_block_ids = [block.block_id for block in blocks]
230
- self._allocators[source_device].swap_out(blocks)
231
- self._allocators[dest_device].swap_in(blocks)
232
- dest_block_ids = [block.block_id for block in blocks]
250
+ src_block_ids = [block.block_id for block in blocks]
251
+ self._allocators[src_device].swap_out(blocks)
252
+ self._allocators[dst_device].swap_in(blocks)
253
+ dst_block_ids = [block.block_id for block in blocks]
254
255
current_swap_mapping: Dict[int, int] = {}
235
- for src, dest in zip(source_block_ids, dest_block_ids):
236
- if src is not None and dest is not None:
237
- self._swap_mapping[src] = dest
238
- current_swap_mapping[src] = dest
256
+ for src_block_id, dst_block_id in zip(src_block_ids, dst_block_ids):
257
+ if src_block_id is not None and dst_block_id is not None:
258
+ self._swap_mapping[src_block_id] = dst_block_id
259
+ current_swap_mapping[src_block_id] = dst_block_id
260
return current_swap_mapping
261
262
def get_num_blocks_touched(self,
@@ -283,23 +304,25 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
304
device = Device.GPU
305
return self._allocators[device].mark_blocks_as_computed(block_ids)
306
307
+ def get_computed_block_ids(self, prev_computed_block_ids: List[int],
308
+ block_ids: List[int],
309
+ skip_last_block_id: bool) -> List[int]:
310
+ # Prefix caching only supported on GPU.
311
+ device = Device.GPU
312
+ return self._allocators[device].get_computed_block_ids(
313
+ prev_computed_block_ids, block_ids, skip_last_block_id)
314
+
315
def get_common_computed_block_ids(
287
- self, seq_block_ids: List[List[int]]) -> List[int]:
316
+ self, computed_seq_block_ids: List[List[int]]) -> List[int]:
317
# Prefix caching only supported on GPU.
318
device = Device.GPU
319
return self._allocators[device].get_common_computed_block_ids(
291
- seq_block_ids)
320
+ computed_seq_block_ids)
321
322
@property
323
def all_block_ids(self) -> FrozenSet[int]:
324
return frozenset(self._block_ids_to_allocator.keys())
325
297
- def promote_to_immutable_block(self, block: Block) -> BlockId:
298
- raise NotImplementedError
299
-
300
- def cow_block_if_not_appendable(self, block: Block) -> Optional[BlockId]:
301
- raise NotImplementedError
302
-
326
def get_and_reset_swaps(self) -> List[Tuple[int, int]]:
327
"""Returns and clears the mapping of source to destination block IDs.
328
Will be called after every swapping operations for now, and after every
@@ -341,6 +364,11 @@ class NullBlock(Block):
364
def token_ids(self) -> List[BlockId]:
365
return self._proxy.token_ids
366
367
+ @property
368
+ def num_tokens_total(self) -> int:
369
+ raise NotImplementedError(
370
+ "num_tokens_total is not used for null block")
371
+
372
@property
373
def num_empty_slots(self) -> BlockId:
374
return self._proxy.num_empty_slots
vllm/core/block/interfaces.py
+44
-12
@@ -28,6 +28,13 @@ class Block(ABC):
28
def token_ids(self) -> List[int]:
29
pass
30
31
+ @property
32
+ @abstractmethod
33
+ def num_tokens_total(self) -> int:
34
+ """The number of tokens till the current block (inclusive)
35
+ """
36
+ pass
37
+
38
@property
39
@abstractmethod
40
def num_empty_slots(self) -> int:
@@ -92,12 +99,18 @@ class Block(ABC):
99
class BlockAllocator(ABC):
100
101
@abstractmethod
95
- def allocate_mutable(self, prev_block: Optional[Block]) -> Block:
102
+ def allocate_mutable_block(self, prev_block: Optional[Block]) -> Block:
103
pass
104
105
@abstractmethod
99
- def allocate_immutable(self, prev_block: Optional[Block],
100
- token_ids: List[int]) -> Block:
106
+ def allocate_immutable_block(self, prev_block: Optional[Block],
107
+ token_ids: List[int]) -> Block:
108
+ pass
109
+
110
+ @abstractmethod
111
+ def allocate_immutable_blocks(
112
+ self, prev_block: Optional[Block],
113
+ block_token_ids: List[List[int]]) -> List[Block]:
114
pass
115
116
@abstractmethod
@@ -146,13 +159,19 @@ class BlockAllocator(ABC):
159
def mark_blocks_as_computed(self, block_ids: List[int]) -> None:
160
pass
161
162
+ @abstractmethod
163
+ def get_computed_block_ids(self, prev_computed_block_ids: List[int],
164
+ block_ids: List[int],
165
+ skip_last_block_id: bool) -> List[int]:
166
+ pass
167
+
168
@abstractmethod
169
def get_common_computed_block_ids(
151
- self, seq_block_ids: List[List[int]]) -> List[int]:
170
+ self, computed_seq_block_ids: List[List[int]]) -> List[int]:
171
pass
172
173
@abstractmethod
155
- def cow_block_if_not_appendable(self, block: Block) -> Optional["BlockId"]:
174
+ def cow_block_if_not_appendable(self, block: Block) -> BlockId:
175
"""NOTE: This should not be used besides Block"""
176
pass
177
@@ -174,13 +193,20 @@ class BlockAllocator(ABC):
193
class DeviceAwareBlockAllocator(ABC):
194
195
@abstractmethod
177
- def allocate_mutable(self, prev_block: Optional[Block],
178
- device: Device) -> Block:
196
+ def allocate_mutable_block(self, prev_block: Optional[Block],
197
+ device: Device) -> Block:
198
+ pass
199
+
200
+ @abstractmethod
201
+ def allocate_immutable_block(self, prev_block: Optional[Block],
202
+ token_ids: List[int],
203
+ device: Device) -> Block:
204
pass
205
206
@abstractmethod
182
- def allocate_immutable(self, prev_block: Optional[Block],
183
- token_ids: List[int], device: Device) -> Block:
207
+ def allocate_immutable_blocks(self, prev_block: Optional[Block],
208
+ block_token_ids: List[List[int]],
209
+ device: Device) -> List[Block]:
210
pass
211
212
@abstractmethod
@@ -217,9 +243,15 @@ class DeviceAwareBlockAllocator(ABC):
243
def mark_blocks_as_computed(self, block_ids: List[int]) -> None:
244
pass
245
246
+ @abstractmethod
247
+ def get_computed_block_ids(self, prev_computed_block_ids: List[int],
248
+ block_ids: List[int],
249
+ skip_last_block_id: bool) -> List[int]:
250
+ pass
251
+
252
@abstractmethod
253
def get_common_computed_block_ids(
222
- self, seq_block_ids: List[List[int]]) -> List[int]:
254
+ self, computed_seq_block_ids: List[List[int]]) -> List[int]:
255
pass
256
257
@abstractmethod
@@ -230,8 +262,8 @@ class DeviceAwareBlockAllocator(ABC):
262
pass
263
264
@abstractmethod
233
- def swap(self, blocks: List[Block], source_device: Device,
234
- dest_device: Device) -> Dict[int, int]:
265
+ def swap(self, blocks: List[Block], src_device: Device,
266
+ dst_device: Device) -> Dict[int, int]:
267
pass
268
269
@abstractmethod
vllm/core/block/naive_block.py
+149
-67
@@ -1,6 +1,7 @@
1
-from typing import FrozenSet, Iterable, List, Optional, Set, Tuple
1
+from collections import deque
2
+from typing import Deque, FrozenSet, Iterable, List, Optional, Tuple
3
3
-from vllm.core.block.common import (CopyOnWriteTracker, RefCounter,
4
+from vllm.core.block.common import (BlockPool, CopyOnWriteTracker, RefCounter,
5
get_all_blocks_recursively)
6
from vllm.core.block.interfaces import Block, BlockAllocator, BlockId, Device
7
from vllm.utils import cdiv
@@ -31,28 +32,39 @@ class NaiveBlockAllocator(BlockAllocator):
32
num_blocks: int,
33
block_size: int,
34
block_ids: Optional[Iterable[int]] = None,
35
+ block_pool: Optional[BlockPool] = None,
36
):
37
if block_ids is None:
38
block_ids = range(num_blocks)
39
38
- self._free_block_indices: Set[BlockId] = set(block_ids)
40
+ self._free_block_indices: Deque[BlockId] = deque(block_ids)
41
self._all_block_indices = frozenset(block_ids)
42
assert len(self._all_block_indices) == num_blocks
43
44
self._refcounter = RefCounter(
45
all_block_indices=self._free_block_indices)
44
- self._create_block = create_block
46
self._block_size = block_size
47
48
self._cow_tracker = CopyOnWriteTracker(
48
- refcounter=self._refcounter.as_readonly(),
49
- allocator=self,
50
- )
51
-
52
- def allocate_immutable(self,
53
- prev_block: Optional[Block],
54
- token_ids: List[int],
55
- device: Optional[Device] = None) -> Block:
49
+ refcounter=self._refcounter.as_readonly())
50
+
51
+ if block_pool is None:
52
+ extra_factor = 4
53
+ # Pre-allocate "num_blocks * extra_factor" block objects.
54
+ # The "* extra_factor" is a buffer to allow more block objects
55
+ # than physical blocks
56
+ self._block_pool = BlockPool(self._block_size, create_block, self,
57
+ num_blocks * extra_factor)
58
+ else:
59
+ # In this case, the block pool is provided by the caller,
60
+ # which means that there is most likely a need to share
61
+ # a block pool between allocators
62
+ self._block_pool = block_pool
63
+
64
+ def allocate_immutable_block(self,
65
+ prev_block: Optional[Block],
66
+ token_ids: List[int],
67
+ device: Optional[Device] = None) -> Block:
68
"""Allocates a new immutable block with the given token IDs, linked to
69
the previous block.
70
@@ -66,13 +78,36 @@ class NaiveBlockAllocator(BlockAllocator):
78
Block: The newly allocated immutable block.
79
"""
80
assert device is None
69
- block = self.allocate_mutable(prev_block=prev_block)
81
+ block = self.allocate_mutable_block(prev_block=prev_block)
82
block.append_token_ids(token_ids)
83
return block
84
73
- def allocate_mutable(self,
74
- prev_block: Optional[Block],
75
- device: Optional[Device] = None) -> Block:
85
+ def allocate_immutable_blocks(
86
+ self,
87
+ prev_block: Optional[Block],
88
+ block_token_ids: List[List[int]],
89
+ device: Optional[Device] = None) -> List[Block]:
90
+ assert device is None
91
+ num_blocks = len(block_token_ids)
92
+
93
+ block_ids = []
94
+ for i in range(num_blocks):
95
+ block_ids.append(self._allocate_block_id())
96
+
97
+ blocks = []
98
+ for i in range(num_blocks):
99
+ prev_block = self._block_pool.init_block(
100
+ prev_block=prev_block,
101
+ token_ids=block_token_ids[i],
102
+ block_size=self._block_size,
103
+ physical_block_id=block_ids[i])
104
+ blocks.append(prev_block)
105
+
106
+ return blocks
107
+
108
+ def allocate_mutable_block(self,
109
+ prev_block: Optional[Block],
110
+ device: Optional[Device] = None) -> Block:
111
"""Allocates a new mutable block, linked to the previous block.
112
113
Args:
@@ -84,20 +119,39 @@ class NaiveBlockAllocator(BlockAllocator):
119
Block: The newly allocated mutable block.
120
"""
121
assert device is None
87
- block_id = self._allocate_new_block_id()
88
- return self._create_block(
89
- prev_block=prev_block,
90
- token_ids=[],
91
- block_id=block_id,
92
- block_size=self._block_size,
93
- allocator=self,
94
- )
95
-
96
- def free(self, block: Block) -> None:
97
- assert block.block_id is not None
98
- self._free_block_id(block.block_id)
122
+ block_id = self._allocate_block_id()
123
+ block = self._block_pool.init_block(prev_block=prev_block,
124
+ token_ids=[],
125
+ block_size=self._block_size,
126
+ physical_block_id=block_id)
127
+ return block
128
+
129
+ def _allocate_block_id(self) -> BlockId:
130
+ if not self._free_block_indices:
131
+ raise BlockAllocator.NoFreeBlocksError()
132
+
133
+ block_id = self._free_block_indices.popleft()
134
+ self._refcounter.incr(block_id)
135
+ return block_id
136
+
137
+ def _free_block_id(self, block: Block) -> None:
138
+ block_id = block.block_id
139
+ assert block_id is not None
140
+
141
+ refcount = self._refcounter.decr(block_id)
142
+ if refcount == 0:
143
+ self._free_block_indices.appendleft(block_id)
144
+
145
block.block_id = None
146
147
+ def free(self, block: Block, keep_block_object: bool = False) -> None:
148
+ # Release the physical block id
149
+ self._free_block_id(block)
150
+
151
+ # Release the block object
152
+ if not keep_block_object:
153
+ self._block_pool.free_block(block)
154
+
155
def fork(self, last_block: Block) -> List[Block]:
156
"""Creates a new sequence of blocks that shares the same underlying
157
memory as the original sequence.
@@ -120,14 +174,13 @@ class NaiveBlockAllocator(BlockAllocator):
174
refcount = self._refcounter.incr(block.block_id)
175
assert refcount != 1, "can't fork free'd block"
176
123
- forked_blocks.append(
124
- self._create_block(
125
- prev_block=prev_block,
126
- token_ids=block.token_ids,
127
- block_id=block.block_id,
128
- block_size=self._block_size,
129
- allocator=self,
130
- ))
177
+ forked_block = self._block_pool.init_block(
178
+ prev_block=prev_block,
179
+ token_ids=block.token_ids,
180
+ block_size=self._block_size,
181
+ physical_block_id=block.block_id)
182
+
183
+ forked_blocks.append(forked_block)
184
prev_block = forked_blocks[-1]
185
186
return forked_blocks
@@ -138,20 +191,6 @@ class NaiveBlockAllocator(BlockAllocator):
191
def get_num_total_blocks(self) -> int:
192
return len(self._all_block_indices)
193
141
- def _allocate_new_block_id(self) -> BlockId:
142
- if not self._free_block_indices:
143
- raise BlockAllocator.NoFreeBlocksError()
144
-
145
- block_id = next(iter(self._free_block_indices))
146
- self._refcounter.incr(block_id)
147
- self._free_block_indices.remove(block_id)
148
- return block_id
149
-
150
- def _free_block_id(self, block_id: BlockId) -> None:
151
- refcount = self._refcounter.decr(block_id)
152
- if refcount == 0:
153
- self._free_block_indices.add(block_id)
154
-
194
def get_physical_block_id(self, absolute_id: int) -> int:
195
"""Returns the zero-offset block id on certain block allocator
196
given the absolute block id.
@@ -173,7 +212,7 @@ class NaiveBlockAllocator(BlockAllocator):
212
def all_block_ids(self) -> FrozenSet[int]:
213
return self._all_block_indices
214
176
- def cow_block_if_not_appendable(self, block: Block) -> Optional[BlockId]:
215
+ def cow_block_if_not_appendable(self, block: Block) -> BlockId:
216
"""Performs a copy-on-write operation on the given block if it is not
217
appendable.
218
@@ -181,11 +220,22 @@ class NaiveBlockAllocator(BlockAllocator):
220
block (Block): The block to check for copy-on-write.
221
222
Returns:
184
- Optional[BlockId]: The block index of the new block if a copy-on
185
- -write operation was performed, or the original block index if
223
+ BlockId: The block index of the new block if a copy-on-write
224
+ operation was performed, or the original block index if
225
no copy-on-write was necessary.
226
"""
188
- return self._cow_tracker.cow_block_if_not_appendable(block)
227
+ src_block_id = block.block_id
228
+ assert src_block_id is not None
229
+
230
+ if self._cow_tracker.is_appendable(block):
231
+ return src_block_id
232
+
233
+ self._free_block_id(block)
234
+ trg_block_id = self._allocate_block_id()
235
+
236
+ self._cow_tracker.record_cow(src_block_id, trg_block_id)
237
+
238
+ return trg_block_id
239
240
def clear_copy_on_writes(self) -> List[Tuple[BlockId, BlockId]]:
241
"""Returns the copy-on-write source->destination mapping and clears it.
@@ -213,8 +263,15 @@ class NaiveBlockAllocator(BlockAllocator):
263
"""
264
pass
265
266
+ def get_computed_block_ids(self, prev_computed_block_ids: List[int],
267
+ block_ids: List[int],
268
+ skip_last_block_id: bool) -> List[int]:
269
+ """No prefix caching here => return empty list
270
+ """
271
+ return []
272
+
273
def get_common_computed_block_ids(
217
- self, seq_block_ids: List[List[int]]) -> List[int]:
274
+ self, computed_seq_block_ids: List[List[int]]) -> List[int]:
275
"""Determine blocks that can be skipped in prefill.
276
277
Since the naive allocator does not support prefix caching, always return
@@ -223,7 +280,7 @@ class NaiveBlockAllocator(BlockAllocator):
280
return []
281
282
def promote_to_immutable_block(self, block: Block) -> BlockId:
226
- raise NotImplementedError
283
+ raise NotImplementedError("There is no promotion for naive blocks")
284
285
def get_num_blocks_touched(self,
286
blocks: List[Block],
@@ -263,17 +320,27 @@ class NaiveBlockAllocator(BlockAllocator):
320
321
def swap_out(self, blocks: List[Block]) -> None:
322
for block in blocks:
266
- self.free(block)
323
+ self._free_block_id(block)
324
325
def swap_in(self, blocks: List[Block]) -> None:
326
for block in blocks:
327
+ # Here we allocate either immutable or mutable block and then
328
+ # extract its block_id. Note that the block object is released
329
+ # and the block_id is assigned to "block" to allow reusing the
330
+ # existing "block" object
331
if block.is_full:
271
- alloc = self.allocate_immutable(block.prev_block,
272
- block.token_ids)
332
+ tmp_block = self.allocate_immutable_block(
333
+ prev_block=block.prev_block, token_ids=block.token_ids)
334
else:
274
- alloc = self.allocate_mutable(block.prev_block)
275
- alloc.append_token_ids(block.token_ids)
276
- block.block_id = alloc.block_id
335
+ tmp_block = self.allocate_mutable_block(
336
+ prev_block=block.prev_block)
337
+ tmp_block.append_token_ids(block.token_ids)
338
+
339
+ block_id = tmp_block.block_id
340
+ tmp_block.block_id = None
341
+ self._block_pool.free_block(tmp_block)
342
+
343
+ block.block_id = block_id # Assign block_id
344
345
346
class NaiveBlock(Block):
@@ -315,11 +382,12 @@ class NaiveBlock(Block):
382
self._append_token_ids_no_cow(token_ids)
383
384
def append_token_ids(self, token_ids: List[int]) -> None:
318
- """Appends the given token IDs to the block, instructing the allocator
319
- to perform a copy-on-write if necessary.
385
+ """Appends the given token IDs to the block and performs a
386
+ copy-on-write if necessary.
387
388
Args:
322
- token_ids (List[int]): The token IDs to be appended to the block.
389
+ token_ids (Optional[List[int]]): The token IDs to be appended
390
+ to the block.
391
"""
392
self._append_token_ids_no_cow(token_ids)
393
@@ -328,7 +396,16 @@ class NaiveBlock(Block):
396
self._cow_target))
397
398
def _append_token_ids_no_cow(self, token_ids: List[int]) -> None:
331
- assert self.num_empty_slots >= len(token_ids)
399
+ """Appends the given token IDs to the block
400
+
401
+ Args:
402
+ token_ids (List[int]): The token IDs to be appended to the block.
403
+ """
404
+ if len(token_ids) == 0:
405
+ return
406
+
407
+ assert len(token_ids) <= self.num_empty_slots
408
+
409
self._token_ids.extend(token_ids)
410
411
@property
@@ -361,12 +438,17 @@ class NaiveBlock(Block):
438
439
@property
440
def num_empty_slots(self) -> int:
364
- return self._block_size - len(self._token_ids)
441
+ return self._block_size - len(self.token_ids)
442
443
@property
444
def token_ids(self) -> List[int]:
445
return self._token_ids
446
447
+ @property
448
+ def num_tokens_total(self) -> int:
449
+ raise NotImplementedError(
450
+ "num_tokens_total is not used for naive block")
451
+
452
@property
453
def block_size(self) -> int:
454
return self._block_size
vllm/core/block/prefix_caching_block.py
+479
-214
@@ -1,13 +1,13 @@
1
"""Token blocks."""
2
3
-from itertools import takewhile
3
from os.path import commonprefix
4
from typing import Dict, FrozenSet, Iterable, List, Optional, Tuple
5
6
from vllm.core.block.common import (CopyOnWriteTracker,
7
get_all_blocks_recursively)
8
from vllm.core.block.interfaces import Block, BlockAllocator, BlockId, Device
10
-from vllm.core.block.naive_block import NaiveBlock, NaiveBlockAllocator
9
+from vllm.core.block.naive_block import (BlockPool, NaiveBlock,
10
+ NaiveBlockAllocator)
11
from vllm.core.evictor_v2 import EvictionPolicy, Evictor, make_evictor
12
from vllm.utils import cdiv
13
@@ -19,6 +19,30 @@ PrefixHash = int
19
_DEFAULT_LAST_ACCESSED_TIME = -1
20
21
22
+class BlockTracker:
23
+ """Used to track the status of a block inside the prefix caching allocator
24
+ """
25
+ __slots__ = ("active", "last_accessed", "computed")
26
+
27
+ def reset(self):
28
+ self.last_accessed: float = _DEFAULT_LAST_ACCESSED_TIME
29
+ self.computed: bool = False
30
+
31
+ def __init__(self):
32
+ self.active: bool = False
33
+ self.reset()
34
+
35
+ def enable(self):
36
+ assert not self.active
37
+ self.active = True
38
+ self.reset()
39
+
40
+ def disable(self):
41
+ assert self.active
42
+ self.active = False
43
+ self.reset()
44
+
45
+
46
class PrefixCachingBlockAllocator(BlockAllocator):
47
"""A block allocator that implements prefix caching.
48
@@ -41,12 +65,26 @@ class PrefixCachingBlockAllocator(BlockAllocator):
65
block_ids: Optional[Iterable[int]] = None,
66
eviction_policy: EvictionPolicy = EvictionPolicy.LRU,
67
):
68
+ if block_ids is None:
69
+ block_ids = range(num_blocks)
70
+
71
+ self._block_size = block_size
72
+
73
# A mapping of prefix hash to block index. All blocks which have a
74
# prefix hash will be in this dict, even if they have refcount 0.
75
self._cached_blocks: Dict[PrefixHash, BlockId] = {}
76
48
- # A mapping of blockId to Block to track those cached blocks
49
- self._blocks: Dict[BlockId, Block] = {}
77
+ # Used to track status of each physical block id
78
+ self._block_tracker: Dict[BlockId, BlockTracker] = {}
79
+ for block_id in block_ids:
80
+ self._block_tracker[block_id] = BlockTracker()
81
+
82
+ # Pre-allocate "num_blocks * extra_factor" block objects.
83
+ # The "* extra_factor" is a buffer to allow more block objects
84
+ # than physical blocks
85
+ extra_factor = 4
86
+ self._block_pool = BlockPool(self._block_size, self._create_block,
87
+ self, num_blocks * extra_factor)
88
89
# An allocator for blocks that do not have prefix hashes.
90
self._hashless_allocator = NaiveBlockAllocator(
@@ -54,10 +92,9 @@ class PrefixCachingBlockAllocator(BlockAllocator):
92
num_blocks=num_blocks,
93
block_size=block_size,
94
block_ids=block_ids,
95
+ block_pool=self._block_pool, # Share block pool here
96
)
97
59
- self._block_size = block_size
60
-
98
# Evitor used to maintain how we want to handle those computed blocks
99
# if we find memory pressure is high.
100
self.evictor: Evictor = make_evictor(eviction_policy)
@@ -68,9 +105,7 @@ class PrefixCachingBlockAllocator(BlockAllocator):
105
self._refcounter = self._hashless_allocator.refcounter
106
107
self._cow_tracker = CopyOnWriteTracker(
71
- refcounter=self._refcounter.as_readonly(),
72
- allocator=self,
73
- )
108
+ refcounter=self._refcounter.as_readonly())
109
110
# Implements Block.Factory.
111
def _create_block(
@@ -90,14 +125,14 @@ class PrefixCachingBlockAllocator(BlockAllocator):
125
token_ids=token_ids,
126
block_size=block_size,
127
block_id=block_id,
93
- prefix_caching_allocator=allocator,
128
+ allocator=allocator,
129
computed=computed,
130
)
131
97
- def allocate_immutable(self,
98
- prev_block: Optional[Block],
99
- token_ids: List[int],
100
- device: Optional[Device] = None) -> Block:
132
+ def allocate_immutable_block(self,
133
+ prev_block: Optional[Block],
134
+ token_ids: List[int],
135
+ device: Optional[Device] = None) -> Block:
136
"""Allocates an immutable block with the given token IDs, reusing cached
137
blocks if possible.
138
@@ -111,29 +146,41 @@ class PrefixCachingBlockAllocator(BlockAllocator):
146
assert device is None
147
assert_prefix_caching_block_or_none(prev_block)
148
114
- block = self._create_block(
115
- prev_block=prev_block,
116
- token_ids=token_ids,
117
- block_size=self._block_size,
118
- allocator=self,
119
- )
149
+ # First, try to create a block that points to cached data
150
+ block = self._block_pool.init_block(prev_block=prev_block,
151
+ token_ids=token_ids,
152
+ block_size=self._block_size,
153
+ physical_block_id=None)
154
assert block.content_hash is not None
155
156
cached_block_id = self._cached_blocks.get(block.content_hash, None)
157
if cached_block_id is not None:
158
block.block_id = cached_block_id
125
- self._incr_refcount_cached_block(block, block.block_id)
159
+ self._incr_refcount_cached_block(block)
160
return block
161
+ self._block_pool.free_block(block)
162
128
- block = self.allocate_mutable(prev_block)
163
+ # No cached block => Allocate a new block
164
+ block = self.allocate_mutable_block(prev_block)
165
block.append_token_ids(token_ids)
130
- assert block.content_hash is not None
131
-
166
return block
167
134
- def allocate_mutable(self,
135
- prev_block: Optional[Block],
136
- device: Optional[Device] = None) -> Block:
168
+ def allocate_immutable_blocks(
169
+ self,
170
+ prev_block: Optional[Block],
171
+ block_token_ids: List[List[int]],
172
+ device: Optional[Device] = None) -> List[Block]:
173
+ blocks = []
174
+ for token_ids in block_token_ids:
175
+ prev_block = self.allocate_immutable_block(prev_block=prev_block,
176
+ token_ids=token_ids,
177
+ device=device)
178
+ blocks.append(prev_block)
179
+ return blocks
180
+
181
+ def allocate_mutable_block(self,
182
+ prev_block: Optional[Block],
183
+ device: Optional[Device] = None) -> Block:
184
"""Allocates a mutable block. If there are no free blocks, this will
185
evict unused cached blocks.
186
@@ -147,116 +194,154 @@ class PrefixCachingBlockAllocator(BlockAllocator):
194
assert device is None
195
assert_prefix_caching_block_or_none(prev_block)
196
150
- try:
151
- block = self._hashless_allocator.allocate_mutable(
152
- prev_block=prev_block)
153
-
154
- assert block.block_id not in self._blocks
155
- assert block.block_id is not None
156
- self._blocks[block.block_id] = block
157
- return block
158
- except BlockAllocator.NoFreeBlocksError:
159
- # We must check the unused cached blocks before raising OOM.
160
- pass
161
-
162
- # If the evictor has blocks available for eviction, evict a block
163
- # and return it.
164
- if self.evictor.num_blocks > 0:
165
- # here we get an evicted block, which is only added
166
- # into evictor if its ref counter is 0
167
- # and since its content would be changed, we need
168
- # to remove it from _cached_blocks's tracking list
169
- block_id, content_hash_to_evict = self.evictor.evict()
170
-
171
- _block_id = self._cached_blocks[content_hash_to_evict]
172
- assert self._refcounter.get(_block_id) == 0
173
- assert _block_id == block_id
174
-
175
- self._cached_blocks.pop(content_hash_to_evict)
176
-
177
- self._refcounter.incr(block_id)
178
-
179
- # Now this block is pop from evictor and ready to write
180
- # with new content which most probably different with
181
- # original content. So need to tell worker to recompute
182
- # its kvcache
183
- block = self._create_block(
184
- prev_block=prev_block,
185
- token_ids=[],
186
- block_size=self._block_size,
187
- allocator=self,
188
- block_id=block_id,
189
- computed=False,
190
- )
191
- assert block.content_hash is None
192
-
193
- assert block.block_id not in self._blocks
194
- assert block.block_id is not None
195
- self._blocks[block.block_id] = block
196
- return block
197
-
198
- # No block available in hashless allocator, nor in unused cache blocks.
199
- raise BlockAllocator.NoFreeBlocksError()
197
+ block_id = self._allocate_block_id()
198
+ block = self._block_pool.init_block(prev_block=prev_block,
199
+ token_ids=[],
200
+ block_size=self._block_size,
201
+ physical_block_id=block_id)
202
+ assert not block.computed
203
+ assert block.content_hash is None
204
+ return block
205
201
- def _incr_refcount_cached_block(self, block: Block,
202
- block_id: BlockId) -> None:
203
- # now _incr_refcount_cached_block comes from two place
204
- # allocate_immutable/promote_to_immutable_block where hit
205
- # _cached_blocks hash key.
206
- # In both cases, it means that already exists a already
207
- # computed block which shared with block now
206
+ def _incr_refcount_cached_block(self, block: Block) -> None:
207
+ # Set this block to be "computed" since it is pointing to a
208
+ # cached block id (which was already computed)
209
block.computed = True
210
211
+ block_id = block.block_id
212
+ assert block_id is not None
213
+
214
refcount = self._refcounter.incr(block_id)
215
if refcount == 1:
212
- # if block get referred, then it shall not be in evictor
213
- # and put it into _blocks for tracking
216
+ # In case a cached block was evicted, restore its tracking
217
if block_id in self.evictor:
218
self.evictor.remove(block_id)
216
- self._blocks[block_id] = block
219
218
- def free(self, block: Block) -> None:
219
- """Decrement the refcount of the block. If the decremented refcount is
220
- zero, store the block in the freelist.
220
+ self._track_block_id(block_id, computed=True)
221
222
- If the block has a content hash (meaning it is immutable), then we will
223
- keep the block around in case future allocations require it.
224
- """
225
- assert (block.block_id
226
- is not None), "freeing unallocated block is undefined"
222
+ def _decr_refcount_cached_block(self, block: Block) -> None:
223
+ # Ensure this is immutable/cached block
224
+ assert block.content_hash is not None
225
+
226
+ block_id = block.block_id
227
+ assert block_id is not None
228
+
229
+ refcount = self._refcounter.decr(block_id)
230
+ if refcount > 0:
231
+ block.block_id = None
232
+ return
233
+ else:
234
+ assert refcount == 0
235
228
- self._free_block_id_for_block(block.block_id, block)
236
+ # No longer used
237
+ assert block.content_hash in self._cached_blocks
238
+
239
+ # Add the cached block to the evictor
240
+ # (This keeps the cached block around so it can be reused)
241
+ self.evictor.add(block_id, block.content_hash, block.num_tokens_total,
242
+ self._block_tracker[block_id].last_accessed)
243
+
244
+ # Stop tracking the block
245
+ self._untrack_block_id(block_id)
246
247
block.block_id = None
248
232
- def _free_block_id_for_block(self, block_id: BlockId,
233
- block: Block) -> None:
234
- assert isinstance(block, PrefixCachingBlock)
235
-
236
- # if we comes from promote_to_immutable_block, it means that
237
- # block.content_hash is never None.
238
- # However we need to release the same content block, so that
239
- # physical block could get reused.
240
- if block.block_id != block_id or block.content_hash is None:
241
- refcount = self._refcounter.get(block_id)
242
- # We have fork case where block would get more than one ref,
243
- # so we cannot free it from tracking if ref cnt large than 1
244
- assert block.block_id is not None
245
- refcount = self._refcounter.get(block.block_id)
246
- if refcount == 1:
247
- del self._blocks[block.block_id]
248
-
249
- return self._hashless_allocator.free(block)
249
+ def _decr_refcount_hashless_block(self, block: Block) -> None:
250
+ block_id = block.block_id
251
+ assert block_id is not None
252
251
- refcount = self._refcounter.decr(block_id)
253
+ # We may have a fork case where block is shared,
254
+ # in which case, we cannot remove it from tracking
255
+ refcount = self._refcounter.get(block_id)
256
+ if refcount == 1:
257
+ self._untrack_block_id(block_id)
258
253
- # If no longer used, add the block to the evictor.
254
- if refcount == 0:
255
- assert block.content_hash in self._cached_blocks
256
- assert block.block_id is not None
257
- del self._blocks[block.block_id]
258
- self.evictor.add(block.block_id, block.content_hash,
259
- block.num_tokens_total, block.last_accessed)
259
+ # Decrement refcount of the block_id, but do not free the block object
260
+ # itself (will be handled by the caller)
261
+ self._hashless_allocator.free(block, keep_block_object=True)
262
+
263
+ def _allocate_block_id(self) -> BlockId:
264
+ """First tries to allocate a block id from the hashless allocator,
265
+ and if there are no blocks, then tries to evict an unused cached block.
266
+ """
267
+ hashless_block_id = self._maybe_allocate_hashless_block_id()
268
+ if hashless_block_id is not None:
269
+ return hashless_block_id
270
+
271
+ evicted_block_id = self._maybe_allocate_evicted_block_id()
272
+ if evicted_block_id is not None:
273
+ return evicted_block_id
274
+
275
+ # No block available in hashless allocator, nor in unused cache blocks.
276
+ raise BlockAllocator.NoFreeBlocksError()
277
+
278
+ def _maybe_allocate_hashless_block_id(self) -> Optional[BlockId]:
279
+ try:
280
+ # Allocate mutable block and extract its block_id
281
+ block = self._hashless_allocator.allocate_mutable_block(
282
+ prev_block=None)
283
+ block_id = block.block_id
284
+ self._block_pool.free_block(block)
285
+
286
+ self._track_block_id(block_id, computed=False)
287
+ return block_id
288
+ except BlockAllocator.NoFreeBlocksError:
289
+ return None
290
+
291
+ def _maybe_allocate_evicted_block_id(self) -> Optional[BlockId]:
292
+ if self.evictor.num_blocks == 0:
293
+ return None
294
+
295
+ # Here we get an evicted block, which is only added
296
+ # into evictor if its ref counter is 0
297
+ # and since its content would be changed, we need
298
+ # to remove it from _cached_blocks's tracking list
299
+ block_id, content_hash_to_evict = self.evictor.evict()
300
+
301
+ # Sanity checks
302
+ assert content_hash_to_evict in self._cached_blocks
303
+ _block_id = self._cached_blocks[content_hash_to_evict]
304
+ assert self._refcounter.get(_block_id) == 0
305
+ assert _block_id == block_id
306
+
307
+ self._cached_blocks.pop(content_hash_to_evict)
308
+
309
+ self._refcounter.incr(block_id)
310
+ self._track_block_id(block_id, computed=False)
311
+
312
+ return block_id
313
+
314
+ def _free_block_id(self, block: Block) -> None:
315
+ """Decrements the refcount of the block. The block may be in two
316
+ possible states: (1) immutable/cached or (2) mutable/hashless.
317
+ In the first case, the refcount is decremented directly and the block
318
+ may be possibly added to the evictor. In other case, hashless
319
+ allocator free(..) with keep_block_object=True is called to only free
320
+ the block id (since the block object may be reused by the caller)
321
+ """
322
+ block_id = block.block_id
323
+ assert block_id is not None, "Freeing unallocated block is undefined"
324
+
325
+ if block.content_hash is not None:
326
+ # Immutable: This type of block is always cached, and we want to
327
+ # keep it in the evictor for future reuse
328
+ self._decr_refcount_cached_block(block)
329
+ else:
330
+ # Mutable: This type of block is not cached, so we release it
331
+ # directly to the hashless allocator
332
+ self._decr_refcount_hashless_block(block)
333
+
334
+ assert block.block_id is None
335
+
336
+ def free(self, block: Block, keep_block_object: bool = False) -> None:
337
+ """Release the block (look at free_block_id(..) docs)
338
+ """
339
+ # Release the physical block index
340
+ self._free_block_id(block)
341
+
342
+ # Release the block object to the pool
343
+ if not keep_block_object:
344
+ self._block_pool.free_block(block)
345
346
def fork(self, last_block: Block) -> List[Block]:
347
"""Creates a new sequence of blocks that shares the same underlying
@@ -274,17 +359,20 @@ class PrefixCachingBlockAllocator(BlockAllocator):
359
forked_blocks: List[Block] = []
360
prev_block = None
361
for block in source_blocks:
277
- refcount = self._refcounter.incr(block.block_id)
278
- assert refcount != 1, "can't fork free'd block"
279
-
280
- forked_blocks.append(
281
- self._create_block(
282
- prev_block=prev_block,
283
- token_ids=block.token_ids,
284
- block_id=block.block_id,
285
- block_size=self._block_size,
286
- allocator=self,
287
- ))
362
+ block_id = block.block_id
363
+ assert block_id is not None
364
+
365
+ refcount = self._refcounter.incr(block_id)
366
+ assert refcount != 1, "can't fork free'd block_id = {}".format(
367
+ block_id)
368
+
369
+ forked_block = self._block_pool.init_block(
370
+ prev_block=prev_block,
371
+ token_ids=block.token_ids,
372
+ block_size=self._block_size,
373
+ physical_block_id=block_id)
374
+
375
+ forked_blocks.append(forked_block)
376
prev_block = forked_blocks[-1]
377
378
return forked_blocks
@@ -329,7 +417,7 @@ class PrefixCachingBlockAllocator(BlockAllocator):
417
418
Note that if we already have a cached block with the same content, we
419
will replace the newly-promoted block's mapping with the existing cached
332
- block.
420
+ block id.
421
422
Args:
423
block: The mutable block to be promoted.
@@ -338,23 +426,30 @@ class PrefixCachingBlockAllocator(BlockAllocator):
426
BlockId: Either the original block index, or the block index of
427
the previously cached block matching the same content.
428
"""
429
+ # Ensure block can be promoted
430
assert block.content_hash is not None
431
assert block.block_id is not None
432
assert self._refcounter.get(block.block_id) > 0
433
345
- # If the content hash does not have a corresponding cached block,
346
- # set this block as the cached block.
434
if block.content_hash not in self._cached_blocks:
435
+ # No cached content hash => Set this block as cached
436
+ # (Note that this block is not computed yet =>
437
+ # Will be computed after free())
438
self._cached_blocks[block.content_hash] = block.block_id
349
- else:
350
- self._free_block_id_for_block(
351
- self._cached_blocks[block.content_hash], block)
352
- self._incr_refcount_cached_block(
353
- block, self._cached_blocks[block.content_hash])
439
+ return block.block_id
440
355
- return self._cached_blocks[block.content_hash]
441
+ # Reuse the cached content hash
442
+ self._decr_refcount_hashless_block(block)
443
+ block.block_id = self._cached_blocks[block.content_hash]
444
357
- def cow_block_if_not_appendable(self, block: Block) -> Optional[BlockId]:
445
+ # Increment refcount of the cached block and (possibly) restore
446
+ # it from the evictor.
447
+ # Note that in this case, the block is marked as computed
448
+ self._incr_refcount_cached_block(block)
449
+
450
+ return block.block_id
451
+
452
+ def cow_block_if_not_appendable(self, block: Block) -> BlockId:
453
"""Performs a copy-on-write operation on the given block if it is not
454
appendable.
455
@@ -362,11 +457,22 @@ class PrefixCachingBlockAllocator(BlockAllocator):
457
block (Block): The block to check for copy-on-write.
458
459
Returns:
365
- Optional[BlockId]: The block index of the new block if a copy-on
366
- -write operation was performed, or the original block index if
460
+ BlockId: The block index of the new block if a copy-on-write
461
+ operation was performed, or the original block index if
462
no copy-on-write was necessary.
463
"""
369
- return self._cow_tracker.cow_block_if_not_appendable(block)
464
+ src_block_id = block.block_id
465
+ assert src_block_id is not None
466
+
467
+ if self._cow_tracker.is_appendable(block):
468
+ return src_block_id
469
+
470
+ self._free_block_id(block)
471
+ trg_block_id = self._allocate_block_id()
472
+
473
+ self._cow_tracker.record_cow(src_block_id, trg_block_id)
474
+
475
+ return trg_block_id
476
477
def clear_copy_on_writes(self) -> List[Tuple[BlockId, BlockId]]:
478
"""Returns the copy-on-write source->destination mapping and clears it.
@@ -386,8 +492,8 @@ class PrefixCachingBlockAllocator(BlockAllocator):
492
"""
493
494
for block_id in block_ids:
389
- if block_id in self._blocks:
390
- self._blocks[block_id].last_accessed = now
495
+ if self._block_tracker[block_id].active:
496
+ self._block_tracker[block_id].last_accessed = now
497
elif block_id in self.evictor:
498
self.evictor.update(block_id, now)
499
else:
@@ -395,25 +501,46 @@ class PrefixCachingBlockAllocator(BlockAllocator):
501
"Mark block as accessed which is not belonged to GPU")
502
503
def mark_blocks_as_computed(self, block_ids: List[int]) -> None:
398
- """Mark blocks as computed, used in prefix caching."""
504
+ raise NotImplementedError("Marking as computed is incremental")
505
400
- for block_id in block_ids:
401
- if block_id in self._blocks:
402
- # only those full block is valid for prefix caching
403
- if self._blocks[block_id].is_full:
404
- self._blocks[block_id].computed = True
405
- elif block_id not in self.evictor:
406
- raise ValueError(f"Mark {block_id=} as computed which "
407
- "is not belonged to GPU")
506
+ def _track_block_id(self, block_id: Optional[BlockId],
507
+ computed: bool) -> None:
508
+ assert block_id is not None
509
+ self._block_tracker[block_id].enable()
510
+ self._block_tracker[block_id].computed = computed
511
+
512
+ def _untrack_block_id(self, block_id: Optional[BlockId]) -> None:
513
+ assert block_id is not None
514
+ self._block_tracker[block_id].disable()
515
516
def block_is_computed(self, block_id: int) -> bool:
410
- if block_id in self._blocks:
411
- return self._blocks[block_id].computed
517
+ if self._block_tracker[block_id].active:
518
+ return self._block_tracker[block_id].computed
519
else:
520
return block_id in self.evictor
521
522
+ def get_computed_block_ids(self,
523
+ prev_computed_block_ids: List[int],
524
+ block_ids: List[int],
525
+ skip_last_block_id: bool = True) -> List[int]:
526
+ prev_prefix_size = len(prev_computed_block_ids)
527
+ cur_size = len(block_ids)
528
+ if skip_last_block_id:
529
+ cur_size -= 1
530
+
531
+ # Sanity checks
532
+ assert cur_size >= 0
533
+ assert prev_prefix_size <= cur_size
534
+
535
+ ret = prev_computed_block_ids
536
+ for i in range(prev_prefix_size, cur_size):
537
+ block_id = block_ids[i]
538
+ if self.block_is_computed(block_id):
539
+ ret.append(block_id)
540
+ return ret
541
+
542
def get_common_computed_block_ids(
416
- self, seq_block_ids: List[List[int]]) -> List[int]:
543
+ self, computed_seq_block_ids: List[List[int]]) -> List[int]:
544
"""Return the block ids that are common for a given sequence group.
545
546
Only those blocks that are immutable and already be marked
@@ -424,14 +551,9 @@ class PrefixCachingBlockAllocator(BlockAllocator):
551
# prompt is cached. This would cause erroneous behavior in model
552
# runner.
553
427
- ids_list = [
428
- list(
429
- takewhile(lambda block_id: self.block_is_computed(block_id),
430
- seq[:-1])) for seq in seq_block_ids
431
- ]
554
# It returns a list of int although type annotation says list of string.
555
return commonprefix([
434
- ids for ids in ids_list # type: ignore
556
+ ids for ids in computed_seq_block_ids # type: ignore
557
if ids != []
558
])
559
@@ -473,10 +595,10 @@ class PrefixCachingBlockAllocator(BlockAllocator):
595
blocks: List of blocks to be swapped out.
596
"""
597
for block in blocks:
476
- self.free(block)
598
+ self._free_block_id(block)
599
600
def swap_in(self, blocks: List[Block]) -> None:
479
- """Execute the swap int actions. Change the block id from
601
+ """Execute the swap in actions. Change the block id from
602
old allocator to current allocator for each block to finish
603
the block table update.
604
@@ -484,13 +606,22 @@ class PrefixCachingBlockAllocator(BlockAllocator):
606
blocks: List of blocks to be swapped in.
607
"""
608
for block in blocks:
609
+ # Here we allocate either immutable or mutable block and then
610
+ # extract its block_id. Note that the block object is released
611
+ # and the block_id is assigned to "block" to allow reusing the
612
+ # existing "block" object
613
if block.is_full:
488
- alloc = self.allocate_immutable(block.prev_block,
489
- block.token_ids)
614
+ tmp_block = self.allocate_immutable_block(
615
+ prev_block=block.prev_block, token_ids=block.token_ids)
616
else:
491
- alloc = self.allocate_mutable(block.prev_block)
492
- alloc.append_token_ids(block.token_ids)
493
- block.block_id = alloc.block_id
617
+ tmp_block = self.allocate_mutable_block(
618
+ prev_block=block.prev_block)
619
+ tmp_block.append_token_ids(block.token_ids)
620
+
621
+ block_id = tmp_block.block_id
622
+ self._block_pool.free_block(tmp_block)
623
+
624
+ block.block_id = block_id # Assign block_id
625
626
627
class PrefixCachingBlock(Block):
@@ -507,7 +638,7 @@ class PrefixCachingBlock(Block):
638
token_ids (List[int]): The initial token IDs to be stored in the block.
639
block_size (int): The maximum number of token IDs that can be stored in
640
the block.
510
- prefix_caching_allocator (BlockAllocator): The prefix
641
+ allocator (BlockAllocator): The prefix
642
caching block allocator associated with this block.
643
block_id (Optional[int], optional): The physical block index
644
of this block. Defaults to None.
@@ -518,31 +649,55 @@ class PrefixCachingBlock(Block):
649
prev_block: Optional[Block],
650
token_ids: List[int],
651
block_size: int,
521
- prefix_caching_allocator: BlockAllocator,
652
+ allocator: BlockAllocator,
653
block_id: Optional[int] = None,
654
computed: bool = False,
655
):
525
- assert isinstance(prefix_caching_allocator,
526
- PrefixCachingBlockAllocator), (
527
- "Currently this class is only tested with "
528
- "PrefixCachingBlockAllocator.")
656
+ assert isinstance(allocator, PrefixCachingBlockAllocator), (
657
+ "Currently this class is only tested with "
658
+ "PrefixCachingBlockAllocator. Got instead allocator = {}".format(
659
+ allocator))
660
assert_prefix_caching_block_or_none(prev_block)
661
662
self._prev_block = prev_block
663
self._cached_content_hash: Optional[int] = None
533
- self._cached_num_tokens_total: Optional[int] = None
534
- self._prefix_caching_allocator = prefix_caching_allocator
664
+ self._cached_num_tokens_total: int = 0
665
+ self._allocator = allocator
666
self._last_accessed: float = _DEFAULT_LAST_ACCESSED_TIME
667
self._computed = computed
668
538
- self._block = NaiveBlock(
539
- prev_block=prev_block,
540
- token_ids=token_ids,
541
- block_size=block_size,
542
- block_id=block_id,
543
- allocator=prefix_caching_allocator,
544
- _cow_target=self,
545
- )
669
+ # On the first time, we create the block object, and next we only
670
+ # reinitialize it
671
+ if hasattr(self, "_block"):
672
+ self._block.__init__( # type: ignore[has-type]
673
+ prev_block=prev_block,
674
+ token_ids=token_ids,
675
+ block_size=block_size,
676
+ block_id=block_id,
677
+ allocator=self._allocator)
678
+ else:
679
+ self._block = NaiveBlock(prev_block=prev_block,
680
+ token_ids=token_ids,
681
+ block_size=block_size,
682
+ block_id=block_id,
683
+ allocator=self._allocator)
684
+
685
+ self._update_num_tokens_total()
686
+
687
+ def _update_num_tokens_total(self):
688
+ """Incrementally computes the number of tokens that there is
689
+ till the current block (included)
690
+ """
691
+ res = 0
692
+
693
+ # Add all previous blocks
694
+ if self._prev_block is not None:
695
+ res += self._prev_block.num_tokens_total
696
+
697
+ # Add current block
698
+ res += len(self.token_ids)
699
+
700
+ self._cached_num_tokens_total = res
701
702
@property
703
def computed(self) -> bool:
@@ -564,22 +719,28 @@ class PrefixCachingBlock(Block):
719
"""Appends the given token IDs to the block and registers the block as
720
immutable if the block becomes full.
721
567
- Internally, the naive block handles CoW.
568
-
722
Args:
723
token_ids (List[int]): The token IDs to be appended to the block.
724
"""
572
- assert token_ids
725
+ # Ensure this is mutable block (not promoted)
726
+ assert self.content_hash is None
727
+ assert not self.computed
728
+
729
+ if len(token_ids) == 0:
730
+ return
731
574
- # naive block handles CoW.
732
+ # Ensure there are input tokens
733
+ assert token_ids, "Got token_ids = {}".format(token_ids)
734
+
735
+ # Naive block handles CoW.
736
self._block.append_token_ids(token_ids)
737
+ self._update_num_tokens_total()
738
739
# If the content hash is present, then the block can be made immutable.
740
# Register ourselves with the allocator, potentially replacing the
741
# physical block index.
742
if self.content_hash is not None:
581
- self.block_id = (self._prefix_caching_allocator.
582
- promote_to_immutable_block(self))
743
+ self.block_id = self._allocator.promote_to_immutable_block(self)
744
745
@property
746
def block_id(self) -> Optional[int]:
@@ -599,23 +760,6 @@ class PrefixCachingBlock(Block):
760
761
@property
762
def num_tokens_total(self) -> int:
602
- """return the total tokens so far.
603
-
604
- Here we iterate the block chain till to the first block, while
605
- cache the result in local to prevent repeated computations.
606
- """
607
- if self._cached_num_tokens_total is not None:
608
- return self._cached_num_tokens_total
609
-
610
- _block: Optional[Block] = self
611
- self._cached_num_tokens_total = 0
612
-
613
- # TODO: current implement here take O(N^2), we expect future
614
- # we have O(1) here
615
- while _block is not None:
616
- self._cached_num_tokens_total += len(_block.token_ids)
617
- _block = _block.prev_block
618
-
763
return self._cached_num_tokens_total
764
765
@property
@@ -638,7 +782,6 @@ class PrefixCachingBlock(Block):
782
For the content-based hash to be defined, the current block must be
783
full.
784
"""
641
-
785
# If the hash is already computed, return it.
786
if self._cached_content_hash is not None:
787
return self._cached_content_hash
@@ -688,7 +831,129 @@ class PrefixCachingBlock(Block):
831
return hash((is_first_block, prev_block_hash, *cur_block_token_ids))
832
833
834
+class ComputedBlocksTracker:
835
+ """Handles caching of per-sequence computed block ids.
836
+ When a sequence appears for the first time, it traverses all of the
837
+ blocks and detects the prefix of blocks that is computed. On the
838
+ subsequent times, it only traverses the new blocks that were added
839
+ and updates the already recorded prefix of blocks with the newly
840
+ computed blocks.
841
+
842
+ To avoid redundant traversals, the algorithm also detects when there
843
+ is a "gap" in the computed prefix. For example, if we have blocks =
844
+ [1,2,3,4,5], and we have detected [1,2,3] as the computed prefix, then
845
+ we won't try to add more computed blocks to [1,2,3] in this sequence
846
+ iteration, and will add more computed blocks only after the sequence is
847
+ freed and reused again.
848
+
849
+ Note that currently, for a given sequence, we also skip the last
850
+ block id for caching purposes, to avoid caching of a full sequence
851
+ """
852
+
853
+ def __init__(self, allocator):
854
+ self._allocator = allocator
855
+ self._cached_computed_seq_blocks: Dict[int, Tuple[List[int],
856
+ bool]] = {}
857
+
858
+ def add_seq(self, seq_id: int) -> None:
859
+ """Start tracking seq_id
860
+ """
861
+ assert seq_id not in self._cached_computed_seq_blocks
862
+ self._cached_computed_seq_blocks[seq_id] = ([], False)
863
+
864
+ def remove_seq(self, seq_id: int) -> None:
865
+ """Stop tracking seq_id
866
+ """
867
+ assert seq_id in self._cached_computed_seq_blocks
868
+ del self._cached_computed_seq_blocks[seq_id]
869
+
870
+ def get_cached_computed_blocks_and_update(
871
+ self, seq_id: int, block_ids: List[int]) -> List[int]:
872
+ """ Look at the class documentation for details
873
+ """
874
+ # Ensure seq_id is already tracked
875
+ assert seq_id in self._cached_computed_seq_blocks
876
+
877
+ # Get cached data (may be empty on the first time)
878
+ prev_computed_block_ids, has_gap = self._cached_computed_seq_blocks[
879
+ seq_id]
880
+
881
+ if has_gap:
882
+ # When gap is detected, we do not add more computed blocks at this
883
+ # sequence iteration
884
+ return prev_computed_block_ids
885
+
886
+ # We do not consider the last block id for caching purposes.
887
+ num_cur_blocks = len(block_ids) - 1
888
+ assert num_cur_blocks >= 0
889
+
890
+ if len(prev_computed_block_ids) >= num_cur_blocks:
891
+ # Cache HIT
892
+ assert len(prev_computed_block_ids) == num_cur_blocks
893
+ return prev_computed_block_ids
894
+
895
+ # If here, then we may possibly add more computed blocks. As a result,
896
+ # traverse the additional blocks after prev_computed_block_ids to
897
+ # detect more computed blocks and add them.
898
+
899
+ # Incremental init for seq_id => Look only at the new blocks
900
+ computed_block_ids = self._allocator.get_computed_block_ids( # noqa: E501
901
+ prev_computed_block_ids,
902
+ block_ids,
903
+ skip_last_block_id=
904
+ True, # We skip last block id to avoid caching of full seq
905
+ )
906
+
907
+ # Detect if there is a "gap"
908
+ has_gap = len(computed_block_ids) < num_cur_blocks
909
+
910
+ # Record
911
+ self._cached_computed_seq_blocks[seq_id] = (computed_block_ids,
912
+ has_gap)
913
+
914
+ return computed_block_ids
915
+
916
+
917
+class LastAccessBlocksTracker:
918
+ """Manages the last access time of the tracked sequences, in order to allow
919
+ an efficient update of allocator's block last access times
920
+ """
921
+
922
+ def __init__(self, allocator):
923
+ self._allocator = allocator
924
+ self._seq_last_access: Dict[int, Optional[float]] = {}
925
+
926
+ def add_seq(self, seq_id: int) -> None:
927
+ """Start tracking seq_id
928
+ """
929
+ assert seq_id not in self._seq_last_access
930
+ self._seq_last_access[seq_id] = None
931
+
932
+ def remove_seq(self, seq_id: int) -> None:
933
+ """Stop tracking seq_id
934
+ """
935
+ assert seq_id in self._seq_last_access
936
+ del self._seq_last_access[seq_id]
937
+
938
+ def update_last_access(self, seq_id: int, time: float) -> None:
939
+ assert seq_id in self._seq_last_access
940
+ self._seq_last_access[seq_id] = time
941
+
942
+ def update_seq_blocks_last_access(self, seq_id: int,
943
+ block_ids: List[int]) -> None:
944
+ assert seq_id in self._seq_last_access
945
+
946
+ ts = self._seq_last_access[seq_id]
947
+
948
+ if ts is None:
949
+ # No last access was recorded, no need to update.
950
+ return
951
+
952
+ self._allocator.mark_blocks_as_accessed(block_ids, ts)
953
+
954
+
955
def assert_prefix_caching_block_or_none(block: Optional[Block]):
956
if block is None:
957
return
694
- assert isinstance(block, PrefixCachingBlock)
958
+ assert isinstance(block,
959
+ PrefixCachingBlock), "Got block = {}".format(block)
vllm/core/block_manager_v2.py
+101
-49
@@ -7,6 +7,8 @@ from typing import Tuple
7
from vllm.core.block.block_table import BlockTable
8
from vllm.core.block.cpu_gpu_block_allocator import CpuGpuBlockAllocator
9
from vllm.core.block.interfaces import Block
10
+from vllm.core.block.prefix_caching_block import (ComputedBlocksTracker,
11
+ LastAccessBlocksTracker)
12
from vllm.core.block.utils import check_no_caching_or_swa_for_blockmgr_encdec
13
from vllm.core.interfaces import AllocStatus, BlockSpaceManager
14
from vllm.sequence import Sequence, SequenceGroup, SequenceStatus
@@ -100,6 +102,11 @@ class BlockSpaceManagerV2(BlockSpaceManager):
102
self.block_tables: Dict[SeqId, BlockTable] = {}
103
self.cross_block_tables: Dict[EncoderSeqId, BlockTable] = {}
104
105
+ self._computed_blocks_tracker = ComputedBlocksTracker(
106
+ self.block_allocator)
107
+ self._last_access_blocks_tracker = LastAccessBlocksTracker(
108
+ self.block_allocator)
109
+
110
def can_allocate(self, seq_group: SequenceGroup) -> AllocStatus:
111
# FIXME(woosuk): Here we assume that all sequences in the group share
112
# the same prompt. This may not be true for preempted sequences.
@@ -157,10 +164,18 @@ class BlockSpaceManagerV2(BlockSpaceManager):
164
block_table: BlockTable = self._allocate_sequence(seq)
165
self.block_tables[seq.seq_id] = block_table
166
167
+ # Track seq
168
+ self._computed_blocks_tracker.add_seq(seq.seq_id)
169
+ self._last_access_blocks_tracker.add_seq(seq.seq_id)
170
+
171
# Assign the block table for each sequence.
172
for seq in waiting_seqs[1:]:
173
self.block_tables[seq.seq_id] = block_table.fork()
174
175
+ # Track seq
176
+ self._computed_blocks_tracker.add_seq(seq.seq_id)
177
+ self._last_access_blocks_tracker.add_seq(seq.seq_id)
178
+
179
# Allocate cross-attention block table for encoder sequence
180
#
181
# NOTE: Here we assume that all sequences in the group have the same
@@ -224,11 +239,23 @@ class BlockSpaceManagerV2(BlockSpaceManager):
239
return new_cows
240
241
def free(self, seq: Sequence) -> None:
227
- if seq.seq_id not in self.block_tables:
242
+ seq_id = seq.seq_id
243
+
244
+ if seq_id not in self.block_tables:
245
# Already freed or haven't been scheduled yet.
246
return
230
- self.block_tables[seq.seq_id].free()
231
- del self.block_tables[seq.seq_id]
247
+
248
+ # Update seq block ids with the latest access time
249
+ self._last_access_blocks_tracker.update_seq_blocks_last_access(
250
+ seq_id, self.block_tables[seq.seq_id].physical_block_ids)
251
+
252
+ # Untrack seq
253
+ self._last_access_blocks_tracker.remove_seq(seq_id)
254
+ self._computed_blocks_tracker.remove_seq(seq_id)
255
+
256
+ # Free table/blocks
257
+ self.block_tables[seq_id].free()
258
+ del self.block_tables[seq_id]
259
260
def free_cross(self, seq_group: SequenceGroup) -> None:
261
request_id = seq_group.request_id
@@ -239,9 +266,7 @@ class BlockSpaceManagerV2(BlockSpaceManager):
266
del self.cross_block_tables[request_id]
267
268
def get_block_table(self, seq: Sequence) -> List[int]:
242
- assert seq.seq_id in self.block_tables
269
block_ids = self.block_tables[seq.seq_id].physical_block_ids
244
- assert all(b is not None for b in block_ids)
270
return block_ids # type: ignore
271
272
def get_cross_block_table(self, seq_group: SequenceGroup) -> List[int]:
@@ -252,20 +277,14 @@ class BlockSpaceManagerV2(BlockSpaceManager):
277
return block_ids # type: ignore
278
279
def access_all_blocks_in_seq(self, seq: Sequence, now: float):
255
- # Update the last accessed time of all the blocks accessed
256
- # in this step.
257
- # And the accessed time is only useful for prefix caching now,
258
- # as it support internal evictor policy for which cached
259
- # block could be refilled, to keep cached content could be reused
260
- # at max extend.
280
if self.enable_caching:
262
- block_table = self.block_tables[seq.seq_id]
263
- block_ids: List[Optional[int]] = []
264
- for block_id in block_table.physical_block_ids:
265
- block_ids.append(block_id)
266
- self.block_allocator.mark_blocks_as_accessed(
267
- block_ids, # type: ignore
268
- now)
281
+ # Record the latest access time for the sequence. The actual update
282
+ # of the block ids is deferred to the sequence free(..) call, since
283
+ # only during freeing of block ids, the blocks are actually added to
284
+ # the evictor (which is when the most updated time is required)
285
+ # (This avoids expensive calls to mark_blocks_as_accessed(..))
286
+ self._last_access_blocks_tracker.update_last_access(
287
+ seq.seq_id, now)
288
289
def mark_blocks_as_computed(self, seq_group: SequenceGroup):
290
# The only need for mark block as computed is for prefix caching,
@@ -285,17 +304,26 @@ class BlockSpaceManagerV2(BlockSpaceManager):
304
This method determines which blocks can be safely skipped for all
305
sequences in the sequence group.
306
"""
288
- seq_block_ids = [
289
- self.block_tables[seq.seq_id].physical_block_ids for seq in seqs
290
- ]
307
+ computed_seq_block_ids = []
308
+ for seq in seqs:
309
+ computed_seq_block_ids.append(
310
+ self._computed_blocks_tracker.
311
+ get_cached_computed_blocks_and_update(
312
+ seq.seq_id,
313
+ self.block_tables[seq.seq_id].physical_block_ids))
314
+
315
# NOTE(sang): This assumes seq_block_ids doesn't contain any None.
316
return self.block_allocator.get_common_computed_block_ids(
293
- seq_block_ids) # type: ignore
317
+ computed_seq_block_ids) # type: ignore
318
319
def fork(self, parent_seq: Sequence, child_seq: Sequence) -> None:
320
src_block_table = self.block_tables[parent_seq.seq_id]
321
self.block_tables[child_seq.seq_id] = src_block_table.fork()
322
323
+ # Track child seq
324
+ self._computed_blocks_tracker.add_seq(child_seq.seq_id)
325
+ self._last_access_blocks_tracker.add_seq(child_seq.seq_id)
326
+
327
def can_swap_in(self, seq_group: SequenceGroup,
328
num_lookahead_slots: int) -> AllocStatus:
329
"""Returns the AllocStatus for the given sequence_group
@@ -323,19 +351,31 @@ class BlockSpaceManagerV2(BlockSpaceManager):
351
List[Tuple[int, int]]: The mapping of swapping block from CPU
352
to GPU.
353
"""
326
- blocks = self._get_blocks_for_swap(seq_group, SequenceStatus.SWAPPED)
327
- current_swap_mapping = self.block_allocator.swap(
328
- blocks=blocks, source_device=Device.CPU, dest_device=Device.GPU)
329
-
330
- block_number_mapping = {
331
- self.block_allocator.get_physical_block_id(Device.CPU,
332
- cpu_block_id):
333
- self.block_allocator.get_physical_block_id(Device.GPU,
334
- gpu_block_id)
335
- for cpu_block_id, gpu_block_id in current_swap_mapping.items()
336
- }
337
- # convert to list of tuples once here
338
- return list(block_number_mapping.items())
354
+ physical_block_id_mapping = []
355
+ for seq in seq_group.get_seqs(status=SequenceStatus.SWAPPED):
356
+ blocks = self.block_tables[seq.seq_id].blocks
357
+ if len(blocks) == 0:
358
+ continue
359
+
360
+ seq_swap_mapping = self.block_allocator.swap(blocks=blocks,
361
+ src_device=Device.CPU,
362
+ dst_device=Device.GPU)
363
+
364
+ # Refresh the block ids of the table (post-swap)
365
+ self.block_tables[seq.seq_id].update(blocks)
366
+
367
+ seq_physical_block_id_mapping = {
368
+ self.block_allocator.get_physical_block_id(
369
+ Device.CPU, cpu_block_id):
370
+ self.block_allocator.get_physical_block_id(
371
+ Device.GPU, gpu_block_id)
372
+ for cpu_block_id, gpu_block_id in seq_swap_mapping.items()
373
+ }
374
+
375
+ physical_block_id_mapping.extend(
376
+ list(seq_physical_block_id_mapping.items()))
377
+
378
+ return physical_block_id_mapping
379
380
def can_swap_out(self, seq_group: SequenceGroup) -> bool:
381
"""Returns whether we can swap out the given sequence_group
@@ -355,7 +395,7 @@ class BlockSpaceManagerV2(BlockSpaceManager):
395
return True
396
return False
397
358
- def swap_out(self, sequence_group: SequenceGroup) -> List[Tuple[int, int]]:
398
+ def swap_out(self, seq_group: SequenceGroup) -> List[Tuple[int, int]]:
399
"""Returns the block id mapping (from GPU to CPU) generated by
400
swapping out the given sequence_group with num_lookahead_slots.
401
@@ -366,19 +406,31 @@ class BlockSpaceManagerV2(BlockSpaceManager):
406
List[Tuple[int, int]]: The mapping of swapping block from
407
GPU to CPU.
408
"""
369
- blocks = self._get_blocks_for_swap(sequence_group,
370
- SequenceStatus.RUNNING)
371
- current_swap_mapping = self.block_allocator.swap(
372
- blocks=blocks, source_device=Device.GPU, dest_device=Device.CPU)
373
- block_number_mapping = {
374
- self.block_allocator.get_physical_block_id(Device.GPU,
375
- gpu_block_id):
376
- self.block_allocator.get_physical_block_id(Device.CPU,
377
- cpu_block_id)
378
- for gpu_block_id, cpu_block_id in current_swap_mapping.items()
379
- }
380
- # convert to list of tuples once here
381
- return list(block_number_mapping.items())
409
+ physical_block_id_mapping = []
410
+ for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
411
+ blocks = self.block_tables[seq.seq_id].blocks
412
+ if len(blocks) == 0:
413
+ continue
414
+
415
+ seq_swap_mapping = self.block_allocator.swap(blocks=blocks,
416
+ src_device=Device.GPU,
417
+ dst_device=Device.CPU)
418
+
419
+ # Refresh the block ids of the table (post-swap)
420
+ self.block_tables[seq.seq_id].update(blocks)
421
+
422
+ seq_physical_block_id_mapping = {
423
+ self.block_allocator.get_physical_block_id(
424
+ Device.GPU, gpu_block_id):
425
+ self.block_allocator.get_physical_block_id(
426
+ Device.CPU, cpu_block_id)
427
+ for gpu_block_id, cpu_block_id in seq_swap_mapping.items()
428
+ }
429
+
430
+ physical_block_id_mapping.extend(
431
+ list(seq_physical_block_id_mapping.items()))
432
+
433
+ return physical_block_id_mapping
434
435
def get_num_free_gpu_blocks(self) -> int:
436
return self.block_allocator.get_num_free_blocks(Device.GPU)
vllm/engine/llm_engine.py
+4
-1
@@ -177,7 +177,8 @@ class LLMEngine:
177
"enforce_eager=%s, kv_cache_dtype=%s, "
178
"quantization_param_path=%s, device_config=%s, "
179
"decoding_config=%r, observability_config=%r, "
180
- "seed=%d, served_model_name=%s)",
180
+ "seed=%d, served_model_name=%s, use_v2_block_manager=%s, "
181
+ "enable_prefix_caching=%s)",
182
VLLM_VERSION,
183
model_config.model,
184
speculative_config,
@@ -204,6 +205,8 @@ class LLMEngine:
205
observability_config,
206
model_config.seed,
207
model_config.served_model_name,
208
+ scheduler_config.use_v2_block_manager,
209
+ cache_config.enable_prefix_caching,
210
)
211
# TODO(woosuk): Print more configs in debug mode.
212
vllm/entrypoints/openai/serving_completion.py
+1
-1
@@ -345,7 +345,7 @@ class OpenAIServingCompletion(OpenAIServing):
345
out_logprobs = prompt_logprobs
346
output_text = prompt_text
347
elif request.echo and request.max_tokens > 0:
348
- token_ids = prompt_token_ids + output.token_ids
348
+ token_ids = prompt_token_ids + list(output.token_ids)
349
out_logprobs = (prompt_logprobs + output.logprobs
350
if request.logprobs is not None else None)
351
output_text = prompt_text + output.text
vllm/model_executor/sampling_metadata.py
+2
-2
@@ -427,8 +427,8 @@ class SamplingTensors:
427
if seq_group.do_sample:
428
for seq_id in seq_ids:
429
seq_data = seq_group.seq_data[seq_id]
430
- prompt_tokens.append(seq_data.prompt_token_ids)
431
- output_tokens.append(seq_data.output_token_ids)
430
+ prompt_tokens.append(list(seq_data.prompt_token_ids))
431
+ output_tokens.append(list(seq_data.output_token_ids))
432
433
sampling_tensors = SamplingTensors.from_lists(
434
temperatures, top_ps, top_ks, min_ps, presence_penalties,
vllm/outputs.py
+2
-2
@@ -1,6 +1,6 @@
1
import time
2
from dataclasses import dataclass
3
-from typing import List, Optional, Union
3
+from typing import List, Optional, Tuple, Union
4
5
from vllm.lora.request import LoRARequest
6
from vllm.sequence import (PromptLogprobs, RequestMetrics, SampleLogprobs,
@@ -28,7 +28,7 @@ class CompletionOutput:
28
29
index: int
30
text: str
31
- token_ids: List[int]
31
+ token_ids: Tuple[int, ...]
32
cumulative_logprob: float
33
logprobs: Optional[SampleLogprobs]
34
finish_reason: Optional[str] = None
vllm/sequence.py
+47
-22
@@ -116,41 +116,66 @@ class SequenceData:
116
prompt_token_ids: List[int],
117
output_token_ids: Optional[List[int]] = None,
118
) -> None:
119
- if output_token_ids is None:
120
- output_token_ids = []
119
+ self._prompt_token_ids: List[int] = list(prompt_token_ids)
120
+ self._prompt_token_ids_tuple: Tuple[int, ...] = tuple(prompt_token_ids)
121
+ self._output_token_ids: List[int] = (
122
+ list(output_token_ids) if output_token_ids is not None else [])
123
122
- self.prompt_token_ids = prompt_token_ids
123
- self._prompt_token_ids_tuple = tuple(prompt_token_ids)
124
- self.output_token_ids = output_token_ids
124
self.cumulative_logprob = 0.0
125
# The number of tokens that are computed (that run against the model).
126
self._num_computed_tokens = 0
127
self._stage: SequenceStage = SequenceStage.PREFILL
128
129
+ self._update_cached_all_tokens()
130
+
131
+ def _update_cached_all_tokens(self):
132
+ self._cached_all_token_ids: List[int] = (self._prompt_token_ids +
133
+ self._output_token_ids)
134
+
135
+ @property
136
+ def prompt_token_ids(self) -> Tuple[int, ...]:
137
+ return self._prompt_token_ids_tuple
138
+
139
+ @prompt_token_ids.setter
140
+ def prompt_token_ids(self, new_prompt_token_ids) -> None:
141
+ self._prompt_token_ids = list(new_prompt_token_ids)
142
+ self._prompt_token_ids_tuple = tuple(new_prompt_token_ids)
143
+ self._update_cached_all_tokens()
144
+
145
+ @property
146
+ def output_token_ids(self) -> Tuple[int, ...]:
147
+ return tuple(self._output_token_ids)
148
+
149
+ @output_token_ids.setter
150
+ def output_token_ids(self, new_output_token_ids) -> None:
151
+ self._output_token_ids = list(new_output_token_ids)
152
+ self._update_cached_all_tokens()
153
+
154
def append_token_id(self, token_id: int, logprob: float) -> None:
131
- self.output_token_ids.append(token_id)
155
+ self._output_token_ids.append(token_id)
156
+ self._cached_all_token_ids.append(token_id)
157
self.cumulative_logprob += logprob
158
159
def get_len(self) -> int:
135
- return len(self.output_token_ids) + len(self.prompt_token_ids)
160
+ return len(self._output_token_ids) + len(self._prompt_token_ids)
161
162
def get_prompt_len(self) -> int:
138
- return len(self.prompt_token_ids)
163
+ return len(self._prompt_token_ids)
164
165
def get_output_len(self) -> int:
141
- return len(self.output_token_ids)
166
+ return len(self._output_token_ids)
167
168
def get_token_ids(self) -> List[int]:
144
- return self.prompt_token_ids + self.output_token_ids
169
+ return self._cached_all_token_ids
170
171
def get_prefix_token_ids(
172
self, num_tokens: int
173
) -> Tuple[Tuple[int, ...], Optional[Tuple[int, ...]]]:
174
"""Get prefix tokens, and make the return value hashable"""
150
- prompt_length = len(self.prompt_token_ids)
175
+ prompt_length = self.get_prompt_len()
176
if num_tokens > prompt_length:
177
return (self._prompt_token_ids_tuple,
153
- tuple(self.output_token_ids[:num_tokens - prompt_length]))
178
+ tuple(self._output_token_ids[:num_tokens - prompt_length]))
179
else:
180
return (self._prompt_token_ids_tuple[:num_tokens], None)
181
@@ -183,14 +208,14 @@ class SequenceData:
208
return self.get_len() - self.get_num_computed_tokens()
209
210
def get_last_token_id(self) -> int:
186
- if not self.output_token_ids:
187
- return self.prompt_token_ids[-1]
188
- return self.output_token_ids[-1]
211
+ if not self._output_token_ids:
212
+ return self._prompt_token_ids[-1]
213
+ return self._output_token_ids[-1]
214
190
- def get_prompt_token_ids(self) -> List[int]:
215
+ def get_prompt_token_ids(self) -> Tuple[int, ...]:
216
return self.prompt_token_ids
217
193
- def get_output_token_ids(self) -> List[int]:
218
+ def get_output_token_ids(self) -> Tuple[int, ...]:
219
return self.output_token_ids
220
221
@property
@@ -199,8 +224,8 @@ class SequenceData:
224
225
def __repr__(self) -> str:
226
return (f"SequenceData("
202
- f"prompt_token_ids={self.prompt_token_ids}, "
203
- f"output_token_ids={self.output_token_ids}, "
227
+ f"prompt_token_ids={self._prompt_token_ids}, "
228
+ f"output_token_ids={self._output_token_ids}, "
229
f"cumulative_logprob={self.cumulative_logprob})")
230
231
@@ -306,14 +331,14 @@ class Sequence:
331
def get_token_ids(self) -> List[int]:
332
return self.data.get_token_ids()
333
309
- def get_prompt_token_ids(self) -> List[int]:
334
+ def get_prompt_token_ids(self) -> Tuple[int, ...]:
335
return self.data.get_prompt_token_ids()
336
337
def get_last_token_id(self) -> int:
338
return self.data.get_last_token_id()
339
315
- def get_output_token_ids(self) -> List[int]:
316
- return self.data.output_token_ids
340
+ def get_output_token_ids(self) -> Tuple[int, ...]:
341
+ return self.data.get_output_token_ids()
342
343
def get_cumulative_logprob(self) -> float:
344
return self.data.cumulative_logprob