A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Bugfix][fast] Fix the get_num_blocks_touched logic (#6849)
Zach Zheng committed
Aug 8, 2024 at 10:43 UTC
782e53ab59308c8cdceba08527574e0e62acc95d
6 files changed
+172
-10
tests/core/block/test_block_manager_v2.py
+62
@@ -311,6 +311,68 @@ def test_swap(block_size, num_cpu_blocks, num_gpu_blocks, num_lookahead_slots,
311
assert before_gpu_blocks == after_gpu_blocks + len(cpu_blocks)
312
313
314
+@pytest.mark.parametrize("block_size", [8])
315
+@pytest.mark.parametrize("num_gpu_blocks", [4])
316
+@pytest.mark.parametrize("num_lookahead_slots", [3, 8, 10])
317
+@pytest.mark.parametrize("enable_caching", [True, False])
318
+def test_can_swap(block_size, num_gpu_blocks, num_lookahead_slots,
319
+ enable_caching):
320
+ """ Verify the block manager can correctly determine if a sequence group
321
+ can be swapped in/out.
322
+ """
323
+ num_cpu_blocks = num_gpu_blocks
324
+ block_manager = BlockSpaceManagerV2(block_size,
325
+ num_cpu_blocks,
326
+ num_gpu_blocks,
327
+ watermark=0,
328
+ enable_caching=enable_caching)
329
+ prompt, seq_group = create_dummy_prompt(
330
+ "1", prompt_length=(num_gpu_blocks - 1) * block_size - 1)
331
+ prompt.status = SequenceStatus.WAITING
332
+ block_manager.allocate(seq_group)
333
+ prompt.status = SequenceStatus.RUNNING
334
+
335
+ # Swap seq group from GPU -> CPU.
336
+ gpu_blocks = block_manager.get_block_table(prompt)
337
+ assert block_manager.can_swap_out(seq_group)
338
+ before_cpu_blocks = block_manager.get_num_free_cpu_blocks()
339
+ before_gpu_blocks = block_manager.get_num_free_gpu_blocks()
340
+ mapping = block_manager.swap_out(seq_group)
341
+ mapping_keys = [key for key, _ in mapping]
342
+ assert mapping_keys == gpu_blocks
343
+ after_cpu_blocks = block_manager.get_num_free_cpu_blocks()
344
+ after_gpu_blocks = block_manager.get_num_free_gpu_blocks()
345
+ assert before_cpu_blocks == after_cpu_blocks + len(gpu_blocks)
346
+ assert before_gpu_blocks + len(gpu_blocks) == after_gpu_blocks
347
+ prompt.status = SequenceStatus.SWAPPED
348
+
349
+ # At this moment, we still have enough free blocks to swap in the seq group.
350
+ if num_lookahead_slots <= block_size:
351
+ assert block_manager.can_swap_in(seq_group,
352
+ num_lookahead_slots) == AllocStatus.OK
353
+ else:
354
+ assert block_manager.can_swap_in(
355
+ seq_group, num_lookahead_slots) == AllocStatus.NEVER
356
+
357
+ # During Swapped out, 2 cached blocks were evicted from the GPU,
358
+ # so the prompt1 can't be swapped in
359
+ prompt2_len = 2 * block_size - 1
360
+ prompt2, seq_group2 = create_dummy_prompt(
361
+ "2",
362
+ prompt_length=prompt2_len,
363
+ prompt_tokens=[10000 + i for i in range(prompt2_len)])
364
+ prompt2.status = SequenceStatus.WAITING
365
+ block_manager.allocate(seq_group2)
366
+
367
+ # Swap seq group from CPU -> GPU.
368
+ if num_lookahead_slots <= block_size:
369
+ assert block_manager.can_swap_in(
370
+ seq_group, num_lookahead_slots) == AllocStatus.LATER
371
+ else:
372
+ assert block_manager.can_swap_in(
373
+ seq_group, num_lookahead_slots) == AllocStatus.NEVER
374
+
375
+
376
# TODO(cade/kaiyang): add comprehensive tests for swapping at allocator level.
377
378
tests/core/block/test_naive_block.py
+42
@@ -100,3 +100,45 @@ class TestNaiveBlockAllocator:
100
for i, block in enumerate(blocks):
101
assert allocator.get_num_free_blocks() == i
102
allocator.free(block)
103
+
104
+ @staticmethod
105
+ @pytest.mark.parametrize("num_blocks", [4])
106
+ @pytest.mark.parametrize("block_size", [8])
107
+ def test_naive_block_get_num_blocks_touched(num_blocks, block_size):
108
+ """ Verify the allocator can correctly return the number of
109
+ blocks touched, with different lookahead slots.
110
+ """
111
+ allocator_src = NaiveBlockAllocator(create_block=NaiveBlock,
112
+ num_blocks=num_blocks,
113
+ block_size=block_size)
114
+ allocator_dst = NaiveBlockAllocator(create_block=NaiveBlock,
115
+ num_blocks=num_blocks,
116
+ block_size=block_size)
117
+
118
+ # Create a chain of cacheable blocks in the dst
119
+ allocate_block = TestNaiveBlockAllocator.create_allocate_lambda(
120
+ "immutable",
121
+ allocator_src,
122
+ prev_block=None,
123
+ token_ids=list(range(block_size)))
124
+ src_blocks = [allocate_block() for _ in range(num_blocks - 1)]
125
+
126
+ # All blocks are cached
127
+ assert allocator_dst.get_num_blocks_touched(
128
+ src_blocks) == num_blocks - 1
129
+
130
+ # Insert one non-full block in the src
131
+ allocate_non_full_block = \
132
+ TestNaiveBlockAllocator.create_allocate_lambda(
133
+ "mutable", allocator_src,
134
+ prev_block=src_blocks[-1],token_ids=[]
135
+ )
136
+ src_blocks.append(allocate_non_full_block())
137
+ src_blocks[-1].append_token_ids([0])
138
+
139
+ assert allocator_dst.get_num_blocks_touched(
140
+ src_blocks, num_lookahead_slots=1) == num_blocks
141
+ assert allocator_dst.get_num_blocks_touched(
142
+ src_blocks, num_lookahead_slots=block_size - 1) == num_blocks
143
+ assert allocator_dst.get_num_blocks_touched(
144
+ src_blocks, num_lookahead_slots=block_size) == (num_blocks + 1)
tests/core/block/test_prefix_caching_block.py
+54
@@ -315,6 +315,60 @@ class TestPrefixCachingBlockAllocator:
315
i)
316
allocator.free(block)
317
318
+ @staticmethod
319
+ @pytest.mark.parametrize("num_blocks", [4])
320
+ @pytest.mark.parametrize("block_size", [8])
321
+ def test_prefix_caching_block_get_num_blocks_touched(
322
+ num_blocks, block_size):
323
+ """ Verify the allocator can correctly return the number of
324
+ blocks touched, when there are cached prefixes and different
325
+ lookahead slots.
326
+ """
327
+ allocator_src = PrefixCachingBlockAllocator(num_blocks=num_blocks,
328
+ block_size=block_size)
329
+ allocator_dst = PrefixCachingBlockAllocator(num_blocks=num_blocks,
330
+ block_size=block_size)
331
+
332
+ # Create token ids that will exhaust all blocks except the last
333
+ token_ids = list(range((num_blocks - 1) * block_size))
334
+
335
+ # Create a chain of cacheable blocks in the dst
336
+ cached_blocks = TestPrefixCachingBlockAllocator.create_immutable_chain(
337
+ block_size=block_size,
338
+ token_ids=token_ids,
339
+ allocator=allocator_dst,
340
+ )
341
+
342
+ # Create a chain of the same blocks in the src
343
+ blocks_to_swap_in = \
344
+ TestPrefixCachingBlockAllocator.create_immutable_chain(
345
+ block_size=block_size,
346
+ token_ids=token_ids,
347
+ allocator=allocator_src,
348
+ )
349
+
350
+ # All blocks are cached
351
+ assert allocator_dst.get_num_blocks_touched(blocks_to_swap_in) == 0
352
+
353
+ # Free the first block in the dst
354
+ allocator_dst.free(cached_blocks[0])
355
+
356
+ # Now the first block becomes dangling, the swapped blocks need
357
+ # to reclaim the first block in the dst
358
+ assert allocator_dst.get_num_blocks_touched(blocks_to_swap_in) == 1
359
+
360
+ # Insert one non-full block in the src
361
+ non_full_block = allocator_src.allocate_mutable_block(
362
+ blocks_to_swap_in[-1])
363
+ non_full_block.append_token_ids([0])
364
+ blocks_to_swap_in.append(non_full_block)
365
+ assert allocator_dst.get_num_blocks_touched(blocks_to_swap_in,
366
+ num_lookahead_slots=1) == 2
367
+ assert allocator_dst.get_num_blocks_touched(
368
+ blocks_to_swap_in, num_lookahead_slots=block_size - 1) == 2
369
+ assert allocator_dst.get_num_blocks_touched(
370
+ blocks_to_swap_in, num_lookahead_slots=block_size) == 3
371
+
372
@staticmethod
373
@pytest.mark.parametrize("num_blocks", [1024])
374
@pytest.mark.parametrize("block_size", [16])
tests/core/utils.py
+5
-3
@@ -15,13 +15,15 @@ def create_dummy_prompt(
15
lora_request: Optional[LoRARequest] = None,
16
use_beam_search: bool = False,
17
best_of: int = 1,
18
+ prompt_tokens: Optional[List[int]] = None,
19
) -> Tuple[Sequence, SequenceGroup]:
20
if not block_size:
21
block_size = prompt_length
22
22
- # Create dummy prompt sequence with tokens 0...block_size-1
23
- # and prompt "0 ... block_size".
24
- prompt_tokens = list(range(prompt_length))
23
+ if prompt_tokens is None:
24
+ # Create dummy prompt sequence with tokens 0...block_size-1
25
+ # and prompt "0 ... block_size".
26
+ prompt_tokens = list(range(prompt_length))
27
prompt_str = " ".join([str(t) for t in prompt_tokens])
28
prompt = Sequence(int(request_id),
29
inputs={
vllm/core/block/naive_block.py
+2
-3
@@ -307,9 +307,8 @@ class NaiveBlockAllocator(BlockAllocator):
307
# TODO(cade): make sure the logic is correct and clean it up.
308
for block in blocks:
309
if not block.is_full and num_lookahead_slots != 0:
310
- if block.num_empty_slots >= num_lookahead_slots:
311
- new_block_count += 1
312
- else:
310
+ new_block_count += 1
311
+ if num_lookahead_slots > block.num_empty_slots:
312
new_block_count += cdiv(
313
num_lookahead_slots - block.num_empty_slots,
314
self._block_size)
vllm/core/block/prefix_caching_block.py
+7
-4
@@ -579,14 +579,17 @@ class PrefixCachingBlockAllocator(BlockAllocator):
579
num_touched_blocks = 0
580
for block in blocks:
581
if not block.is_full:
582
- if block.num_empty_slots >= num_lookahead_slots:
583
- num_touched_blocks += 1
584
- else:
582
+ num_touched_blocks += 1
583
+ if num_lookahead_slots > block.num_empty_slots:
584
num_touched_blocks += cdiv(
585
num_lookahead_slots - block.num_empty_slots,
586
self._block_size)
587
else:
589
- if not self.is_block_cached(block):
588
+ # If the block has a match in the cache and the cached block
589
+ # is not referenced, then we still count it as a touched block
590
+ if not self.is_block_cached(block) or \
591
+ (block.content_hash is not None and \
592
+ self._cached_blocks[block.content_hash] in self.evictor):
593
num_touched_blocks += 1
594
return num_touched_blocks
595