@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 782e53ab59
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Bugfix][fast] Fix the get_num_blocks_touched logic (#6849)

Zach Zheng committed Aug 8, 2024 at 10:43 UTC 782e53ab59308c8cdceba08527574e0e62acc95d
6 files changed +172 -10
tests/core/block/test_block_manager_v2.py
+62
@@ -311,6 +311,68 @@ def test_swap(block_size, num_cpu_blocks, num_gpu_blocks, num_lookahead_slots,
311 assert before_gpu_blocks == after_gpu_blocks + len(cpu_blocks)
312
313
314 +@pytest.mark.parametrize("block_size", [8])
315 +@pytest.mark.parametrize("num_gpu_blocks", [4])
316 +@pytest.mark.parametrize("num_lookahead_slots", [3, 8, 10])
317 +@pytest.mark.parametrize("enable_caching", [True, False])
318 +def test_can_swap(block_size, num_gpu_blocks, num_lookahead_slots,
319 + enable_caching):
320 + """ Verify the block manager can correctly determine if a sequence group
321 + can be swapped in/out.
322 + """
323 + num_cpu_blocks = num_gpu_blocks
324 + block_manager = BlockSpaceManagerV2(block_size,
325 + num_cpu_blocks,
326 + num_gpu_blocks,
327 + watermark=0,
328 + enable_caching=enable_caching)
329 + prompt, seq_group = create_dummy_prompt(
330 + "1", prompt_length=(num_gpu_blocks - 1) * block_size - 1)
331 + prompt.status = SequenceStatus.WAITING
332 + block_manager.allocate(seq_group)
333 + prompt.status = SequenceStatus.RUNNING
334 +
335 + # Swap seq group from GPU -> CPU.
336 + gpu_blocks = block_manager.get_block_table(prompt)
337 + assert block_manager.can_swap_out(seq_group)
338 + before_cpu_blocks = block_manager.get_num_free_cpu_blocks()
339 + before_gpu_blocks = block_manager.get_num_free_gpu_blocks()
340 + mapping = block_manager.swap_out(seq_group)
341 + mapping_keys = [key for key, _ in mapping]
342 + assert mapping_keys == gpu_blocks
343 + after_cpu_blocks = block_manager.get_num_free_cpu_blocks()
344 + after_gpu_blocks = block_manager.get_num_free_gpu_blocks()
345 + assert before_cpu_blocks == after_cpu_blocks + len(gpu_blocks)
346 + assert before_gpu_blocks + len(gpu_blocks) == after_gpu_blocks
347 + prompt.status = SequenceStatus.SWAPPED
348 +
349 + # At this moment, we still have enough free blocks to swap in the seq group.
350 + if num_lookahead_slots <= block_size:
351 + assert block_manager.can_swap_in(seq_group,
352 + num_lookahead_slots) == AllocStatus.OK
353 + else:
354 + assert block_manager.can_swap_in(
355 + seq_group, num_lookahead_slots) == AllocStatus.NEVER
356 +
357 + # During Swapped out, 2 cached blocks were evicted from the GPU,
358 + # so the prompt1 can't be swapped in
359 + prompt2_len = 2 * block_size - 1
360 + prompt2, seq_group2 = create_dummy_prompt(
361 + "2",
362 + prompt_length=prompt2_len,
363 + prompt_tokens=[10000 + i for i in range(prompt2_len)])
364 + prompt2.status = SequenceStatus.WAITING
365 + block_manager.allocate(seq_group2)
366 +
367 + # Swap seq group from CPU -> GPU.
368 + if num_lookahead_slots <= block_size:
369 + assert block_manager.can_swap_in(
370 + seq_group, num_lookahead_slots) == AllocStatus.LATER
371 + else:
372 + assert block_manager.can_swap_in(
373 + seq_group, num_lookahead_slots) == AllocStatus.NEVER
374 +
375 +
376 # TODO(cade/kaiyang): add comprehensive tests for swapping at allocator level.
377
378
tests/core/block/test_naive_block.py
+42
@@ -100,3 +100,45 @@ class TestNaiveBlockAllocator:
100 for i, block in enumerate(blocks):
101 assert allocator.get_num_free_blocks() == i
102 allocator.free(block)
103 +
104 + @staticmethod
105 + @pytest.mark.parametrize("num_blocks", [4])
106 + @pytest.mark.parametrize("block_size", [8])
107 + def test_naive_block_get_num_blocks_touched(num_blocks, block_size):
108 + """ Verify the allocator can correctly return the number of
109 + blocks touched, with different lookahead slots.
110 + """
111 + allocator_src = NaiveBlockAllocator(create_block=NaiveBlock,
112 + num_blocks=num_blocks,
113 + block_size=block_size)
114 + allocator_dst = NaiveBlockAllocator(create_block=NaiveBlock,
115 + num_blocks=num_blocks,
116 + block_size=block_size)
117 +
118 + # Create a chain of cacheable blocks in the dst
119 + allocate_block = TestNaiveBlockAllocator.create_allocate_lambda(
120 + "immutable",
121 + allocator_src,
122 + prev_block=None,
123 + token_ids=list(range(block_size)))
124 + src_blocks = [allocate_block() for _ in range(num_blocks - 1)]
125 +
126 + # All blocks are cached
127 + assert allocator_dst.get_num_blocks_touched(
128 + src_blocks) == num_blocks - 1
129 +
130 + # Insert one non-full block in the src
131 + allocate_non_full_block = \
132 + TestNaiveBlockAllocator.create_allocate_lambda(
133 + "mutable", allocator_src,
134 + prev_block=src_blocks[-1],token_ids=[]
135 + )
136 + src_blocks.append(allocate_non_full_block())
137 + src_blocks[-1].append_token_ids([0])
138 +
139 + assert allocator_dst.get_num_blocks_touched(
140 + src_blocks, num_lookahead_slots=1) == num_blocks
141 + assert allocator_dst.get_num_blocks_touched(
142 + src_blocks, num_lookahead_slots=block_size - 1) == num_blocks
143 + assert allocator_dst.get_num_blocks_touched(
144 + src_blocks, num_lookahead_slots=block_size) == (num_blocks + 1)
tests/core/block/test_prefix_caching_block.py
+54
@@ -315,6 +315,60 @@ class TestPrefixCachingBlockAllocator:
315 i)
316 allocator.free(block)
317
318 + @staticmethod
319 + @pytest.mark.parametrize("num_blocks", [4])
320 + @pytest.mark.parametrize("block_size", [8])
321 + def test_prefix_caching_block_get_num_blocks_touched(
322 + num_blocks, block_size):
323 + """ Verify the allocator can correctly return the number of
324 + blocks touched, when there are cached prefixes and different
325 + lookahead slots.
326 + """
327 + allocator_src = PrefixCachingBlockAllocator(num_blocks=num_blocks,
328 + block_size=block_size)
329 + allocator_dst = PrefixCachingBlockAllocator(num_blocks=num_blocks,
330 + block_size=block_size)
331 +
332 + # Create token ids that will exhaust all blocks except the last
333 + token_ids = list(range((num_blocks - 1) * block_size))
334 +
335 + # Create a chain of cacheable blocks in the dst
336 + cached_blocks = TestPrefixCachingBlockAllocator.create_immutable_chain(
337 + block_size=block_size,
338 + token_ids=token_ids,
339 + allocator=allocator_dst,
340 + )
341 +
342 + # Create a chain of the same blocks in the src
343 + blocks_to_swap_in = \
344 + TestPrefixCachingBlockAllocator.create_immutable_chain(
345 + block_size=block_size,
346 + token_ids=token_ids,
347 + allocator=allocator_src,
348 + )
349 +
350 + # All blocks are cached
351 + assert allocator_dst.get_num_blocks_touched(blocks_to_swap_in) == 0
352 +
353 + # Free the first block in the dst
354 + allocator_dst.free(cached_blocks[0])
355 +
356 + # Now the first block becomes dangling, the swapped blocks need
357 + # to reclaim the first block in the dst
358 + assert allocator_dst.get_num_blocks_touched(blocks_to_swap_in) == 1
359 +
360 + # Insert one non-full block in the src
361 + non_full_block = allocator_src.allocate_mutable_block(
362 + blocks_to_swap_in[-1])
363 + non_full_block.append_token_ids([0])
364 + blocks_to_swap_in.append(non_full_block)
365 + assert allocator_dst.get_num_blocks_touched(blocks_to_swap_in,
366 + num_lookahead_slots=1) == 2
367 + assert allocator_dst.get_num_blocks_touched(
368 + blocks_to_swap_in, num_lookahead_slots=block_size - 1) == 2
369 + assert allocator_dst.get_num_blocks_touched(
370 + blocks_to_swap_in, num_lookahead_slots=block_size) == 3
371 +
372 @staticmethod
373 @pytest.mark.parametrize("num_blocks", [1024])
374 @pytest.mark.parametrize("block_size", [16])
tests/core/utils.py
+5 -3
@@ -15,13 +15,15 @@ def create_dummy_prompt(
15 lora_request: Optional[LoRARequest] = None,
16 use_beam_search: bool = False,
17 best_of: int = 1,
18 + prompt_tokens: Optional[List[int]] = None,
19 ) -> Tuple[Sequence, SequenceGroup]:
20 if not block_size:
21 block_size = prompt_length
22
22 - # Create dummy prompt sequence with tokens 0...block_size-1
23 - # and prompt "0 ... block_size".
24 - prompt_tokens = list(range(prompt_length))
23 + if prompt_tokens is None:
24 + # Create dummy prompt sequence with tokens 0...block_size-1
25 + # and prompt "0 ... block_size".
26 + prompt_tokens = list(range(prompt_length))
27 prompt_str = " ".join([str(t) for t in prompt_tokens])
28 prompt = Sequence(int(request_id),
29 inputs={
vllm/core/block/naive_block.py
+2 -3
@@ -307,9 +307,8 @@ class NaiveBlockAllocator(BlockAllocator):
307 # TODO(cade): make sure the logic is correct and clean it up.
308 for block in blocks:
309 if not block.is_full and num_lookahead_slots != 0:
310 - if block.num_empty_slots >= num_lookahead_slots:
311 - new_block_count += 1
312 - else:
310 + new_block_count += 1
311 + if num_lookahead_slots > block.num_empty_slots:
312 new_block_count += cdiv(
313 num_lookahead_slots - block.num_empty_slots,
314 self._block_size)
vllm/core/block/prefix_caching_block.py
+7 -4
@@ -579,14 +579,17 @@ class PrefixCachingBlockAllocator(BlockAllocator):
579 num_touched_blocks = 0
580 for block in blocks:
581 if not block.is_full:
582 - if block.num_empty_slots >= num_lookahead_slots:
583 - num_touched_blocks += 1
584 - else:
582 + num_touched_blocks += 1
583 + if num_lookahead_slots > block.num_empty_slots:
584 num_touched_blocks += cdiv(
585 num_lookahead_slots - block.num_empty_slots,
586 self._block_size)
587 else:
589 - if not self.is_block_cached(block):
588 + # If the block has a match in the cache and the cached block
589 + # is not referenced, then we still count it as a touched block
590 + if not self.is_block_cached(block) or \
591 + (block.content_hash is not None and \
592 + self._cached_blocks[block.content_hash] in self.evictor):
593 num_touched_blocks += 1
594 return num_touched_blocks
595