@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5bf8789b2a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Bugfix] Block manager v2 with preemption and lookahead slots (#8824)

sroy745 committed Sep 28, 2024 at 18:17 UTC 5bf8789b2a28df1305f92b9999fe60264f839caa
9 files changed +133 -116
tests/basic_correctness/test_preemption.py
+7 -2
@@ -23,8 +23,10 @@ MODELS = [
23 @pytest.fixture(scope="module", autouse=True)
24 def check_settings():
25 assert ENABLE_ARTIFICIAL_PREEMPT is True, (
26 - "Use an env var VLLM_TEST_ENABLE_ARTIFICIAL_PREEMPT=1. "
27 - "`VLLM_TEST_ENABLE_ARTIFICIAL_PREEMPT=1 pytest "
26 + "Use an env var VLLM_TEST_ENABLE_ARTIFICIAL_PREEMPT=1, "
27 + "VLLM_ALLOW_DEPRECATED_BEAM_SEARCH=1. "
28 + "`VLLM_TEST_ENABLE_ARTIFICIAL_PREEMPT=1 "
29 + "VLLM_ALLOW_DEPRECATED_BEAM_SEARCH=1 pytest "
30 "tests/basic_correctness/test_preemption.py`")
31
32
@@ -199,6 +201,7 @@ def test_swap(
201 @pytest.mark.parametrize("dtype", ["float"])
202 @pytest.mark.parametrize("max_tokens", [96])
203 @pytest.mark.parametrize("beam_width", [4])
204 +@pytest.mark.parametrize("use_v2_block_manager", [True, False])
205 def test_swap_infeasible(
206 vllm_runner,
207 example_prompts,
@@ -207,6 +210,7 @@ def test_swap_infeasible(
210 max_tokens: int,
211 beam_width: int,
212 worker_use_ray: bool,
213 + use_v2_block_manager: bool,
214 ) -> None:
215 """Verify infeasible swap request will be ignored."""
216 BLOCK_SIZE = 16
@@ -223,6 +227,7 @@ def test_swap_infeasible(
227 num_gpu_blocks_override=prefill_blocks + decode_blocks,
228 max_model_len=(prefill_blocks + decode_blocks) * BLOCK_SIZE,
229 worker_use_ray=worker_use_ray,
230 + use_v2_block_manager=use_v2_block_manager,
231 ) as vllm_model:
232 sampling_params = SamplingParams(n=beam_width,
233 use_beam_search=True,
tests/core/block/test_block_manager_v2.py
+46 -1
@@ -373,6 +373,52 @@ def test_can_swap(block_size, num_gpu_blocks, num_lookahead_slots,
373 seq_group, num_lookahead_slots) == AllocStatus.NEVER
374
375
376 +@pytest.mark.parametrize("num_lookahead_slots", [0, 2, 10])
377 +@pytest.mark.parametrize("enable_caching", [False, True])
378 +def test_swap_in_infeasible(num_lookahead_slots, enable_caching):
379 + """Verifies that swapping fails if there is not enough free blocks
380 + to account for unseen tokens and lookahead_slots.
381 + """
382 + block_size = 8
383 + num_cpu_blocks = 1
384 + num_gpu_blocks = 1
385 + block_manager = BlockSpaceManagerV2(block_size,
386 + num_cpu_blocks,
387 + num_gpu_blocks,
388 + watermark=0,
389 + enable_caching=enable_caching)
390 + prompt_length = block_size - 3
391 + assert prompt_length > 0
392 + prompt, seq_group = create_dummy_prompt("1", prompt_length=prompt_length)
393 + prompt.status = SequenceStatus.WAITING
394 + block_manager.allocate(seq_group)
395 + # Emulate a forward pass by appending a single token.
396 + # The block manager then knows how many unprocessed
397 + # tokens will be written in the next forward pass.
398 + token_id = 0
399 + prompt.status = SequenceStatus.RUNNING
400 + prompt.append_token_id(token_id, {token_id: Logprob(0.0)})
401 +
402 + # Swap seq group from GPU -> CPU.
403 + assert block_manager.can_swap_out(seq_group)
404 + block_manager.swap_out(seq_group)
405 + prompt.status = SequenceStatus.SWAPPED
406 +
407 + # Swap seq group from CPU -> GPU.
408 + # The number of unseen tokens is 1. If the number of existing
409 + # tokens plus the unseen ones and number of lookahead slots exceeds
410 + # the total number of available GPU blocks then the swap
411 + # should fail.
412 + num_unseen_tokens = 1
413 + if (num_lookahead_slots + num_unseen_tokens +
414 + prompt_length) <= (block_size * num_gpu_blocks):
415 + assert block_manager.can_swap_in(seq_group,
416 + num_lookahead_slots) == AllocStatus.OK
417 + else:
418 + assert block_manager.can_swap_in(
419 + seq_group, num_lookahead_slots) == AllocStatus.NEVER
420 +
421 +
422 # TODO(cade/kaiyang): add comprehensive tests for swapping at allocator level.
423
424
@@ -400,7 +446,6 @@ def test_sliding_window(block_size, prompt_len, num_slots_to_append,
446 if max_n is None:
447 max_n = min_n
448 used = num_gpu_blocks - block_manager.get_num_free_gpu_blocks()
403 - #print("check", min_n, used, max_n)
449 assert min_n <= used
450 assert used <= max_n
451
tests/core/block/test_naive_block.py
+10 -9
@@ -104,9 +104,9 @@ class TestNaiveBlockAllocator:
104 @staticmethod
105 @pytest.mark.parametrize("num_blocks", [4])
106 @pytest.mark.parametrize("block_size", [8])
107 - def test_naive_block_get_num_blocks_touched(num_blocks, block_size):
107 + def test_naive_block_get_num_full_blocks_touched(num_blocks, block_size):
108 """ Verify the allocator can correctly return the number of
109 - blocks touched, with different lookahead slots.
109 + full blocks touched.
110 """
111 allocator_src = NaiveBlockAllocator(create_block=NaiveBlock,
112 num_blocks=num_blocks,
@@ -124,7 +124,7 @@ class TestNaiveBlockAllocator:
124 src_blocks = [allocate_block() for _ in range(num_blocks - 1)]
125
126 # All blocks are cached
127 - assert allocator_dst.get_num_blocks_touched(
127 + assert allocator_dst.get_num_full_blocks_touched(
128 src_blocks) == num_blocks - 1
129
130 # Insert one non-full block in the src
@@ -136,9 +136,10 @@ class TestNaiveBlockAllocator:
136 src_blocks.append(allocate_non_full_block())
137 src_blocks[-1].append_token_ids([0])
138
139 - assert allocator_dst.get_num_blocks_touched(
140 - src_blocks, num_lookahead_slots=1) == num_blocks
141 - assert allocator_dst.get_num_blocks_touched(
142 - src_blocks, num_lookahead_slots=block_size - 1) == num_blocks
143 - assert allocator_dst.get_num_blocks_touched(
144 - src_blocks, num_lookahead_slots=block_size) == (num_blocks + 1)
139 + assert allocator_dst.get_num_full_blocks_touched(
140 + src_blocks) == num_blocks - 1
141 + # Fill up the last source block and then invoke
142 + # get_num_blocks_touched
143 + src_blocks[-1].append_token_ids([0] * (block_size - 1))
144 + assert allocator_dst.get_num_full_blocks_touched(
145 + src_blocks) == num_blocks
tests/core/block/test_prefix_caching_block.py
+13 -12
@@ -318,11 +318,10 @@ class TestPrefixCachingBlockAllocator:
318 @staticmethod
319 @pytest.mark.parametrize("num_blocks", [4])
320 @pytest.mark.parametrize("block_size", [8])
321 - def test_prefix_caching_block_get_num_blocks_touched(
321 + def test_prefix_caching_block_get_num_full_blocks_touched(
322 num_blocks, block_size):
323 """ Verify the allocator can correctly return the number of
324 - blocks touched, when there are cached prefixes and different
325 - lookahead slots.
324 + blocks touched, when there are cached prefixes.
325 """
326 allocator_src = PrefixCachingBlockAllocator(num_blocks=num_blocks,
327 block_size=block_size)
@@ -346,28 +345,30 @@ class TestPrefixCachingBlockAllocator:
345 token_ids=token_ids,
346 allocator=allocator_src,
347 )
349 -
348 # All blocks are cached
351 - assert allocator_dst.get_num_blocks_touched(blocks_to_swap_in) == 0
349 + assert allocator_dst.get_num_full_blocks_touched(
350 + blocks_to_swap_in) == 0
351
352 # Free the first block in the dst
353 allocator_dst.free(cached_blocks[0])
354
355 # Now the first block becomes dangling, the swapped blocks need
356 # to reclaim the first block in the dst
358 - assert allocator_dst.get_num_blocks_touched(blocks_to_swap_in) == 1
357 + assert allocator_dst.get_num_full_blocks_touched(
358 + blocks_to_swap_in) == 1
359
360 # Insert one non-full block in the src
361 non_full_block = allocator_src.allocate_mutable_block(
362 blocks_to_swap_in[-1])
363 non_full_block.append_token_ids([0])
364 blocks_to_swap_in.append(non_full_block)
365 - assert allocator_dst.get_num_blocks_touched(blocks_to_swap_in,
366 - num_lookahead_slots=1) == 2
367 - assert allocator_dst.get_num_blocks_touched(
368 - blocks_to_swap_in, num_lookahead_slots=block_size - 1) == 2
369 - assert allocator_dst.get_num_blocks_touched(
370 - blocks_to_swap_in, num_lookahead_slots=block_size) == 3
365 + assert allocator_dst.get_num_full_blocks_touched(
366 + blocks_to_swap_in) == 1
367 + # Fill up the last mutable block and invoke get_num_blocks_touched.
368 + # Note: The last block is not cached so it will be touched.
369 + non_full_block.append_token_ids([0] * (block_size - 1))
370 + assert allocator_dst.get_num_full_blocks_touched(
371 + blocks_to_swap_in) == 2
372
373 @staticmethod
374 @pytest.mark.parametrize("num_blocks", [1024])
vllm/core/block/cpu_gpu_block_allocator.py
+7 -10
@@ -259,25 +259,22 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
259 current_swap_mapping[src_block_id] = dst_block_id
260 return current_swap_mapping
261
262 - def get_num_blocks_touched(self,
263 - blocks: List[Block],
264 - device: Device,
265 - num_lookahead_slots: int = 0) -> int:
266 - """Returns the number of blocks that will be touched by
262 + def get_num_full_blocks_touched(self, blocks: List[Block],
263 + device: Device) -> int:
264 + """Returns the number of full blocks that will be touched by
265 swapping in/out the given blocks on to the 'device'.
266
267 Args:
268 blocks: List of blocks to be swapped.
269 device (Device): Device to swap the 'blocks' on.
272 - num_lookahead_slots (int): Number of lookahead slots used in
273 - speculative decoding, default to 0.
270
271 Returns:
276 - int: the number of blocks that will be touched by
272 + int: the number of full blocks that will be touched by
273 swapping in/out the given blocks on to the 'device'.
274 + Non full blocks are ignored when deciding the number
275 + of blocks to touch.
276 """
279 - return self._allocators[device].get_num_blocks_touched(
280 - blocks, num_lookahead_slots)
277 + return self._allocators[device].get_num_full_blocks_touched(blocks)
278
279 def clear_copy_on_writes(self) -> List[Tuple[int, int]]:
280 """Clears the copy-on-write (CoW) state and returns the mapping of
vllm/core/block/interfaces.py
+3 -7
@@ -181,9 +181,7 @@ class BlockAllocator(ABC):
181 pass
182
183 @abstractmethod
184 - def get_num_blocks_touched(self,
185 - blocks: List[Block],
186 - num_lookahead_slots: int = 0) -> int:
184 + def get_num_full_blocks_touched(self, blocks: List[Block]) -> int:
185 pass
186
187 @abstractmethod
@@ -260,10 +258,8 @@ class DeviceAwareBlockAllocator(ABC):
258 pass
259
260 @abstractmethod
263 - def get_num_blocks_touched(self,
264 - blocks: List[Block],
265 - device: Device,
266 - num_lookahead_slots: int = 0) -> int:
261 + def get_num_full_blocks_touched(self, blocks: List[Block],
262 + device: Device) -> int:
263 pass
264
265 @abstractmethod
vllm/core/block/naive_block.py
+10 -25
@@ -4,7 +4,6 @@ from typing import Deque, FrozenSet, Iterable, List, Optional, Tuple
4 from vllm.core.block.common import (BlockPool, CopyOnWriteTracker, RefCounter,
5 get_all_blocks_recursively)
6 from vllm.core.block.interfaces import Block, BlockAllocator, BlockId, Device
7 -from vllm.utils import cdiv
7
8 Refcount = int
9
@@ -282,40 +281,26 @@ class NaiveBlockAllocator(BlockAllocator):
281 def promote_to_immutable_block(self, block: Block) -> BlockId:
282 raise NotImplementedError("There is no promotion for naive blocks")
283
285 - def get_num_blocks_touched(self,
286 - blocks: List[Block],
287 - num_lookahead_slots: int = 0) -> int:
288 - """Determine the number of blocks that will be touched by
289 - swapping in/out the given blocks from certain sequence
290 - group with the provided num_lookahead_slots.
284 + def get_num_full_blocks_touched(self, blocks: List[Block]) -> int:
285 + """Returns the number of full blocks that will be touched by
286 + swapping in/out.
287
288 Args:
293 - blocks (List[Block]): The potential blocks to swap.
294 - num_lookahead_slots (int): number of lookahead slots (0 for swap
295 - out).
296 -
289 + blocks: List of blocks to be swapped.
290 Returns:
298 - int: the number of blocks that will be touched by
299 - swapping in/out the given blocks and num_lookahead_slots.
291 + int: the number of full blocks that will be touched by
292 + swapping in/out the given blocks. Non full blocks are ignored
293 + when deciding the number of blocks to touch.
294 """
295 # NOTE: for naive block, we use set to eliminate common blocks among
296 # seqs, also we compare the empty slots in the mutable blocks with
297 # lookahead slots to get the number of unique new block that are
298 # needed.
299 old_block_set = set()
306 - new_block_count = 0
307 - # TODO(cade): make sure the logic is correct and clean it up.
300 for block in blocks:
309 - if not block.is_full and num_lookahead_slots != 0:
310 - new_block_count += 1
311 - if num_lookahead_slots > block.num_empty_slots:
312 - new_block_count += cdiv(
313 - num_lookahead_slots - block.num_empty_slots,
314 - self._block_size)
315 - else:
316 - old_block_set.add(block.block_id)
317 - num_touched_blocks = new_block_count + len(old_block_set)
318 - return num_touched_blocks
301 + if block.is_full:
302 + old_block_set.add(block)
303 + return len(old_block_set)
304
305 def swap_out(self, blocks: List[Block]) -> None:
306 for block in blocks:
vllm/core/block/prefix_caching_block.py
+15 -26
@@ -8,7 +8,6 @@ from vllm.core.block.interfaces import Block, BlockAllocator, BlockId, Device
8 from vllm.core.block.naive_block import (BlockPool, NaiveBlock,
9 NaiveBlockAllocator)
10 from vllm.core.evictor_v2 import EvictionPolicy, Evictor, make_evictor
11 -from vllm.utils import cdiv
11
12 PrefixHash = int
13
@@ -576,37 +575,27 @@ class PrefixCachingBlockAllocator(BlockAllocator):
575 if ids
576 ])
577
579 - def get_num_blocks_touched(self,
580 - blocks: List[Block],
581 - num_lookahead_slots: int = 0) -> int:
582 - """Determine the number of blocks that will be touched by
583 - swapping in/out the given blocks from certain sequence
584 - group with the provided num_lookahead_slots.
578 + def get_num_full_blocks_touched(self, blocks: List[Block]) -> int:
579 + """Returns the number of full blocks that will be touched by
580 + swapping in/out.
581
582 Args:
587 - blocks (List[Block]): The potential blocks to swap.
588 - num_lookahead_slots (int): number of lookahead slots (0 for
589 - swap out).
590 -
583 + blocks: List of blocks to be swapped.
584 Returns:
592 - int: the number of blocks that will be touched by
593 - swapping in/out the given blocks and num_lookahead_slots.
585 + int: the number of full blocks that will be touched by
586 + swapping in/out the given blocks. Non full blocks are ignored
587 + when deciding the number of blocks to touch.
588 """
595 - num_touched_blocks = 0
589 + num_touched_blocks: int = 0
590 for block in blocks:
597 - if not block.is_full:
591 + # If the block has a match in the cache and the cached
592 + # block is not referenced, then we still count it as a
593 + # touched block
594 + if block.is_full and (not self.is_block_cached(block) or \
595 + (block.content_hash is not None and \
596 + self._cached_blocks[block.content_hash] in \
597 + self.evictor)):
598 num_touched_blocks += 1
599 - if num_lookahead_slots > block.num_empty_slots:
600 - num_touched_blocks += cdiv(
601 - num_lookahead_slots - block.num_empty_slots,
602 - self._block_size)
603 - else:
604 - # If the block has a match in the cache and the cached block
605 - # is not referenced, then we still count it as a touched block
606 - if not self.is_block_cached(block) or \
607 - (block.content_hash is not None and \
608 - self._cached_blocks[block.content_hash] in self.evictor):
609 - num_touched_blocks += 1
599 return num_touched_blocks
600
601 def swap_out(self, blocks: List[Block]) -> None:
vllm/core/block_manager_v2.py
+22 -24
@@ -1,5 +1,4 @@
1 """A block manager that manages token blocks."""
2 -from itertools import chain
2 from typing import Dict, List, Optional
3 from typing import Sequence as GenericSequence
4 from typing import Tuple
@@ -470,12 +469,31 @@ class BlockSpaceManagerV2(BlockSpaceManager):
469 AllocStatus: The AllocStatus for swapping in/out the given
470 sequence_group on to the 'device'.
471 """
473 - blocks = self._get_blocks_for_swap(seq_group, status)
474 - num_blocks_touched = self.block_allocator.get_num_blocks_touched(
475 - blocks, device, num_lookahead_slots)
472 + # First determine the number of blocks that will be touched by this
473 + # swap. Then verify if there are available blocks in the device
474 + # to perform the swap.
475 + num_blocks_touched = 0
476 + blocks: List[Block] = []
477 + for seq in seq_group.get_seqs(status=status):
478 + block_table = self.block_tables[seq.seq_id]
479 + if block_table.blocks is not None:
480 + # Compute the number blocks to touch for the tokens to be
481 + # appended. This does NOT include the full blocks that need
482 + # to be touched for the swap.
483 + num_blocks_touched += \
484 + block_table.get_num_blocks_touched_by_append_slots(
485 + block_table.get_unseen_token_ids(seq.get_token_ids()),
486 + num_lookahead_slots=num_lookahead_slots)
487 + blocks.extend(block_table.blocks)
488 + # Compute the number of full blocks to touch and add it to the
489 + # existing count of blocks to touch.
490 + num_blocks_touched += self.block_allocator.get_num_full_blocks_touched(
491 + blocks, device=device)
492 +
493 watermark_blocks = 0
494 if device == Device.GPU:
495 watermark_blocks = self.watermark_blocks
496 +
497 if self.block_allocator.get_num_total_blocks(
498 device) < num_blocks_touched:
499 return AllocStatus.NEVER
@@ -484,23 +502,3 @@ class BlockSpaceManagerV2(BlockSpaceManager):
502 return AllocStatus.OK
503 else:
504 return AllocStatus.LATER
487 -
488 - def _get_blocks_for_swap(self, seq_group: SequenceGroup,
489 - status: SequenceStatus) -> List[Block]:
490 - """Returns the list of blocks those are touched by the seq_group
491 -
492 - Args:
493 - sequence_group (SequenceGroup): The sequence group to swap in.
494 - status (SequenceStatus): The status of sequence which is needed
495 - for action. RUNNING for swap out and SWAPPED for swap in
496 -
497 - Returns:
498 - The list of blocks those are touched by the seq_group.
499 - """
500 - blocks: Dict[int, List[Block]] = {}
501 - for seq in seq_group.get_seqs(status=status):
502 - block_table = self.block_tables[seq.seq_id]
503 - if block_table.blocks is not None:
504 - blocks[seq.seq_id] = block_table.blocks
505 - combined_blocks = list(chain(*blocks.values()))
506 - return combined_blocks