@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 10c38e3e46
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Misc]: Implement CPU/GPU swapping in BlockManagerV2 (#3834)

Kaiyang Chen committed Jun 4, 2024 at 04:37 UTC 10c38e3e46d95a02576c92c25d5b3e88e3c6e282
16 files changed +529 -49
format.sh
+1 -1
@@ -118,7 +118,7 @@ mypy vllm/model_executor --config-file pyproject.toml
118 # https://github.com/codespell-project/codespell/issues/1915
119 # Avoiding the "./" prefix and using "/**" globs for directories appears to solve the problem
120 CODESPELL_EXCLUDES=(
121 - '--skip' 'tests/prompts/**,./benchmarks/sonnet.txt,tests/lora/data/**,build/**'
121 + '--skip' 'tests/prompts/**,./benchmarks/sonnet.txt,*tests/lora/data/**,build/**'
122 )
123
124 # check spelling of specified files
tests/core/block/e2e/test_correctness.py
+40 -9
@@ -24,7 +24,13 @@ from .conftest import get_token_ids_from_llm_generator
24 @pytest.mark.parametrize("baseline_llm_kwargs", [{
25 "use_v2_block_manager": False
26 }])
27 -@pytest.mark.parametrize("test_llm_kwargs", [{"use_v2_block_manager": True}])
27 +@pytest.mark.parametrize("test_llm_kwargs", [{
28 + "use_v2_block_manager": True,
29 + "preemption_mode": "swap"
30 +}, {
31 + "use_v2_block_manager": True,
32 + "preemption_mode": "recompute"
33 +}])
34 @pytest.mark.parametrize("batch_size", [10])
35 @pytest.mark.parametrize("seed", [1])
36 def test_v1_v2_greedy_equality_with_preemption(baseline_llm_generator,
@@ -95,7 +101,13 @@ def test_v1_v2_greedy_equality_with_preemption(baseline_llm_generator,
101 @pytest.mark.parametrize("baseline_llm_kwargs", [{
102 "use_v2_block_manager": False
103 }])
98 -@pytest.mark.parametrize("test_llm_kwargs", [{"use_v2_block_manager": True}])
104 +@pytest.mark.parametrize("test_llm_kwargs", [{
105 + "use_v2_block_manager": True,
106 + "preemption_mode": "swap"
107 +}, {
108 + "use_v2_block_manager": True,
109 + "preemption_mode": "recompute"
110 +}])
111 @pytest.mark.parametrize("batch_size", [10])
112 @pytest.mark.parametrize("seed", [1])
113 def test_v1_v2_greedy_equality_with_cow(baseline_llm_generator,
@@ -179,11 +191,18 @@ def test_v1_v2_greedy_equality_with_cow(baseline_llm_generator,
191 }])
192 @pytest.mark.parametrize(
193 "test_llm_kwargs",
182 - [{
183 - # We run one test with block_size < lookahead_slots, one test with
184 - # block_size > lookahead_slots
185 - "num_lookahead_slots": 10,
186 - }])
194 + [
195 + {
196 + # We run one test with block_size < lookahead_slots, one test with
197 + # block_size > lookahead_slots
198 + "num_lookahead_slots": 10,
199 + "preemption_mode": "swap",
200 + },
201 + {
202 + "num_lookahead_slots": 10,
203 + "preemption_mode": "recompute",
204 + }
205 + ])
206 @pytest.mark.parametrize("batch_size", [4])
207 @pytest.mark.parametrize("seed", [1])
208 def test_lookahead_greedy_equality_with_preemption(baseline_llm_generator,
@@ -322,7 +341,13 @@ def test_chunked_prefill_block_manager_v2(baseline_llm_generator,
341 @pytest.mark.parametrize("baseline_llm_kwargs", [{
342 "use_v2_block_manager": False
343 }])
325 -@pytest.mark.parametrize("test_llm_kwargs", [{"use_v2_block_manager": True}])
344 +@pytest.mark.parametrize("test_llm_kwargs", [{
345 + "use_v2_block_manager": True,
346 + "preemption_mode": "swap"
347 +}, {
348 + "use_v2_block_manager": True,
349 + "preemption_mode": "recompute"
350 +}])
351 @pytest.mark.parametrize("batch_size", [10])
352 @pytest.mark.parametrize("seed", [1])
353 def test_v1_v2_greedy_equality_prefix_caching_enabled_with_preemption(
@@ -397,7 +422,13 @@ def test_v1_v2_greedy_equality_prefix_caching_enabled_with_preemption(
422 @pytest.mark.parametrize("baseline_llm_kwargs", [{
423 "enable_prefix_caching": False
424 }])
400 -@pytest.mark.parametrize("test_llm_kwargs", [{"enable_prefix_caching": True}])
425 +@pytest.mark.parametrize("test_llm_kwargs", [{
426 + "enable_prefix_caching": True,
427 + "preemption_mode": "swap"
428 +}, {
429 + "enable_prefix_caching": True,
430 + "preemption_mode": "recompute"
431 +}])
432 @pytest.mark.parametrize("batch_size", [10])
433 @pytest.mark.parametrize("seed", [1])
434 def test_auto_prefix_caching_with_preemption(baseline_llm_generator,
tests/core/block/test_block_manager_v2.py
+57 -1
@@ -7,7 +7,8 @@ from vllm.core.interfaces import AllocStatus
7 from vllm.sequence import Logprob, SequenceStatus
8 from vllm.utils import chunk_list
9
10 -from ..utils import create_seq_group, create_seq_group_encoder_decoder
10 +from ..utils import (create_dummy_prompt, create_seq_group,
11 + create_seq_group_encoder_decoder)
12
13
14 @pytest.mark.parametrize("block_size", [16])
@@ -255,6 +256,61 @@ def test_append_slots(block_size, prompt_len, num_slots_to_append,
256 assert num_consumed_blocks == expected_consumed_blocks
257
258
259 +@pytest.mark.parametrize("block_size", [8])
260 +@pytest.mark.parametrize("num_cpu_blocks", [4])
261 +@pytest.mark.parametrize("num_gpu_blocks", [4])
262 +@pytest.mark.parametrize("num_lookahead_slots", [0, 2, 10])
263 +@pytest.mark.parametrize("enable_caching", [False, True])
264 +def test_swap(block_size, num_cpu_blocks, num_gpu_blocks, num_lookahead_slots,
265 + enable_caching):
266 + """Verify blocks number on src/desc device is correct after swapping in/out
267 + sequence group (not missing or extra blocks).
268 + """
269 + block_manager = BlockSpaceManagerV2(block_size,
270 + num_cpu_blocks,
271 + num_gpu_blocks,
272 + watermark=0,
273 + enable_caching=enable_caching)
274 + prompt, seq_group = create_dummy_prompt("1", prompt_length=block_size - 1)
275 + prompt.status = SequenceStatus.WAITING
276 + block_manager.allocate(seq_group)
277 + # Emulate a forward pass by appending a single token.
278 + # The block manager then knows how many unprocessed
279 + # tokens will be written in the next forward pass.
280 + token_id = 0
281 + prompt.status = SequenceStatus.RUNNING
282 + prompt.append_token_id(token_id, {token_id: Logprob(0.0)})
283 +
284 + # Swap seq group from GPU -> CPU.
285 + gpu_blocks = block_manager.get_block_table(prompt)
286 + assert block_manager.can_swap_out(seq_group)
287 + before_cpu_blocks = block_manager.get_num_free_cpu_blocks()
288 + before_gpu_blocks = block_manager.get_num_free_gpu_blocks()
289 + mapping = block_manager.swap_out(seq_group)
290 + mapping_keys = [key for key, _ in mapping]
291 + assert mapping_keys == gpu_blocks
292 + after_cpu_blocks = block_manager.get_num_free_cpu_blocks()
293 + after_gpu_blocks = block_manager.get_num_free_gpu_blocks()
294 + assert before_cpu_blocks == after_cpu_blocks + len(gpu_blocks)
295 + assert before_gpu_blocks + len(gpu_blocks) == after_gpu_blocks
296 + prompt.status = SequenceStatus.SWAPPED
297 +
298 + # Swap seq group from CPU -> GPU.
299 + assert block_manager.can_swap_in(seq_group, num_lookahead_slots)
300 + before_cpu_blocks = block_manager.get_num_free_cpu_blocks()
301 + before_gpu_blocks = block_manager.get_num_free_gpu_blocks()
302 + mapping = block_manager.swap_in(seq_group)
303 + cpu_blocks = block_manager.get_block_table(prompt)
304 + mapping_keys = [key for key, _ in mapping]
305 + assert mapping_keys == [cpu_blocks[0]]
306 + after_cpu_blocks = block_manager.get_num_free_cpu_blocks()
307 + after_gpu_blocks = block_manager.get_num_free_gpu_blocks()
308 + assert before_gpu_blocks == after_gpu_blocks + len(cpu_blocks)
309 +
310 +
311 +# TODO(cade/kaiyang): add comprehensive tests for swapping at allocator level.
312 +
313 +
314 @pytest.mark.parametrize("block_size", [8, 16])
315 @pytest.mark.parametrize("prompt_len", [10, 300, 1000])
316 @pytest.mark.parametrize("num_slots_to_append", [50])
vllm/config.py
+17 -11
@@ -651,19 +651,24 @@ class SchedulerConfig:
651 enable_chunked_prefill: If True, prefill requests can be chunked based
652 on the remaining max_num_batched_tokens.
653 embedding_mode: Whether the running model is for embedding.
654 + preemption_mode: Whether to perform preemption by swapping or
655 + recomputation. If not specified, we determine the mode as follows:
656 + We use recomputation by default since it incurs lower overhead than
657 + swapping. However, when the sequence group has multiple sequences
658 + (e.g., beam search), recomputation is not currently supported. In
659 + such a case, we use swapping instead.
660 """
661
656 - def __init__(
657 - self,
658 - max_num_batched_tokens: Optional[int],
659 - max_num_seqs: int,
660 - max_model_len: int,
661 - use_v2_block_manager: bool = False,
662 - num_lookahead_slots: int = 0,
663 - delay_factor: float = 0.0,
664 - enable_chunked_prefill: bool = False,
665 - embedding_mode: Optional[bool] = False,
666 - ) -> None:
662 + def __init__(self,
663 + max_num_batched_tokens: Optional[int],
664 + max_num_seqs: int,
665 + max_model_len: int,
666 + use_v2_block_manager: bool = False,
667 + num_lookahead_slots: int = 0,
668 + delay_factor: float = 0.0,
669 + enable_chunked_prefill: bool = False,
670 + embedding_mode: Optional[bool] = False,
671 + preemption_mode: Optional[str] = None) -> None:
672 if max_num_batched_tokens is not None:
673 self.max_num_batched_tokens = max_num_batched_tokens
674 else:
@@ -689,6 +694,7 @@ class SchedulerConfig:
694 self.delay_factor = delay_factor
695 self.chunked_prefill_enabled = enable_chunked_prefill
696 self.embedding_mode = embedding_mode
697 + self.preemption_mode = preemption_mode
698
699 self._verify_args()
700
vllm/core/block/block_table.py
+4
@@ -283,6 +283,10 @@ class BlockTable:
283 def _is_allocated(self) -> bool:
284 return len(self._blocks) > 0
285
286 + @property
287 + def blocks(self) -> Optional[List[Block]]:
288 + return self._blocks
289 +
290 @property
291 def _num_empty_slots(self) -> int:
292 assert self._is_allocated
vllm/core/block/common.py
-1
@@ -140,7 +140,6 @@ class CopyOnWriteTracker:
140 assert refcount != 0
141 if refcount > 1:
142 src_block_id = block_id
143 -
143 # Decrement refcount of the old block.
144 self._allocator.free(block)
145
vllm/core/block/cpu_gpu_block_allocator.py
+77 -5
@@ -90,11 +90,8 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
90 gpu_block_allocator=gpu_allocator,
91 )
92
93 - def __init__(
94 - self,
95 - cpu_block_allocator: BlockAllocator,
96 - gpu_block_allocator: BlockAllocator,
97 - ):
93 + def __init__(self, cpu_block_allocator: BlockAllocator,
94 + gpu_block_allocator: BlockAllocator):
95 assert not (
96 cpu_block_allocator.all_block_ids
97 & gpu_block_allocator.all_block_ids
@@ -105,6 +102,7 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
102 Device.GPU: gpu_block_allocator,
103 }
104
105 + self._swap_mapping: Dict[int, int] = {}
106 self._null_block: Optional[Block] = None
107
108 self._block_ids_to_allocator: Dict[int, BlockAllocator] = {}
@@ -198,6 +196,68 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
196 def get_num_total_blocks(self, device: Device) -> int:
197 return self._allocators[device].get_num_total_blocks()
198
199 + def get_physical_block_id(self, device: Device, absolute_id: int) -> int:
200 + """Returns the zero-offset block id on certain device given the
201 + absolute block id.
202 +
203 + Args:
204 + device (Device): The device for which to query relative block id.
205 + absolute_id (int): The absolute block id for the block in
206 + whole allocator.
207 +
208 + Returns:
209 + int: The zero-offset block id on certain device.
210 + """
211 + return self._allocators[device].get_physical_block_id(absolute_id)
212 +
213 + def swap(self, blocks: List[Block], source_device: Device,
214 + dest_device: Device) -> Dict[int, int]:
215 + """Execute the swap for the given blocks from source_device
216 + on to dest_device, save the current swap mapping and append
217 + them to the accumulated `self._swap_mapping` for each
218 + scheduling move.
219 +
220 + Args:
221 + blocks: List of blocks to be swapped.
222 + source_device (Device): Device to swap the 'blocks' from.
223 + dest_device (Device): Device to swap the 'blocks' to.
224 +
225 + Returns:
226 + Dict[int, int]: Swap mapping from source_device
227 + on to dest_device.
228 + """
229 + source_block_ids = [block.block_id for block in blocks]
230 + self._allocators[source_device].swap_out(blocks)
231 + self._allocators[dest_device].swap_in(blocks)
232 + dest_block_ids = [block.block_id for block in blocks]
233 +
234 + current_swap_mapping: Dict[int, int] = {}
235 + for src, dest in zip(source_block_ids, dest_block_ids):
236 + if src is not None and dest is not None:
237 + self._swap_mapping[src] = dest
238 + current_swap_mapping[src] = dest
239 + return current_swap_mapping
240 +
241 + def get_num_blocks_touched(self,
242 + blocks: List[Block],
243 + device: Device,
244 + num_lookahead_slots: int = 0) -> int:
245 + """Returns the number of blocks that will be touched by
246 + swapping in/out the given blocks on to the 'device'.
247 +
248 + Args:
249 + blocks: List of blocks to be swapped.
250 + device (Device): Device to swap the 'blocks' on.
251 + num_lookahead_slots (int): Number of lookahead slots used in
252 + speculative decoding, default to 0.
253 +
254 + Returns:
255 + int: the number of blocks that will be touched by
256 + swapping in/out the given blocks on to the 'device'.
257 + """
258 + return self._allocators[device].get_num_blocks_touched(
259 + blocks, num_lookahead_slots)
260 +
261 def clear_copy_on_writes(self) -> List[Tuple[int, int]]:
262 """Clears the copy-on-write (CoW) state and returns the mapping of
263 source to destination block IDs.
@@ -240,6 +300,18 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
300 def cow_block_if_not_appendable(self, block: Block) -> Optional[BlockId]:
301 raise NotImplementedError
302
303 + def get_and_reset_swaps(self) -> List[Tuple[int, int]]:
304 + """Returns and clears the mapping of source to destination block IDs.
305 + Will be called after every swapping operations for now, and after every
306 + schedule when BlockManagerV2 become default. Currently not useful.
307 +
308 + Returns:
309 + List[Tuple[int, int]]: A mapping of source to destination block IDs.
310 + """
311 + mapping = self._swap_mapping.copy()
312 + self._swap_mapping.clear()
313 + return list(mapping.items())
314 +
315
316 class NullBlock(Block):
317 """
vllm/core/block/interfaces.py
+35 -1
@@ -1,5 +1,5 @@
1 from abc import ABC, abstractmethod
2 -from typing import FrozenSet, List, Optional, Protocol, Tuple
2 +from typing import Dict, FrozenSet, List, Optional, Protocol, Tuple
3
4 from vllm.utils import Device
5
@@ -116,6 +116,18 @@ class BlockAllocator(ABC):
116 def get_num_free_blocks(self) -> int:
117 pass
118
119 + @abstractmethod
120 + def get_physical_block_id(self, absolute_id: int) -> int:
121 + pass
122 +
123 + @abstractmethod
124 + def swap_out(self, blocks: List[Block]) -> None:
125 + pass
126 +
127 + @abstractmethod
128 + def swap_in(self, blocks: List[Block]) -> None:
129 + pass
130 +
131 @property
132 @abstractmethod
133 def all_block_ids(self) -> FrozenSet[int]:
@@ -149,6 +161,12 @@ class BlockAllocator(ABC):
161 """NOTE: This should not be used besides Block"""
162 pass
163
164 + @abstractmethod
165 + def get_num_blocks_touched(self,
166 + blocks: List[Block],
167 + num_lookahead_slots: int = 0) -> int:
168 + pass
169 +
170 class NoFreeBlocksError(ValueError):
171 pass
172
@@ -204,6 +222,22 @@ class DeviceAwareBlockAllocator(ABC):
222 self, seq_block_ids: List[List[int]]) -> List[int]:
223 pass
224
225 + @abstractmethod
226 + def get_num_blocks_touched(self,
227 + blocks: List[Block],
228 + device: Device,
229 + num_lookahead_slots: int = 0) -> int:
230 + pass
231 +
232 + @abstractmethod
233 + def swap(self, blocks: List[Block], source_device: Device,
234 + dest_device: Device) -> Dict[int, int]:
235 + pass
236 +
237 + @abstractmethod
238 + def get_physical_block_id(self, device: Device, absolute_id: int) -> int:
239 + pass
240 +
241 @abstractmethod
242 def allocate_or_get_null_block(self) -> Block:
243 """
vllm/core/block/naive_block.py
+64 -2
@@ -3,6 +3,7 @@ from typing import FrozenSet, Iterable, List, Optional, Set, Tuple
3 from vllm.core.block.common import (CopyOnWriteTracker, RefCounter,
4 get_all_blocks_recursively)
5 from vllm.core.block.interfaces import Block, BlockAllocator, BlockId, Device
6 +from vllm.utils import cdiv
7
8 Refcount = int
9
@@ -95,8 +96,6 @@ class NaiveBlockAllocator(BlockAllocator):
96 def free(self, block: Block) -> None:
97 assert block.block_id is not None
98 self._free_block_id(block.block_id)
98 -
99 - # Mark the block as having no allocation.
99 block.block_id = None
100
101 def fork(self, last_block: Block) -> List[Block]:
@@ -153,6 +152,19 @@ class NaiveBlockAllocator(BlockAllocator):
152 if refcount == 0:
153 self._free_block_indices.add(block_id)
154
155 + def get_physical_block_id(self, absolute_id: int) -> int:
156 + """Returns the zero-offset block id on certain block allocator
157 + given the absolute block id.
158 +
159 + Args:
160 + absolute_id (int): The absolute block id for the block
161 + in whole allocator.
162 +
163 + Returns:
164 + int: The zero-offset block id on certain device.
165 + """
166 + return sorted(self._all_block_indices).index(absolute_id)
167 +
168 @property
169 def refcounter(self):
170 return self._refcounter
@@ -213,6 +225,56 @@ class NaiveBlockAllocator(BlockAllocator):
225 def promote_to_immutable_block(self, block: Block) -> BlockId:
226 raise NotImplementedError
227
228 + def get_num_blocks_touched(self,
229 + blocks: List[Block],
230 + num_lookahead_slots: int = 0) -> int:
231 + """Determine the number of blocks that will be touched by
232 + swapping in/out the given blocks from certain sequence
233 + group with the provided num_lookahead_slots.
234 +
235 + Args:
236 + blocks (List[Block]): The potential blocks to swap.
237 + num_lookahead_slots (int): number of lookahead slots (0 for swap
238 + out).
239 +
240 + Returns:
241 + int: the number of blocks that will be touched by
242 + swapping in/out the given blocks and num_lookahead_slots.
243 + """
244 + # NOTE: for naive block, we use set to eliminate common blocks among
245 + # seqs, also we compare the empty slots in the mutable blocks with
246 + # lookahead slots to get the number of unique new block that are
247 + # needed.
248 + old_block_set = set()
249 + new_block_count = 0
250 + # TODO(cade): make sure the logic is correct and clean it up.
251 + for block in blocks:
252 + if not block.is_full and num_lookahead_slots != 0:
253 + if block.num_empty_slots >= num_lookahead_slots:
254 + new_block_count += 1
255 + else:
256 + new_block_count += cdiv(
257 + num_lookahead_slots - block.num_empty_slots,
258 + self._block_size)
259 + else:
260 + old_block_set.add(block.block_id)
261 + num_touched_blocks = new_block_count + len(old_block_set)
262 + return num_touched_blocks
263 +
264 + def swap_out(self, blocks: List[Block]) -> None:
265 + for block in blocks:
266 + self.free(block)
267 +
268 + def swap_in(self, blocks: List[Block]) -> None:
269 + for block in blocks:
270 + if block.is_full:
271 + alloc = self.allocate_immutable(block.prev_block,
272 + block.token_ids)
273 + else:
274 + alloc = self.allocate_mutable(block.prev_block)
275 + alloc.append_token_ids(block.token_ids)
276 + block.block_id = alloc.block_id
277 +
278
279 class NaiveBlock(Block):
280 """An implementation of the Block class that does not support prefix
vllm/core/block/prefix_caching_block.py
+78
@@ -1,4 +1,5 @@
1 """Token blocks."""
2 +
3 from itertools import takewhile
4 from os.path import commonprefix
5 from typing import Dict, FrozenSet, Iterable, List, Optional, Tuple
@@ -8,6 +9,7 @@ from vllm.core.block.common import (CopyOnWriteTracker,
9 from vllm.core.block.interfaces import Block, BlockAllocator, BlockId, Device
10 from vllm.core.block.naive_block import NaiveBlock, NaiveBlockAllocator
11 from vllm.core.evictor_v2 import EvictionPolicy, Evictor, make_evictor
12 +from vllm.utils import cdiv
13
14 PrefixHash = int
15
@@ -294,10 +296,29 @@ class PrefixCachingBlockAllocator(BlockAllocator):
296 def get_num_total_blocks(self) -> int:
297 return self._hashless_allocator.get_num_total_blocks()
298
299 + def get_physical_block_id(self, absolute_id: int) -> int:
300 + """Returns the zero-offset block id on certain block allocator
301 + given the absolute block id.
302 +
303 + Args:
304 + absolute_id (int): The absolute block id for the block
305 + in whole allocator.
306 +
307 + Returns:
308 + int: The rzero-offset block id on certain device.
309 + """
310 + return sorted(self.all_block_ids).index(absolute_id)
311 +
312 @property
313 def all_block_ids(self) -> FrozenSet[int]:
314 return self._hashless_allocator.all_block_ids
315
316 + def is_block_cached(self, block: Block) -> bool:
317 + assert block.content_hash is not None
318 + if block.content_hash in self._cached_blocks:
319 + return True
320 + return False
321 +
322 def promote_to_immutable_block(self, block: Block) -> BlockId:
323 """Once a mutable block is full, it can be promoted to an immutable
324 block. This means that its content can be referenced by future blocks
@@ -411,6 +432,63 @@ class PrefixCachingBlockAllocator(BlockAllocator):
432 if ids != []
433 ])
434
435 + def get_num_blocks_touched(self,
436 + blocks: List[Block],
437 + num_lookahead_slots: int = 0) -> int:
438 + """Determine the number of blocks that will be touched by
439 + swapping in/out the given blocks from certain sequence
440 + group with the provided num_lookahead_slots.
441 +
442 + Args:
443 + blocks (List[Block]): The potential blocks to swap.
444 + num_lookahead_slots (int): number of lookahead slots (0 for
445 + swap out).
446 +
447 + Returns:
448 + int: the number of blocks that will be touched by
449 + swapping in/out the given blocks and num_lookahead_slots.
450 + """
451 + num_touched_blocks = 0
452 + for block in blocks:
453 + if not block.is_full:
454 + if block.num_empty_slots >= num_lookahead_slots:
455 + num_touched_blocks += 1
456 + else:
457 + num_touched_blocks += cdiv(
458 + num_lookahead_slots - block.num_empty_slots,
459 + self._block_size)
460 + else:
461 + if not self.is_block_cached(block):
462 + num_touched_blocks += 1
463 + return num_touched_blocks
464 +
465 + def swap_out(self, blocks: List[Block]) -> None:
466 + """Execute the swap out actions. Basically just free the
467 + given blocks.
468 +
469 + Args:
470 + blocks: List of blocks to be swapped out.
471 + """
472 + for block in blocks:
473 + self.free(block)
474 +
475 + def swap_in(self, blocks: List[Block]) -> None:
476 + """Execute the swap int actions. Change the block id from
477 + old allocator to current allocator for each block to finish
478 + the block table update.
479 +
480 + Args:
481 + blocks: List of blocks to be swapped in.
482 + """
483 + for block in blocks:
484 + if block.is_full:
485 + alloc = self.allocate_immutable(block.prev_block,
486 + block.token_ids)
487 + else:
488 + alloc = self.allocate_mutable(block.prev_block)
489 + alloc.append_token_ids(block.token_ids)
490 + block.block_id = alloc.block_id
491 +
492
493 class PrefixCachingBlock(Block):
494 """A block implementation that supports prefix caching.
vllm/core/block_manager_v1.py
+1 -5
@@ -541,11 +541,7 @@ class BlockSpaceManagerV1(BlockSpaceManager):
541
542 return new_block_table
543
544 - def swap_in(self,
545 - seq_group: SequenceGroup,
546 - num_lookahead_slots: int = 0) -> List[Tuple[int, int]]:
547 - assert (num_lookahead_slots == 0
548 - ), "BlockSpaceManagerV1 does not support lookahead allocation"
544 + def swap_in(self, seq_group: SequenceGroup) -> List[Tuple[int, int]]:
545
546 request_id = seq_group.request_id
547
vllm/core/block_manager_v2.py
+133 -7
@@ -1,10 +1,12 @@
1 """A block manager that manages token blocks."""
2 +from itertools import chain
3 from typing import Dict, List, Optional
4 from typing import Sequence as GenericSequence
5 from typing import Tuple
6
7 from vllm.core.block.block_table import BlockTable
8 from vllm.core.block.cpu_gpu_block_allocator import CpuGpuBlockAllocator
9 +from vllm.core.block.interfaces import Block
10 from vllm.core.block.utils import check_no_caching_or_swa_for_blockmgr_encdec
11 from vllm.core.interfaces import AllocStatus, BlockSpaceManager
12 from vllm.sequence import Sequence, SequenceGroup, SequenceStatus
@@ -217,7 +219,6 @@ class BlockSpaceManagerV2(BlockSpaceManager):
219 num_lookahead_slots=num_lookahead_slots,
220 num_computed_slots=seq.data.get_num_computed_tokens(),
221 )
220 -
222 # Return any new copy-on-writes.
223 new_cows = self.block_allocator.clear_copy_on_writes()
224 return new_cows
@@ -297,20 +298,145 @@ class BlockSpaceManagerV2(BlockSpaceManager):
298
299 def can_swap_in(self, seq_group: SequenceGroup,
300 num_lookahead_slots: int) -> AllocStatus:
300 - return AllocStatus.LATER
301 + """Returns the AllocStatus for the given sequence_group
302 + with num_lookahead_slots.
303 +
304 + Args:
305 + sequence_group (SequenceGroup): The sequence group to swap in.
306 + num_lookahead_slots (int): Number of lookahead slots used in
307 + speculative decoding, default to 0.
308 +
309 + Returns:
310 + AllocStatus: The AllocStatus for the given sequence group.
311 + """
312 + return self._can_swap(seq_group, Device.GPU, SequenceStatus.SWAPPED,
313 + num_lookahead_slots)
314 +
315 + def swap_in(self, seq_group: SequenceGroup) -> List[Tuple[int, int]]:
316 + """Returns the block id mapping (from CPU to GPU) generated by
317 + swapping in the given seq_group with num_lookahead_slots.
318
302 - def swap_in(self, seq_group: SequenceGroup,
303 - num_lookahead_slots: int) -> List[Tuple[int, int]]:
304 - raise NotImplementedError
319 + Args:
320 + seq_group (SequenceGroup): The sequence group to swap in.
321 +
322 + Returns:
323 + List[Tuple[int, int]]: The mapping of swapping block from CPU
324 + to GPU.
325 + """
326 + blocks = self._get_blocks_for_swap(seq_group, SequenceStatus.SWAPPED)
327 + current_swap_mapping = self.block_allocator.swap(
328 + blocks=blocks, source_device=Device.CPU, dest_device=Device.GPU)
329 +
330 + block_number_mapping = {
331 + self.block_allocator.get_physical_block_id(Device.CPU,
332 + cpu_block_id):
333 + self.block_allocator.get_physical_block_id(Device.GPU,
334 + gpu_block_id)
335 + for cpu_block_id, gpu_block_id in current_swap_mapping.items()
336 + }
337 + # convert to list of tuples once here
338 + return list(block_number_mapping.items())
339
340 def can_swap_out(self, seq_group: SequenceGroup) -> bool:
341 + """Returns whether we can swap out the given sequence_group
342 + with num_lookahead_slots.
343 +
344 + Args:
345 + seq_group (SequenceGroup): The sequence group to swap in.
346 + num_lookahead_slots (int): Number of lookahead slots used in
347 + speculative decoding, default to 0.
348 +
349 + Returns:
350 + bool: Whether it's possible to swap out current sequence group.
351 + """
352 + alloc_status = self._can_swap(seq_group, Device.CPU,
353 + SequenceStatus.RUNNING)
354 + if alloc_status == AllocStatus.OK:
355 + return True
356 return False
357
309 - def swap_out(self, seq_group: SequenceGroup) -> List[Tuple[int, int]]:
310 - raise NotImplementedError
358 + def swap_out(self, sequence_group: SequenceGroup) -> List[Tuple[int, int]]:
359 + """Returns the block id mapping (from GPU to CPU) generated by
360 + swapping out the given sequence_group with num_lookahead_slots.
361 +
362 + Args:
363 + sequence_group (SequenceGroup): The sequence group to swap in.
364 +
365 + Returns:
366 + List[Tuple[int, int]]: The mapping of swapping block from
367 + GPU to CPU.
368 + """
369 + blocks = self._get_blocks_for_swap(sequence_group,
370 + SequenceStatus.RUNNING)
371 + current_swap_mapping = self.block_allocator.swap(
372 + blocks=blocks, source_device=Device.GPU, dest_device=Device.CPU)
373 + block_number_mapping = {
374 + self.block_allocator.get_physical_block_id(Device.GPU,
375 + gpu_block_id):
376 + self.block_allocator.get_physical_block_id(Device.CPU,
377 + cpu_block_id)
378 + for gpu_block_id, cpu_block_id in current_swap_mapping.items()
379 + }
380 + # convert to list of tuples once here
381 + return list(block_number_mapping.items())
382
383 def get_num_free_gpu_blocks(self) -> int:
384 return self.block_allocator.get_num_free_blocks(Device.GPU)
385
386 def get_num_free_cpu_blocks(self) -> int:
387 return self.block_allocator.get_num_free_blocks(Device.CPU)
388 +
389 + def _can_swap(self,
390 + seq_group: SequenceGroup,
391 + device: Device,
392 + status: SequenceStatus,
393 + num_lookahead_slots: int = 0) -> AllocStatus:
394 + """Returns the AllocStatus for swapping in/out the given sequence_group
395 + on to the 'device'.
396 +
397 + Args:
398 + sequence_group (SequenceGroup): The sequence group to swap in.
399 + device (Device): device to swap the 'seq_group' on.
400 + status (SequenceStatus): The status of sequence which is needed
401 + for action. RUNNING for swap out and SWAPPED for swap in
402 + num_lookahead_slots (int): Number of lookahead slots used in
403 + speculative decoding, default to 0.
404 +
405 + Returns:
406 + AllocStatus: The AllocStatus for swapping in/out the given
407 + sequence_group on to the 'device'.
408 + """
409 + blocks = self._get_blocks_for_swap(seq_group, status)
410 + num_blocks_touched = self.block_allocator.get_num_blocks_touched(
411 + blocks, device, num_lookahead_slots)
412 + watermark_blocks = 0
413 + if device == Device.GPU:
414 + watermark_blocks = self.watermark_blocks
415 + if self.block_allocator.get_num_total_blocks(
416 + device) < num_blocks_touched:
417 + return AllocStatus.NEVER
418 + elif self.block_allocator.get_num_free_blocks(
419 + device) - num_blocks_touched >= watermark_blocks:
420 + return AllocStatus.OK
421 + else:
422 + return AllocStatus.LATER
423 +
424 + def _get_blocks_for_swap(self, seq_group: SequenceGroup,
425 + status: SequenceStatus) -> List[Block]:
426 + """Returns the list of blocks those are touched by the seq_group
427 +
428 + Args:
429 + sequence_group (SequenceGroup): The sequence group to swap in.
430 + status (SequenceStatus): The status of sequence which is needed
431 + for action. RUNNING for swap out and SWAPPED for swap in
432 +
433 + Returns:
434 + The list of blocks those are touched by the seq_group.
435 + """
436 + blocks: Dict[int, List[Block]] = {}
437 + for seq in seq_group.get_seqs(status=status):
438 + block_table = self.block_tables[seq.seq_id]
439 + if block_table.blocks is not None:
440 + blocks[seq.seq_id] = block_table.blocks
441 + combined_blocks = list(chain(*blocks.values()))
442 + return combined_blocks
vllm/core/embedding_model_block_manager.py
+1 -2
@@ -46,8 +46,7 @@ class EmbeddingModelBlockSpaceManager(BlockSpaceManager):
46 num_lookahead_slots: int) -> AllocStatus:
47 return AllocStatus.OK
48
49 - def swap_in(self, seq_group: SequenceGroup,
50 - num_lookahead_slots: int) -> List[Tuple[int, int]]:
49 + def swap_in(self, seq_group: SequenceGroup) -> List[Tuple[int, int]]:
50 return None # type: ignore
51
52 def can_swap_out(self, seq_group: SequenceGroup) -> bool:
vllm/core/interfaces.py
+1 -2
@@ -73,8 +73,7 @@ class BlockSpaceManager(ABC):
73 pass
74
75 @abstractmethod
76 - def swap_in(self, seq_group: SequenceGroup,
77 - num_lookahead_slots: int) -> List[Tuple[int, int]]:
76 + def swap_in(self, seq_group: SequenceGroup) -> List[Tuple[int, int]]:
77 pass
78
79 @abstractmethod
vllm/core/scheduler.py
+11 -2
@@ -297,6 +297,8 @@ class Scheduler:
297 self.prev_prompt = False
298 # Latency of the last prompt step
299 self.last_prompt_latency = 0.0
300 + # preemption mode, RECOMPUTE or SWAP
301 + self.user_specified_preemption_mode = scheduler_config.preemption_mode
302
303 # The following field is test-only. It is used to inject artificial
304 # preemption.
@@ -522,7 +524,9 @@ class Scheduler:
524 seq_group = swapped_queue[0]
525
526 # If the sequence group cannot be swapped in, stop.
525 - alloc_status = self.block_manager.can_swap_in(seq_group)
527 + is_prefill = seq_group.is_prefill()
528 + alloc_status = self.block_manager.can_swap_in(
529 + seq_group, self._get_num_lookahead_slots(is_prefill))
530 if alloc_status == AllocStatus.LATER:
531 break
532 elif alloc_status == AllocStatus.NEVER:
@@ -1067,12 +1071,17 @@ class Scheduler:
1071 # over sequence groups with a single sequence.
1072 # TODO(woosuk): Support recomputation for sequence groups with multiple
1073 # sequences. This may require a more sophisticated CUDA kernel.
1070 - if preemption_mode is None:
1074 + if self.user_specified_preemption_mode is None:
1075 if seq_group.get_max_num_running_seqs() == 1:
1076 preemption_mode = PreemptionMode.RECOMPUTE
1077 else:
1078 preemption_mode = PreemptionMode.SWAP
1079
1080 + elif self.user_specified_preemption_mode == "swap":
1081 + preemption_mode = PreemptionMode.SWAP
1082 + else:
1083 + preemption_mode = PreemptionMode.RECOMPUTE
1084 +
1085 if self.num_cumulative_preemption % 50 == 0:
1086 logger.warning(
1087 "Sequence group %s is preempted by %s mode because there is "
vllm/engine/arg_utils.py
+9
@@ -75,6 +75,7 @@ class EngineArgs:
75 num_gpu_blocks_override: Optional[int] = None
76 num_lookahead_slots: int = 0
77 model_loader_extra_config: Optional[dict] = None
78 + preemption_mode: Optional[str] = None
79
80 # Related to Vision-language models such as llava
81 image_input_type: Optional[str] = None
@@ -564,6 +565,13 @@ class EngineArgs:
565 'corresponding to the chosen load_format. '
566 'This should be a JSON string that will be '
567 'parsed into a dictionary.')
568 + parser.add_argument(
569 + '--preemption_mode',
570 + type=str,
571 + default=None,
572 + help='If \'recompute\', the engine performs preemption by block '
573 + 'swapping; If \'swap\', the engine performs preemption by block '
574 + 'swapping.')
575
576 parser.add_argument(
577 "--served-model-name",
@@ -667,6 +675,7 @@ class EngineArgs:
675 delay_factor=self.scheduler_delay_factor,
676 enable_chunked_prefill=self.enable_chunked_prefill,
677 embedding_mode=model_config.embedding_mode,
678 + preemption_mode=self.preemption_mode,
679 )
680 lora_config = LoRAConfig(
681 max_lora_rank=self.max_lora_rank,