A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Misc]: Implement CPU/GPU swapping in BlockManagerV2 (#3834)
Kaiyang Chen committed
Jun 4, 2024 at 04:37 UTC
10c38e3e46d95a02576c92c25d5b3e88e3c6e282
16 files changed
+529
-49
format.sh
+1
-1
@@ -118,7 +118,7 @@ mypy vllm/model_executor --config-file pyproject.toml
118
# https://github.com/codespell-project/codespell/issues/1915
119
# Avoiding the "./" prefix and using "/**" globs for directories appears to solve the problem
120
CODESPELL_EXCLUDES=(
121
- '--skip' 'tests/prompts/**,./benchmarks/sonnet.txt,tests/lora/data/**,build/**'
121
+ '--skip' 'tests/prompts/**,./benchmarks/sonnet.txt,*tests/lora/data/**,build/**'
122
)
123
124
# check spelling of specified files
tests/core/block/e2e/test_correctness.py
+40
-9
@@ -24,7 +24,13 @@ from .conftest import get_token_ids_from_llm_generator
24
@pytest.mark.parametrize("baseline_llm_kwargs", [{
25
"use_v2_block_manager": False
26
}])
27
-@pytest.mark.parametrize("test_llm_kwargs", [{"use_v2_block_manager": True}])
27
+@pytest.mark.parametrize("test_llm_kwargs", [{
28
+ "use_v2_block_manager": True,
29
+ "preemption_mode": "swap"
30
+}, {
31
+ "use_v2_block_manager": True,
32
+ "preemption_mode": "recompute"
33
+}])
34
@pytest.mark.parametrize("batch_size", [10])
35
@pytest.mark.parametrize("seed", [1])
36
def test_v1_v2_greedy_equality_with_preemption(baseline_llm_generator,
@@ -95,7 +101,13 @@ def test_v1_v2_greedy_equality_with_preemption(baseline_llm_generator,
101
@pytest.mark.parametrize("baseline_llm_kwargs", [{
102
"use_v2_block_manager": False
103
}])
98
-@pytest.mark.parametrize("test_llm_kwargs", [{"use_v2_block_manager": True}])
104
+@pytest.mark.parametrize("test_llm_kwargs", [{
105
+ "use_v2_block_manager": True,
106
+ "preemption_mode": "swap"
107
+}, {
108
+ "use_v2_block_manager": True,
109
+ "preemption_mode": "recompute"
110
+}])
111
@pytest.mark.parametrize("batch_size", [10])
112
@pytest.mark.parametrize("seed", [1])
113
def test_v1_v2_greedy_equality_with_cow(baseline_llm_generator,
@@ -179,11 +191,18 @@ def test_v1_v2_greedy_equality_with_cow(baseline_llm_generator,
191
}])
192
@pytest.mark.parametrize(
193
"test_llm_kwargs",
182
- [{
183
- # We run one test with block_size < lookahead_slots, one test with
184
- # block_size > lookahead_slots
185
- "num_lookahead_slots": 10,
186
- }])
194
+ [
195
+ {
196
+ # We run one test with block_size < lookahead_slots, one test with
197
+ # block_size > lookahead_slots
198
+ "num_lookahead_slots": 10,
199
+ "preemption_mode": "swap",
200
+ },
201
+ {
202
+ "num_lookahead_slots": 10,
203
+ "preemption_mode": "recompute",
204
+ }
205
+ ])
206
@pytest.mark.parametrize("batch_size", [4])
207
@pytest.mark.parametrize("seed", [1])
208
def test_lookahead_greedy_equality_with_preemption(baseline_llm_generator,
@@ -322,7 +341,13 @@ def test_chunked_prefill_block_manager_v2(baseline_llm_generator,
341
@pytest.mark.parametrize("baseline_llm_kwargs", [{
342
"use_v2_block_manager": False
343
}])
325
-@pytest.mark.parametrize("test_llm_kwargs", [{"use_v2_block_manager": True}])
344
+@pytest.mark.parametrize("test_llm_kwargs", [{
345
+ "use_v2_block_manager": True,
346
+ "preemption_mode": "swap"
347
+}, {
348
+ "use_v2_block_manager": True,
349
+ "preemption_mode": "recompute"
350
+}])
351
@pytest.mark.parametrize("batch_size", [10])
352
@pytest.mark.parametrize("seed", [1])
353
def test_v1_v2_greedy_equality_prefix_caching_enabled_with_preemption(
@@ -397,7 +422,13 @@ def test_v1_v2_greedy_equality_prefix_caching_enabled_with_preemption(
422
@pytest.mark.parametrize("baseline_llm_kwargs", [{
423
"enable_prefix_caching": False
424
}])
400
-@pytest.mark.parametrize("test_llm_kwargs", [{"enable_prefix_caching": True}])
425
+@pytest.mark.parametrize("test_llm_kwargs", [{
426
+ "enable_prefix_caching": True,
427
+ "preemption_mode": "swap"
428
+}, {
429
+ "enable_prefix_caching": True,
430
+ "preemption_mode": "recompute"
431
+}])
432
@pytest.mark.parametrize("batch_size", [10])
433
@pytest.mark.parametrize("seed", [1])
434
def test_auto_prefix_caching_with_preemption(baseline_llm_generator,
tests/core/block/test_block_manager_v2.py
+57
-1
@@ -7,7 +7,8 @@ from vllm.core.interfaces import AllocStatus
7
from vllm.sequence import Logprob, SequenceStatus
8
from vllm.utils import chunk_list
9
10
-from ..utils import create_seq_group, create_seq_group_encoder_decoder
10
+from ..utils import (create_dummy_prompt, create_seq_group,
11
+ create_seq_group_encoder_decoder)
12
13
14
@pytest.mark.parametrize("block_size", [16])
@@ -255,6 +256,61 @@ def test_append_slots(block_size, prompt_len, num_slots_to_append,
256
assert num_consumed_blocks == expected_consumed_blocks
257
258
259
+@pytest.mark.parametrize("block_size", [8])
260
+@pytest.mark.parametrize("num_cpu_blocks", [4])
261
+@pytest.mark.parametrize("num_gpu_blocks", [4])
262
+@pytest.mark.parametrize("num_lookahead_slots", [0, 2, 10])
263
+@pytest.mark.parametrize("enable_caching", [False, True])
264
+def test_swap(block_size, num_cpu_blocks, num_gpu_blocks, num_lookahead_slots,
265
+ enable_caching):
266
+ """Verify blocks number on src/desc device is correct after swapping in/out
267
+ sequence group (not missing or extra blocks).
268
+ """
269
+ block_manager = BlockSpaceManagerV2(block_size,
270
+ num_cpu_blocks,
271
+ num_gpu_blocks,
272
+ watermark=0,
273
+ enable_caching=enable_caching)
274
+ prompt, seq_group = create_dummy_prompt("1", prompt_length=block_size - 1)
275
+ prompt.status = SequenceStatus.WAITING
276
+ block_manager.allocate(seq_group)
277
+ # Emulate a forward pass by appending a single token.
278
+ # The block manager then knows how many unprocessed
279
+ # tokens will be written in the next forward pass.
280
+ token_id = 0
281
+ prompt.status = SequenceStatus.RUNNING
282
+ prompt.append_token_id(token_id, {token_id: Logprob(0.0)})
283
+
284
+ # Swap seq group from GPU -> CPU.
285
+ gpu_blocks = block_manager.get_block_table(prompt)
286
+ assert block_manager.can_swap_out(seq_group)
287
+ before_cpu_blocks = block_manager.get_num_free_cpu_blocks()
288
+ before_gpu_blocks = block_manager.get_num_free_gpu_blocks()
289
+ mapping = block_manager.swap_out(seq_group)
290
+ mapping_keys = [key for key, _ in mapping]
291
+ assert mapping_keys == gpu_blocks
292
+ after_cpu_blocks = block_manager.get_num_free_cpu_blocks()
293
+ after_gpu_blocks = block_manager.get_num_free_gpu_blocks()
294
+ assert before_cpu_blocks == after_cpu_blocks + len(gpu_blocks)
295
+ assert before_gpu_blocks + len(gpu_blocks) == after_gpu_blocks
296
+ prompt.status = SequenceStatus.SWAPPED
297
+
298
+ # Swap seq group from CPU -> GPU.
299
+ assert block_manager.can_swap_in(seq_group, num_lookahead_slots)
300
+ before_cpu_blocks = block_manager.get_num_free_cpu_blocks()
301
+ before_gpu_blocks = block_manager.get_num_free_gpu_blocks()
302
+ mapping = block_manager.swap_in(seq_group)
303
+ cpu_blocks = block_manager.get_block_table(prompt)
304
+ mapping_keys = [key for key, _ in mapping]
305
+ assert mapping_keys == [cpu_blocks[0]]
306
+ after_cpu_blocks = block_manager.get_num_free_cpu_blocks()
307
+ after_gpu_blocks = block_manager.get_num_free_gpu_blocks()
308
+ assert before_gpu_blocks == after_gpu_blocks + len(cpu_blocks)
309
+
310
+
311
+# TODO(cade/kaiyang): add comprehensive tests for swapping at allocator level.
312
+
313
+
314
@pytest.mark.parametrize("block_size", [8, 16])
315
@pytest.mark.parametrize("prompt_len", [10, 300, 1000])
316
@pytest.mark.parametrize("num_slots_to_append", [50])
vllm/config.py
+17
-11
@@ -651,19 +651,24 @@ class SchedulerConfig:
651
enable_chunked_prefill: If True, prefill requests can be chunked based
652
on the remaining max_num_batched_tokens.
653
embedding_mode: Whether the running model is for embedding.
654
+ preemption_mode: Whether to perform preemption by swapping or
655
+ recomputation. If not specified, we determine the mode as follows:
656
+ We use recomputation by default since it incurs lower overhead than
657
+ swapping. However, when the sequence group has multiple sequences
658
+ (e.g., beam search), recomputation is not currently supported. In
659
+ such a case, we use swapping instead.
660
"""
661
656
- def __init__(
657
- self,
658
- max_num_batched_tokens: Optional[int],
659
- max_num_seqs: int,
660
- max_model_len: int,
661
- use_v2_block_manager: bool = False,
662
- num_lookahead_slots: int = 0,
663
- delay_factor: float = 0.0,
664
- enable_chunked_prefill: bool = False,
665
- embedding_mode: Optional[bool] = False,
666
- ) -> None:
662
+ def __init__(self,
663
+ max_num_batched_tokens: Optional[int],
664
+ max_num_seqs: int,
665
+ max_model_len: int,
666
+ use_v2_block_manager: bool = False,
667
+ num_lookahead_slots: int = 0,
668
+ delay_factor: float = 0.0,
669
+ enable_chunked_prefill: bool = False,
670
+ embedding_mode: Optional[bool] = False,
671
+ preemption_mode: Optional[str] = None) -> None:
672
if max_num_batched_tokens is not None:
673
self.max_num_batched_tokens = max_num_batched_tokens
674
else:
@@ -689,6 +694,7 @@ class SchedulerConfig:
694
self.delay_factor = delay_factor
695
self.chunked_prefill_enabled = enable_chunked_prefill
696
self.embedding_mode = embedding_mode
697
+ self.preemption_mode = preemption_mode
698
699
self._verify_args()
700
vllm/core/block/block_table.py
+4
@@ -283,6 +283,10 @@ class BlockTable:
283
def _is_allocated(self) -> bool:
284
return len(self._blocks) > 0
285
286
+ @property
287
+ def blocks(self) -> Optional[List[Block]]:
288
+ return self._blocks
289
+
290
@property
291
def _num_empty_slots(self) -> int:
292
assert self._is_allocated
vllm/core/block/common.py
-1
@@ -140,7 +140,6 @@ class CopyOnWriteTracker:
140
assert refcount != 0
141
if refcount > 1:
142
src_block_id = block_id
143
-
143
# Decrement refcount of the old block.
144
self._allocator.free(block)
145
vllm/core/block/cpu_gpu_block_allocator.py
+77
-5
@@ -90,11 +90,8 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
90
gpu_block_allocator=gpu_allocator,
91
)
92
93
- def __init__(
94
- self,
95
- cpu_block_allocator: BlockAllocator,
96
- gpu_block_allocator: BlockAllocator,
97
- ):
93
+ def __init__(self, cpu_block_allocator: BlockAllocator,
94
+ gpu_block_allocator: BlockAllocator):
95
assert not (
96
cpu_block_allocator.all_block_ids
97
& gpu_block_allocator.all_block_ids
@@ -105,6 +102,7 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
102
Device.GPU: gpu_block_allocator,
103
}
104
105
+ self._swap_mapping: Dict[int, int] = {}
106
self._null_block: Optional[Block] = None
107
108
self._block_ids_to_allocator: Dict[int, BlockAllocator] = {}
@@ -198,6 +196,68 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
196
def get_num_total_blocks(self, device: Device) -> int:
197
return self._allocators[device].get_num_total_blocks()
198
199
+ def get_physical_block_id(self, device: Device, absolute_id: int) -> int:
200
+ """Returns the zero-offset block id on certain device given the
201
+ absolute block id.
202
+
203
+ Args:
204
+ device (Device): The device for which to query relative block id.
205
+ absolute_id (int): The absolute block id for the block in
206
+ whole allocator.
207
+
208
+ Returns:
209
+ int: The zero-offset block id on certain device.
210
+ """
211
+ return self._allocators[device].get_physical_block_id(absolute_id)
212
+
213
+ def swap(self, blocks: List[Block], source_device: Device,
214
+ dest_device: Device) -> Dict[int, int]:
215
+ """Execute the swap for the given blocks from source_device
216
+ on to dest_device, save the current swap mapping and append
217
+ them to the accumulated `self._swap_mapping` for each
218
+ scheduling move.
219
+
220
+ Args:
221
+ blocks: List of blocks to be swapped.
222
+ source_device (Device): Device to swap the 'blocks' from.
223
+ dest_device (Device): Device to swap the 'blocks' to.
224
+
225
+ Returns:
226
+ Dict[int, int]: Swap mapping from source_device
227
+ on to dest_device.
228
+ """
229
+ source_block_ids = [block.block_id for block in blocks]
230
+ self._allocators[source_device].swap_out(blocks)
231
+ self._allocators[dest_device].swap_in(blocks)
232
+ dest_block_ids = [block.block_id for block in blocks]
233
+
234
+ current_swap_mapping: Dict[int, int] = {}
235
+ for src, dest in zip(source_block_ids, dest_block_ids):
236
+ if src is not None and dest is not None:
237
+ self._swap_mapping[src] = dest
238
+ current_swap_mapping[src] = dest
239
+ return current_swap_mapping
240
+
241
+ def get_num_blocks_touched(self,
242
+ blocks: List[Block],
243
+ device: Device,
244
+ num_lookahead_slots: int = 0) -> int:
245
+ """Returns the number of blocks that will be touched by
246
+ swapping in/out the given blocks on to the 'device'.
247
+
248
+ Args:
249
+ blocks: List of blocks to be swapped.
250
+ device (Device): Device to swap the 'blocks' on.
251
+ num_lookahead_slots (int): Number of lookahead slots used in
252
+ speculative decoding, default to 0.
253
+
254
+ Returns:
255
+ int: the number of blocks that will be touched by
256
+ swapping in/out the given blocks on to the 'device'.
257
+ """
258
+ return self._allocators[device].get_num_blocks_touched(
259
+ blocks, num_lookahead_slots)
260
+
261
def clear_copy_on_writes(self) -> List[Tuple[int, int]]:
262
"""Clears the copy-on-write (CoW) state and returns the mapping of
263
source to destination block IDs.
@@ -240,6 +300,18 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
300
def cow_block_if_not_appendable(self, block: Block) -> Optional[BlockId]:
301
raise NotImplementedError
302
303
+ def get_and_reset_swaps(self) -> List[Tuple[int, int]]:
304
+ """Returns and clears the mapping of source to destination block IDs.
305
+ Will be called after every swapping operations for now, and after every
306
+ schedule when BlockManagerV2 become default. Currently not useful.
307
+
308
+ Returns:
309
+ List[Tuple[int, int]]: A mapping of source to destination block IDs.
310
+ """
311
+ mapping = self._swap_mapping.copy()
312
+ self._swap_mapping.clear()
313
+ return list(mapping.items())
314
+
315
316
class NullBlock(Block):
317
"""
vllm/core/block/interfaces.py
+35
-1
@@ -1,5 +1,5 @@
1
from abc import ABC, abstractmethod
2
-from typing import FrozenSet, List, Optional, Protocol, Tuple
2
+from typing import Dict, FrozenSet, List, Optional, Protocol, Tuple
3
4
from vllm.utils import Device
5
@@ -116,6 +116,18 @@ class BlockAllocator(ABC):
116
def get_num_free_blocks(self) -> int:
117
pass
118
119
+ @abstractmethod
120
+ def get_physical_block_id(self, absolute_id: int) -> int:
121
+ pass
122
+
123
+ @abstractmethod
124
+ def swap_out(self, blocks: List[Block]) -> None:
125
+ pass
126
+
127
+ @abstractmethod
128
+ def swap_in(self, blocks: List[Block]) -> None:
129
+ pass
130
+
131
@property
132
@abstractmethod
133
def all_block_ids(self) -> FrozenSet[int]:
@@ -149,6 +161,12 @@ class BlockAllocator(ABC):
161
"""NOTE: This should not be used besides Block"""
162
pass
163
164
+ @abstractmethod
165
+ def get_num_blocks_touched(self,
166
+ blocks: List[Block],
167
+ num_lookahead_slots: int = 0) -> int:
168
+ pass
169
+
170
class NoFreeBlocksError(ValueError):
171
pass
172
@@ -204,6 +222,22 @@ class DeviceAwareBlockAllocator(ABC):
222
self, seq_block_ids: List[List[int]]) -> List[int]:
223
pass
224
225
+ @abstractmethod
226
+ def get_num_blocks_touched(self,
227
+ blocks: List[Block],
228
+ device: Device,
229
+ num_lookahead_slots: int = 0) -> int:
230
+ pass
231
+
232
+ @abstractmethod
233
+ def swap(self, blocks: List[Block], source_device: Device,
234
+ dest_device: Device) -> Dict[int, int]:
235
+ pass
236
+
237
+ @abstractmethod
238
+ def get_physical_block_id(self, device: Device, absolute_id: int) -> int:
239
+ pass
240
+
241
@abstractmethod
242
def allocate_or_get_null_block(self) -> Block:
243
"""
vllm/core/block/naive_block.py
+64
-2
@@ -3,6 +3,7 @@ from typing import FrozenSet, Iterable, List, Optional, Set, Tuple
3
from vllm.core.block.common import (CopyOnWriteTracker, RefCounter,
4
get_all_blocks_recursively)
5
from vllm.core.block.interfaces import Block, BlockAllocator, BlockId, Device
6
+from vllm.utils import cdiv
7
8
Refcount = int
9
@@ -95,8 +96,6 @@ class NaiveBlockAllocator(BlockAllocator):
96
def free(self, block: Block) -> None:
97
assert block.block_id is not None
98
self._free_block_id(block.block_id)
98
-
99
- # Mark the block as having no allocation.
99
block.block_id = None
100
101
def fork(self, last_block: Block) -> List[Block]:
@@ -153,6 +152,19 @@ class NaiveBlockAllocator(BlockAllocator):
152
if refcount == 0:
153
self._free_block_indices.add(block_id)
154
155
+ def get_physical_block_id(self, absolute_id: int) -> int:
156
+ """Returns the zero-offset block id on certain block allocator
157
+ given the absolute block id.
158
+
159
+ Args:
160
+ absolute_id (int): The absolute block id for the block
161
+ in whole allocator.
162
+
163
+ Returns:
164
+ int: The zero-offset block id on certain device.
165
+ """
166
+ return sorted(self._all_block_indices).index(absolute_id)
167
+
168
@property
169
def refcounter(self):
170
return self._refcounter
@@ -213,6 +225,56 @@ class NaiveBlockAllocator(BlockAllocator):
225
def promote_to_immutable_block(self, block: Block) -> BlockId:
226
raise NotImplementedError
227
228
+ def get_num_blocks_touched(self,
229
+ blocks: List[Block],
230
+ num_lookahead_slots: int = 0) -> int:
231
+ """Determine the number of blocks that will be touched by
232
+ swapping in/out the given blocks from certain sequence
233
+ group with the provided num_lookahead_slots.
234
+
235
+ Args:
236
+ blocks (List[Block]): The potential blocks to swap.
237
+ num_lookahead_slots (int): number of lookahead slots (0 for swap
238
+ out).
239
+
240
+ Returns:
241
+ int: the number of blocks that will be touched by
242
+ swapping in/out the given blocks and num_lookahead_slots.
243
+ """
244
+ # NOTE: for naive block, we use set to eliminate common blocks among
245
+ # seqs, also we compare the empty slots in the mutable blocks with
246
+ # lookahead slots to get the number of unique new block that are
247
+ # needed.
248
+ old_block_set = set()
249
+ new_block_count = 0
250
+ # TODO(cade): make sure the logic is correct and clean it up.
251
+ for block in blocks:
252
+ if not block.is_full and num_lookahead_slots != 0:
253
+ if block.num_empty_slots >= num_lookahead_slots:
254
+ new_block_count += 1
255
+ else:
256
+ new_block_count += cdiv(
257
+ num_lookahead_slots - block.num_empty_slots,
258
+ self._block_size)
259
+ else:
260
+ old_block_set.add(block.block_id)
261
+ num_touched_blocks = new_block_count + len(old_block_set)
262
+ return num_touched_blocks
263
+
264
+ def swap_out(self, blocks: List[Block]) -> None:
265
+ for block in blocks:
266
+ self.free(block)
267
+
268
+ def swap_in(self, blocks: List[Block]) -> None:
269
+ for block in blocks:
270
+ if block.is_full:
271
+ alloc = self.allocate_immutable(block.prev_block,
272
+ block.token_ids)
273
+ else:
274
+ alloc = self.allocate_mutable(block.prev_block)
275
+ alloc.append_token_ids(block.token_ids)
276
+ block.block_id = alloc.block_id
277
+
278
279
class NaiveBlock(Block):
280
"""An implementation of the Block class that does not support prefix
vllm/core/block/prefix_caching_block.py
+78
@@ -1,4 +1,5 @@
1
"""Token blocks."""
2
+
3
from itertools import takewhile
4
from os.path import commonprefix
5
from typing import Dict, FrozenSet, Iterable, List, Optional, Tuple
@@ -8,6 +9,7 @@ from vllm.core.block.common import (CopyOnWriteTracker,
9
from vllm.core.block.interfaces import Block, BlockAllocator, BlockId, Device
10
from vllm.core.block.naive_block import NaiveBlock, NaiveBlockAllocator
11
from vllm.core.evictor_v2 import EvictionPolicy, Evictor, make_evictor
12
+from vllm.utils import cdiv
13
14
PrefixHash = int
15
@@ -294,10 +296,29 @@ class PrefixCachingBlockAllocator(BlockAllocator):
296
def get_num_total_blocks(self) -> int:
297
return self._hashless_allocator.get_num_total_blocks()
298
299
+ def get_physical_block_id(self, absolute_id: int) -> int:
300
+ """Returns the zero-offset block id on certain block allocator
301
+ given the absolute block id.
302
+
303
+ Args:
304
+ absolute_id (int): The absolute block id for the block
305
+ in whole allocator.
306
+
307
+ Returns:
308
+ int: The rzero-offset block id on certain device.
309
+ """
310
+ return sorted(self.all_block_ids).index(absolute_id)
311
+
312
@property
313
def all_block_ids(self) -> FrozenSet[int]:
314
return self._hashless_allocator.all_block_ids
315
316
+ def is_block_cached(self, block: Block) -> bool:
317
+ assert block.content_hash is not None
318
+ if block.content_hash in self._cached_blocks:
319
+ return True
320
+ return False
321
+
322
def promote_to_immutable_block(self, block: Block) -> BlockId:
323
"""Once a mutable block is full, it can be promoted to an immutable
324
block. This means that its content can be referenced by future blocks
@@ -411,6 +432,63 @@ class PrefixCachingBlockAllocator(BlockAllocator):
432
if ids != []
433
])
434
435
+ def get_num_blocks_touched(self,
436
+ blocks: List[Block],
437
+ num_lookahead_slots: int = 0) -> int:
438
+ """Determine the number of blocks that will be touched by
439
+ swapping in/out the given blocks from certain sequence
440
+ group with the provided num_lookahead_slots.
441
+
442
+ Args:
443
+ blocks (List[Block]): The potential blocks to swap.
444
+ num_lookahead_slots (int): number of lookahead slots (0 for
445
+ swap out).
446
+
447
+ Returns:
448
+ int: the number of blocks that will be touched by
449
+ swapping in/out the given blocks and num_lookahead_slots.
450
+ """
451
+ num_touched_blocks = 0
452
+ for block in blocks:
453
+ if not block.is_full:
454
+ if block.num_empty_slots >= num_lookahead_slots:
455
+ num_touched_blocks += 1
456
+ else:
457
+ num_touched_blocks += cdiv(
458
+ num_lookahead_slots - block.num_empty_slots,
459
+ self._block_size)
460
+ else:
461
+ if not self.is_block_cached(block):
462
+ num_touched_blocks += 1
463
+ return num_touched_blocks
464
+
465
+ def swap_out(self, blocks: List[Block]) -> None:
466
+ """Execute the swap out actions. Basically just free the
467
+ given blocks.
468
+
469
+ Args:
470
+ blocks: List of blocks to be swapped out.
471
+ """
472
+ for block in blocks:
473
+ self.free(block)
474
+
475
+ def swap_in(self, blocks: List[Block]) -> None:
476
+ """Execute the swap int actions. Change the block id from
477
+ old allocator to current allocator for each block to finish
478
+ the block table update.
479
+
480
+ Args:
481
+ blocks: List of blocks to be swapped in.
482
+ """
483
+ for block in blocks:
484
+ if block.is_full:
485
+ alloc = self.allocate_immutable(block.prev_block,
486
+ block.token_ids)
487
+ else:
488
+ alloc = self.allocate_mutable(block.prev_block)
489
+ alloc.append_token_ids(block.token_ids)
490
+ block.block_id = alloc.block_id
491
+
492
493
class PrefixCachingBlock(Block):
494
"""A block implementation that supports prefix caching.
vllm/core/block_manager_v1.py
+1
-5
@@ -541,11 +541,7 @@ class BlockSpaceManagerV1(BlockSpaceManager):
541
542
return new_block_table
543
544
- def swap_in(self,
545
- seq_group: SequenceGroup,
546
- num_lookahead_slots: int = 0) -> List[Tuple[int, int]]:
547
- assert (num_lookahead_slots == 0
548
- ), "BlockSpaceManagerV1 does not support lookahead allocation"
544
+ def swap_in(self, seq_group: SequenceGroup) -> List[Tuple[int, int]]:
545
546
request_id = seq_group.request_id
547
vllm/core/block_manager_v2.py
+133
-7
@@ -1,10 +1,12 @@
1
"""A block manager that manages token blocks."""
2
+from itertools import chain
3
from typing import Dict, List, Optional
4
from typing import Sequence as GenericSequence
5
from typing import Tuple
6
7
from vllm.core.block.block_table import BlockTable
8
from vllm.core.block.cpu_gpu_block_allocator import CpuGpuBlockAllocator
9
+from vllm.core.block.interfaces import Block
10
from vllm.core.block.utils import check_no_caching_or_swa_for_blockmgr_encdec
11
from vllm.core.interfaces import AllocStatus, BlockSpaceManager
12
from vllm.sequence import Sequence, SequenceGroup, SequenceStatus
@@ -217,7 +219,6 @@ class BlockSpaceManagerV2(BlockSpaceManager):
219
num_lookahead_slots=num_lookahead_slots,
220
num_computed_slots=seq.data.get_num_computed_tokens(),
221
)
220
-
222
# Return any new copy-on-writes.
223
new_cows = self.block_allocator.clear_copy_on_writes()
224
return new_cows
@@ -297,20 +298,145 @@ class BlockSpaceManagerV2(BlockSpaceManager):
298
299
def can_swap_in(self, seq_group: SequenceGroup,
300
num_lookahead_slots: int) -> AllocStatus:
300
- return AllocStatus.LATER
301
+ """Returns the AllocStatus for the given sequence_group
302
+ with num_lookahead_slots.
303
+
304
+ Args:
305
+ sequence_group (SequenceGroup): The sequence group to swap in.
306
+ num_lookahead_slots (int): Number of lookahead slots used in
307
+ speculative decoding, default to 0.
308
+
309
+ Returns:
310
+ AllocStatus: The AllocStatus for the given sequence group.
311
+ """
312
+ return self._can_swap(seq_group, Device.GPU, SequenceStatus.SWAPPED,
313
+ num_lookahead_slots)
314
+
315
+ def swap_in(self, seq_group: SequenceGroup) -> List[Tuple[int, int]]:
316
+ """Returns the block id mapping (from CPU to GPU) generated by
317
+ swapping in the given seq_group with num_lookahead_slots.
318
302
- def swap_in(self, seq_group: SequenceGroup,
303
- num_lookahead_slots: int) -> List[Tuple[int, int]]:
304
- raise NotImplementedError
319
+ Args:
320
+ seq_group (SequenceGroup): The sequence group to swap in.
321
+
322
+ Returns:
323
+ List[Tuple[int, int]]: The mapping of swapping block from CPU
324
+ to GPU.
325
+ """
326
+ blocks = self._get_blocks_for_swap(seq_group, SequenceStatus.SWAPPED)
327
+ current_swap_mapping = self.block_allocator.swap(
328
+ blocks=blocks, source_device=Device.CPU, dest_device=Device.GPU)
329
+
330
+ block_number_mapping = {
331
+ self.block_allocator.get_physical_block_id(Device.CPU,
332
+ cpu_block_id):
333
+ self.block_allocator.get_physical_block_id(Device.GPU,
334
+ gpu_block_id)
335
+ for cpu_block_id, gpu_block_id in current_swap_mapping.items()
336
+ }
337
+ # convert to list of tuples once here
338
+ return list(block_number_mapping.items())
339
340
def can_swap_out(self, seq_group: SequenceGroup) -> bool:
341
+ """Returns whether we can swap out the given sequence_group
342
+ with num_lookahead_slots.
343
+
344
+ Args:
345
+ seq_group (SequenceGroup): The sequence group to swap in.
346
+ num_lookahead_slots (int): Number of lookahead slots used in
347
+ speculative decoding, default to 0.
348
+
349
+ Returns:
350
+ bool: Whether it's possible to swap out current sequence group.
351
+ """
352
+ alloc_status = self._can_swap(seq_group, Device.CPU,
353
+ SequenceStatus.RUNNING)
354
+ if alloc_status == AllocStatus.OK:
355
+ return True
356
return False
357
309
- def swap_out(self, seq_group: SequenceGroup) -> List[Tuple[int, int]]:
310
- raise NotImplementedError
358
+ def swap_out(self, sequence_group: SequenceGroup) -> List[Tuple[int, int]]:
359
+ """Returns the block id mapping (from GPU to CPU) generated by
360
+ swapping out the given sequence_group with num_lookahead_slots.
361
+
362
+ Args:
363
+ sequence_group (SequenceGroup): The sequence group to swap in.
364
+
365
+ Returns:
366
+ List[Tuple[int, int]]: The mapping of swapping block from
367
+ GPU to CPU.
368
+ """
369
+ blocks = self._get_blocks_for_swap(sequence_group,
370
+ SequenceStatus.RUNNING)
371
+ current_swap_mapping = self.block_allocator.swap(
372
+ blocks=blocks, source_device=Device.GPU, dest_device=Device.CPU)
373
+ block_number_mapping = {
374
+ self.block_allocator.get_physical_block_id(Device.GPU,
375
+ gpu_block_id):
376
+ self.block_allocator.get_physical_block_id(Device.CPU,
377
+ cpu_block_id)
378
+ for gpu_block_id, cpu_block_id in current_swap_mapping.items()
379
+ }
380
+ # convert to list of tuples once here
381
+ return list(block_number_mapping.items())
382
383
def get_num_free_gpu_blocks(self) -> int:
384
return self.block_allocator.get_num_free_blocks(Device.GPU)
385
386
def get_num_free_cpu_blocks(self) -> int:
387
return self.block_allocator.get_num_free_blocks(Device.CPU)
388
+
389
+ def _can_swap(self,
390
+ seq_group: SequenceGroup,
391
+ device: Device,
392
+ status: SequenceStatus,
393
+ num_lookahead_slots: int = 0) -> AllocStatus:
394
+ """Returns the AllocStatus for swapping in/out the given sequence_group
395
+ on to the 'device'.
396
+
397
+ Args:
398
+ sequence_group (SequenceGroup): The sequence group to swap in.
399
+ device (Device): device to swap the 'seq_group' on.
400
+ status (SequenceStatus): The status of sequence which is needed
401
+ for action. RUNNING for swap out and SWAPPED for swap in
402
+ num_lookahead_slots (int): Number of lookahead slots used in
403
+ speculative decoding, default to 0.
404
+
405
+ Returns:
406
+ AllocStatus: The AllocStatus for swapping in/out the given
407
+ sequence_group on to the 'device'.
408
+ """
409
+ blocks = self._get_blocks_for_swap(seq_group, status)
410
+ num_blocks_touched = self.block_allocator.get_num_blocks_touched(
411
+ blocks, device, num_lookahead_slots)
412
+ watermark_blocks = 0
413
+ if device == Device.GPU:
414
+ watermark_blocks = self.watermark_blocks
415
+ if self.block_allocator.get_num_total_blocks(
416
+ device) < num_blocks_touched:
417
+ return AllocStatus.NEVER
418
+ elif self.block_allocator.get_num_free_blocks(
419
+ device) - num_blocks_touched >= watermark_blocks:
420
+ return AllocStatus.OK
421
+ else:
422
+ return AllocStatus.LATER
423
+
424
+ def _get_blocks_for_swap(self, seq_group: SequenceGroup,
425
+ status: SequenceStatus) -> List[Block]:
426
+ """Returns the list of blocks those are touched by the seq_group
427
+
428
+ Args:
429
+ sequence_group (SequenceGroup): The sequence group to swap in.
430
+ status (SequenceStatus): The status of sequence which is needed
431
+ for action. RUNNING for swap out and SWAPPED for swap in
432
+
433
+ Returns:
434
+ The list of blocks those are touched by the seq_group.
435
+ """
436
+ blocks: Dict[int, List[Block]] = {}
437
+ for seq in seq_group.get_seqs(status=status):
438
+ block_table = self.block_tables[seq.seq_id]
439
+ if block_table.blocks is not None:
440
+ blocks[seq.seq_id] = block_table.blocks
441
+ combined_blocks = list(chain(*blocks.values()))
442
+ return combined_blocks
vllm/core/embedding_model_block_manager.py
+1
-2
@@ -46,8 +46,7 @@ class EmbeddingModelBlockSpaceManager(BlockSpaceManager):
46
num_lookahead_slots: int) -> AllocStatus:
47
return AllocStatus.OK
48
49
- def swap_in(self, seq_group: SequenceGroup,
50
- num_lookahead_slots: int) -> List[Tuple[int, int]]:
49
+ def swap_in(self, seq_group: SequenceGroup) -> List[Tuple[int, int]]:
50
return None # type: ignore
51
52
def can_swap_out(self, seq_group: SequenceGroup) -> bool:
vllm/core/interfaces.py
+1
-2
@@ -73,8 +73,7 @@ class BlockSpaceManager(ABC):
73
pass
74
75
@abstractmethod
76
- def swap_in(self, seq_group: SequenceGroup,
77
- num_lookahead_slots: int) -> List[Tuple[int, int]]:
76
+ def swap_in(self, seq_group: SequenceGroup) -> List[Tuple[int, int]]:
77
pass
78
79
@abstractmethod
vllm/core/scheduler.py
+11
-2
@@ -297,6 +297,8 @@ class Scheduler:
297
self.prev_prompt = False
298
# Latency of the last prompt step
299
self.last_prompt_latency = 0.0
300
+ # preemption mode, RECOMPUTE or SWAP
301
+ self.user_specified_preemption_mode = scheduler_config.preemption_mode
302
303
# The following field is test-only. It is used to inject artificial
304
# preemption.
@@ -522,7 +524,9 @@ class Scheduler:
524
seq_group = swapped_queue[0]
525
526
# If the sequence group cannot be swapped in, stop.
525
- alloc_status = self.block_manager.can_swap_in(seq_group)
527
+ is_prefill = seq_group.is_prefill()
528
+ alloc_status = self.block_manager.can_swap_in(
529
+ seq_group, self._get_num_lookahead_slots(is_prefill))
530
if alloc_status == AllocStatus.LATER:
531
break
532
elif alloc_status == AllocStatus.NEVER:
@@ -1067,12 +1071,17 @@ class Scheduler:
1071
# over sequence groups with a single sequence.
1072
# TODO(woosuk): Support recomputation for sequence groups with multiple
1073
# sequences. This may require a more sophisticated CUDA kernel.
1070
- if preemption_mode is None:
1074
+ if self.user_specified_preemption_mode is None:
1075
if seq_group.get_max_num_running_seqs() == 1:
1076
preemption_mode = PreemptionMode.RECOMPUTE
1077
else:
1078
preemption_mode = PreemptionMode.SWAP
1079
1080
+ elif self.user_specified_preemption_mode == "swap":
1081
+ preemption_mode = PreemptionMode.SWAP
1082
+ else:
1083
+ preemption_mode = PreemptionMode.RECOMPUTE
1084
+
1085
if self.num_cumulative_preemption % 50 == 0:
1086
logger.warning(
1087
"Sequence group %s is preempted by %s mode because there is "
vllm/engine/arg_utils.py
+9
@@ -75,6 +75,7 @@ class EngineArgs:
75
num_gpu_blocks_override: Optional[int] = None
76
num_lookahead_slots: int = 0
77
model_loader_extra_config: Optional[dict] = None
78
+ preemption_mode: Optional[str] = None
79
80
# Related to Vision-language models such as llava
81
image_input_type: Optional[str] = None
@@ -564,6 +565,13 @@ class EngineArgs:
565
'corresponding to the chosen load_format. '
566
'This should be a JSON string that will be '
567
'parsed into a dictionary.')
568
+ parser.add_argument(
569
+ '--preemption_mode',
570
+ type=str,
571
+ default=None,
572
+ help='If \'recompute\', the engine performs preemption by block '
573
+ 'swapping; If \'swap\', the engine performs preemption by block '
574
+ 'swapping.')
575
576
parser.add_argument(
577
"--served-model-name",
@@ -667,6 +675,7 @@ class EngineArgs:
675
delay_factor=self.scheduler_delay_factor,
676
enable_chunked_prefill=self.enable_chunked_prefill,
677
embedding_mode=model_config.embedding_mode,
678
+ preemption_mode=self.preemption_mode,
679
)
680
lora_config = LoRAConfig(
681
max_lora_rank=self.max_lora_rank,