A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Core][Optimization] change copy-on-write from dict[int, list] to list (#4648)
youkaichao committed
May 7, 2024 at 11:06 UTC
469f85c7829c301b6dec48725951b5501c18d611
12 files changed
+44
-44
tests/core/block/test_block_table.py
+2
-4
@@ -410,8 +410,7 @@ def test_cow(block_size: int, sequence_len: int, append_len: int,
410
expected_src = static_block_table.physical_block_ids[cow_block_id]
411
expected_dst = appender_block_table.physical_block_ids[cow_block_id]
412
413
- assert expected_src in cows
414
- assert expected_dst in cows[expected_src]
413
+ assert (expected_src, expected_dst) in cows
414
else:
415
# Otherwise, there should be no copy-on-write.
416
assert not cows
@@ -490,8 +489,7 @@ def test_cow_lookahead_simple(block_size: int, sequence_len: int,
489
expected_src = static_block_table.physical_block_ids[cow_block_id]
490
expected_dst = appender_block_table.physical_block_ids[cow_block_id]
491
493
- assert expected_src in cows
494
- assert expected_dst in cows[expected_src]
492
+ assert (expected_src, expected_dst) in cows
493
494
static_block_table.free()
495
appender_block_table.free()
tests/core/test_block_manager.py
+5
-1
@@ -1,4 +1,5 @@
1
import time
2
+from collections import defaultdict
3
from typing import List
4
5
import pytest
@@ -155,7 +156,10 @@ def test_append_slot_cow():
156
157
cows = block_manager.append_slots(child)
158
assert cows
158
- for src_block, dst_blocks in cows.items():
159
+ dict_cows = defaultdict(list)
160
+ for src_block, dst_block in cows:
161
+ dict_cows[src_block].append(dst_block)
162
+ for src_block, dst_blocks in dict_cows.items():
163
assert src_block not in dst_blocks
164
165
after_blocks = block_manager.get_num_free_gpu_blocks()
tests/core/test_scheduler.py
+2
-2
@@ -636,7 +636,7 @@ def test_schedule_decode_blocks_to_copy_update():
636
637
# The last request should be swapped out.
638
scheduler.block_manager.append_slots = MagicMock()
639
- scheduler.block_manager.append_slots.return_value = {2: [3]}
639
+ scheduler.block_manager.append_slots.return_value = [(2, 3)]
640
641
budget = create_token_budget()
642
remaining_running, output = scheduler._schedule_running(
@@ -845,7 +845,7 @@ def test_schedule_swapped_blocks_to_copy():
845
846
# The last request should be swapped out.
847
scheduler.block_manager.append_slots = MagicMock()
848
- scheduler.block_manager.append_slots.return_value = {2: [3]}
848
+ scheduler.block_manager.append_slots.return_value = [(2, 3)]
849
850
budget = create_token_budget()
851
remaining_swapped, output = scheduler._schedule_swapped(
vllm/core/block/common.py
+10
-11
@@ -1,5 +1,4 @@
1
-from collections import defaultdict
2
-from typing import Dict, Iterable, List, Optional, Protocol
1
+from typing import Dict, Iterable, List, Optional, Protocol, Tuple
2
3
from vllm.core.block.interfaces import Block, BlockAllocator
4
@@ -111,7 +110,7 @@ class CopyOnWriteTracker:
110
refcounter: RefCounterProtocol,
111
allocator: BlockAllocator,
112
):
114
- self._copy_on_writes: Dict[BlockId, List[BlockId]] = defaultdict(list)
113
+ self._copy_on_writes: List[Tuple[BlockId, BlockId]] = []
114
self._refcounter = refcounter
115
self._allocator = allocator
116
@@ -152,25 +151,25 @@ class CopyOnWriteTracker:
151
# Track src/dst copy.
152
assert src_block_id is not None
153
assert block_id is not None
155
- self._copy_on_writes[src_block_id].append(block_id)
154
+ self._copy_on_writes.append((src_block_id, block_id))
155
156
return block_id
157
159
- def clear_cows(self) -> Dict[BlockId, List[BlockId]]:
158
+ def clear_cows(self) -> List[Tuple[BlockId, BlockId]]:
159
"""Clears the copy-on-write tracking information and returns the current
160
state.
161
163
- This method returns a dictionary mapping source block indices to lists
164
- of destination block indices for the current copy-on-write operations.
162
+ This method returns a list mapping source block indices to
163
+ destination block indices for the current copy-on-write operations.
164
It then clears the internal tracking information.
165
166
Returns:
168
- Dict[BlockId, List[BlockId]]: A dictionary mapping source
169
- block indices to lists of destination block indices for the
167
+ List[Tuple[BlockId, BlockId]]: A list mapping source
168
+ block indices to destination block indices for the
169
current copy-on-write operations.
170
"""
172
- cows = dict(self._copy_on_writes)
173
- self._copy_on_writes.clear()
171
+ cows = self._copy_on_writes
172
+ self._copy_on_writes = []
173
return cows
174
175
vllm/core/block/cpu_gpu_block_allocator.py
+4
-4
@@ -1,4 +1,4 @@
1
-from typing import Dict, FrozenSet, List, Optional
1
+from typing import Dict, FrozenSet, List, Optional, Tuple
2
3
from vllm.core.block.interfaces import (Block, BlockAllocator, BlockId,
4
DeviceAwareBlockAllocator)
@@ -185,13 +185,13 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
185
def get_num_total_blocks(self, device: Device) -> int:
186
return self._allocators[device].get_num_total_blocks()
187
188
- def clear_copy_on_writes(self) -> Dict[int, List[int]]:
188
+ def clear_copy_on_writes(self) -> List[Tuple[int, int]]:
189
"""Clears the copy-on-write (CoW) state and returns the mapping of
190
source to destination block IDs.
191
192
Returns:
193
- Dict[int, List[int]]: A dictionary mapping source block IDs to lists
194
- of destination block IDs.
193
+ List[Tuple[int, int]]: A list mapping source block IDs to
194
+ destination block IDs.
195
"""
196
# CoW only supported on GPU
197
device = Device.GPU
vllm/core/block/interfaces.py
+3
-3
@@ -1,5 +1,5 @@
1
from abc import ABC, abstractmethod
2
-from typing import Dict, FrozenSet, List, Optional, Protocol
2
+from typing import FrozenSet, List, Optional, Protocol, Tuple
3
4
from vllm.utils import Device
5
@@ -122,7 +122,7 @@ class BlockAllocator(ABC):
122
pass
123
124
@abstractmethod
125
- def clear_copy_on_writes(self) -> Dict[int, List[int]]:
125
+ def clear_copy_on_writes(self) -> List[Tuple[int, int]]:
126
pass
127
128
@abstractmethod
@@ -187,7 +187,7 @@ class DeviceAwareBlockAllocator(ABC):
187
pass
188
189
@abstractmethod
190
- def clear_copy_on_writes(self) -> Dict[int, List[int]]:
190
+ def clear_copy_on_writes(self) -> List[Tuple[int, int]]:
191
pass
192
193
@abstractmethod
vllm/core/block/naive_block.py
+4
-4
@@ -1,4 +1,4 @@
1
-from typing import Dict, FrozenSet, Iterable, List, Optional, Set
1
+from typing import FrozenSet, Iterable, List, Optional, Set, Tuple
2
3
from vllm.core.block.common import (CopyOnWriteTracker, RefCounter,
4
get_all_blocks_recursively)
@@ -175,12 +175,12 @@ class NaiveBlockAllocator(BlockAllocator):
175
"""
176
return self._cow_tracker.cow_block_if_not_appendable(block)
177
178
- def clear_copy_on_writes(self) -> Dict[BlockId, List[BlockId]]:
178
+ def clear_copy_on_writes(self) -> List[Tuple[BlockId, BlockId]]:
179
"""Returns the copy-on-write source->destination mapping and clears it.
180
181
Returns:
182
- Dict[BlockId, List[BlockId]]: A dictionary mapping source
183
- block indices to lists of destination block indices.
182
+ List[Tuple[BlockId, BlockId]]: A list mapping source
183
+ block indices to destination block indices.
184
"""
185
return self._cow_tracker.clear_cows()
186
vllm/core/block/prefix_caching_block.py
+4
-4
@@ -1,7 +1,7 @@
1
"""Token blocks."""
2
from itertools import takewhile
3
from os.path import commonprefix
4
-from typing import Dict, FrozenSet, Iterable, List, Optional
4
+from typing import Dict, FrozenSet, Iterable, List, Optional, Tuple
5
6
from vllm.core.block.common import (CopyOnWriteTracker,
7
get_all_blocks_recursively)
@@ -337,12 +337,12 @@ class PrefixCachingBlockAllocator(BlockAllocator):
337
"""
338
return self._cow_tracker.cow_block_if_not_appendable(block)
339
340
- def clear_copy_on_writes(self) -> Dict[BlockId, List[BlockId]]:
340
+ def clear_copy_on_writes(self) -> List[Tuple[BlockId, BlockId]]:
341
"""Returns the copy-on-write source->destination mapping and clears it.
342
343
Returns:
344
- Dict[BlockId, List[BlockId]]: A dictionary mapping source
345
- block indices to lists of destination block indices.
344
+ List[Tuple[BlockId, BlockId]]: A list mapping source
345
+ block indices to destination block indices.
346
"""
347
return self._cow_tracker.clear_cows()
348
vllm/core/block_manager_v1.py
+5
-5
@@ -5,7 +5,7 @@ from itertools import count, takewhile
5
from os.path import commonprefix
6
from typing import Dict, List, Optional
7
from typing import Sequence as GenericSequence
8
-from typing import Set
8
+from typing import Set, Tuple
9
10
from vllm.block import BlockTable, PhysicalTokenBlock
11
from vllm.core.evictor_v1 import EvictionPolicy, Evictor, make_evictor
@@ -386,7 +386,7 @@ class BlockSpaceManagerV1(BlockSpaceManager):
386
self,
387
seq: Sequence,
388
num_lookahead_slots: int = 0,
389
- ) -> Dict[int, List[int]]:
389
+ ) -> List[Tuple[int, int]]:
390
"""Allocate a physical slot for a new token."""
391
logical_blocks = seq.logical_token_blocks
392
block_table = self.block_tables[seq.seq_id]
@@ -405,7 +405,7 @@ class BlockSpaceManagerV1(BlockSpaceManager):
405
# Allocate a new physical block.
406
new_block = self._allocate_last_physical_block(seq)
407
block_table.append(new_block)
408
- return {}
408
+ return []
409
410
# We want to append the token to the last physical block.
411
last_block = block_table[-1]
@@ -418,7 +418,7 @@ class BlockSpaceManagerV1(BlockSpaceManager):
418
maybe_new_block = self._maybe_promote_last_block(
419
seq, last_block)
420
block_table[-1] = maybe_new_block
421
- return {}
421
+ return []
422
else:
423
# The last block is shared with other sequences.
424
# Copy on Write: Allocate a new block and copy the tokens.
@@ -426,7 +426,7 @@ class BlockSpaceManagerV1(BlockSpaceManager):
426
427
block_table[-1] = new_block
428
self.gpu_allocator.free(last_block)
429
- return {last_block.block_number: [new_block.block_number]}
429
+ return [(last_block.block_number, new_block.block_number)]
430
431
def fork(self, parent_seq: Sequence, child_seq: Sequence) -> None:
432
# NOTE: fork does not allocate a new physical block.
vllm/core/block_manager_v2.py
+2
-1
@@ -1,6 +1,7 @@
1
"""A block manager that manages token blocks."""
2
from typing import Dict, List, Optional
3
from typing import Sequence as GenericSequence
4
+from typing import Tuple
5
6
from vllm.core.block.block_table import BlockTable
7
from vllm.core.block.cpu_gpu_block_allocator import CpuGpuBlockAllocator
@@ -166,7 +167,7 @@ class BlockSpaceManagerV2(BlockSpaceManager):
167
self,
168
seq: Sequence,
169
num_lookahead_slots: int,
169
- ) -> Dict[int, List[int]]:
170
+ ) -> List[Tuple[int, int]]:
171
172
block_table = self.block_tables[seq.seq_id]
173
vllm/core/interfaces.py
+2
-1
@@ -2,6 +2,7 @@ import enum
2
from abc import ABC, abstractmethod
3
from typing import Dict, List
4
from typing import Sequence as GenericSequence
5
+from typing import Tuple
6
7
from vllm.sequence import Sequence, SequenceGroup
8
@@ -54,7 +55,7 @@ class BlockSpaceManager(ABC):
55
self,
56
seq: Sequence,
57
num_lookahead_slots: int,
57
- ) -> Dict[int, List[int]]:
58
+ ) -> List[Tuple[int, int]]:
59
pass
60
61
@abstractmethod
vllm/core/scheduler.py
+1
-4
@@ -1027,10 +1027,7 @@ class Scheduler:
1027
1028
for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
1029
cows = self.block_manager.append_slots(seq, num_lookahead_slots)
1030
-
1031
- for src, dests in cows.items():
1032
- for dest in dests:
1033
- blocks_to_copy.append((src, dest))
1030
+ blocks_to_copy.extend(cows)
1031
1032
def _preempt(
1033
self,