@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 469f85c782
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Core][Optimization] change copy-on-write from dict[int, list] to list (#4648)

youkaichao committed May 7, 2024 at 11:06 UTC 469f85c7829c301b6dec48725951b5501c18d611
12 files changed +44 -44
tests/core/block/test_block_table.py
+2 -4
@@ -410,8 +410,7 @@ def test_cow(block_size: int, sequence_len: int, append_len: int,
410 expected_src = static_block_table.physical_block_ids[cow_block_id]
411 expected_dst = appender_block_table.physical_block_ids[cow_block_id]
412
413 - assert expected_src in cows
414 - assert expected_dst in cows[expected_src]
413 + assert (expected_src, expected_dst) in cows
414 else:
415 # Otherwise, there should be no copy-on-write.
416 assert not cows
@@ -490,8 +489,7 @@ def test_cow_lookahead_simple(block_size: int, sequence_len: int,
489 expected_src = static_block_table.physical_block_ids[cow_block_id]
490 expected_dst = appender_block_table.physical_block_ids[cow_block_id]
491
493 - assert expected_src in cows
494 - assert expected_dst in cows[expected_src]
492 + assert (expected_src, expected_dst) in cows
493
494 static_block_table.free()
495 appender_block_table.free()
tests/core/test_block_manager.py
+5 -1
@@ -1,4 +1,5 @@
1 import time
2 +from collections import defaultdict
3 from typing import List
4
5 import pytest
@@ -155,7 +156,10 @@ def test_append_slot_cow():
156
157 cows = block_manager.append_slots(child)
158 assert cows
158 - for src_block, dst_blocks in cows.items():
159 + dict_cows = defaultdict(list)
160 + for src_block, dst_block in cows:
161 + dict_cows[src_block].append(dst_block)
162 + for src_block, dst_blocks in dict_cows.items():
163 assert src_block not in dst_blocks
164
165 after_blocks = block_manager.get_num_free_gpu_blocks()
tests/core/test_scheduler.py
+2 -2
@@ -636,7 +636,7 @@ def test_schedule_decode_blocks_to_copy_update():
636
637 # The last request should be swapped out.
638 scheduler.block_manager.append_slots = MagicMock()
639 - scheduler.block_manager.append_slots.return_value = {2: [3]}
639 + scheduler.block_manager.append_slots.return_value = [(2, 3)]
640
641 budget = create_token_budget()
642 remaining_running, output = scheduler._schedule_running(
@@ -845,7 +845,7 @@ def test_schedule_swapped_blocks_to_copy():
845
846 # The last request should be swapped out.
847 scheduler.block_manager.append_slots = MagicMock()
848 - scheduler.block_manager.append_slots.return_value = {2: [3]}
848 + scheduler.block_manager.append_slots.return_value = [(2, 3)]
849
850 budget = create_token_budget()
851 remaining_swapped, output = scheduler._schedule_swapped(
vllm/core/block/common.py
+10 -11
@@ -1,5 +1,4 @@
1 -from collections import defaultdict
2 -from typing import Dict, Iterable, List, Optional, Protocol
1 +from typing import Dict, Iterable, List, Optional, Protocol, Tuple
2
3 from vllm.core.block.interfaces import Block, BlockAllocator
4
@@ -111,7 +110,7 @@ class CopyOnWriteTracker:
110 refcounter: RefCounterProtocol,
111 allocator: BlockAllocator,
112 ):
114 - self._copy_on_writes: Dict[BlockId, List[BlockId]] = defaultdict(list)
113 + self._copy_on_writes: List[Tuple[BlockId, BlockId]] = []
114 self._refcounter = refcounter
115 self._allocator = allocator
116
@@ -152,25 +151,25 @@ class CopyOnWriteTracker:
151 # Track src/dst copy.
152 assert src_block_id is not None
153 assert block_id is not None
155 - self._copy_on_writes[src_block_id].append(block_id)
154 + self._copy_on_writes.append((src_block_id, block_id))
155
156 return block_id
157
159 - def clear_cows(self) -> Dict[BlockId, List[BlockId]]:
158 + def clear_cows(self) -> List[Tuple[BlockId, BlockId]]:
159 """Clears the copy-on-write tracking information and returns the current
160 state.
161
163 - This method returns a dictionary mapping source block indices to lists
164 - of destination block indices for the current copy-on-write operations.
162 + This method returns a list mapping source block indices to
163 + destination block indices for the current copy-on-write operations.
164 It then clears the internal tracking information.
165
166 Returns:
168 - Dict[BlockId, List[BlockId]]: A dictionary mapping source
169 - block indices to lists of destination block indices for the
167 + List[Tuple[BlockId, BlockId]]: A list mapping source
168 + block indices to destination block indices for the
169 current copy-on-write operations.
170 """
172 - cows = dict(self._copy_on_writes)
173 - self._copy_on_writes.clear()
171 + cows = self._copy_on_writes
172 + self._copy_on_writes = []
173 return cows
174
175
vllm/core/block/cpu_gpu_block_allocator.py
+4 -4
@@ -1,4 +1,4 @@
1 -from typing import Dict, FrozenSet, List, Optional
1 +from typing import Dict, FrozenSet, List, Optional, Tuple
2
3 from vllm.core.block.interfaces import (Block, BlockAllocator, BlockId,
4 DeviceAwareBlockAllocator)
@@ -185,13 +185,13 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
185 def get_num_total_blocks(self, device: Device) -> int:
186 return self._allocators[device].get_num_total_blocks()
187
188 - def clear_copy_on_writes(self) -> Dict[int, List[int]]:
188 + def clear_copy_on_writes(self) -> List[Tuple[int, int]]:
189 """Clears the copy-on-write (CoW) state and returns the mapping of
190 source to destination block IDs.
191
192 Returns:
193 - Dict[int, List[int]]: A dictionary mapping source block IDs to lists
194 - of destination block IDs.
193 + List[Tuple[int, int]]: A list mapping source block IDs to
194 + destination block IDs.
195 """
196 # CoW only supported on GPU
197 device = Device.GPU
vllm/core/block/interfaces.py
+3 -3
@@ -1,5 +1,5 @@
1 from abc import ABC, abstractmethod
2 -from typing import Dict, FrozenSet, List, Optional, Protocol
2 +from typing import FrozenSet, List, Optional, Protocol, Tuple
3
4 from vllm.utils import Device
5
@@ -122,7 +122,7 @@ class BlockAllocator(ABC):
122 pass
123
124 @abstractmethod
125 - def clear_copy_on_writes(self) -> Dict[int, List[int]]:
125 + def clear_copy_on_writes(self) -> List[Tuple[int, int]]:
126 pass
127
128 @abstractmethod
@@ -187,7 +187,7 @@ class DeviceAwareBlockAllocator(ABC):
187 pass
188
189 @abstractmethod
190 - def clear_copy_on_writes(self) -> Dict[int, List[int]]:
190 + def clear_copy_on_writes(self) -> List[Tuple[int, int]]:
191 pass
192
193 @abstractmethod
vllm/core/block/naive_block.py
+4 -4
@@ -1,4 +1,4 @@
1 -from typing import Dict, FrozenSet, Iterable, List, Optional, Set
1 +from typing import FrozenSet, Iterable, List, Optional, Set, Tuple
2
3 from vllm.core.block.common import (CopyOnWriteTracker, RefCounter,
4 get_all_blocks_recursively)
@@ -175,12 +175,12 @@ class NaiveBlockAllocator(BlockAllocator):
175 """
176 return self._cow_tracker.cow_block_if_not_appendable(block)
177
178 - def clear_copy_on_writes(self) -> Dict[BlockId, List[BlockId]]:
178 + def clear_copy_on_writes(self) -> List[Tuple[BlockId, BlockId]]:
179 """Returns the copy-on-write source->destination mapping and clears it.
180
181 Returns:
182 - Dict[BlockId, List[BlockId]]: A dictionary mapping source
183 - block indices to lists of destination block indices.
182 + List[Tuple[BlockId, BlockId]]: A list mapping source
183 + block indices to destination block indices.
184 """
185 return self._cow_tracker.clear_cows()
186
vllm/core/block/prefix_caching_block.py
+4 -4
@@ -1,7 +1,7 @@
1 """Token blocks."""
2 from itertools import takewhile
3 from os.path import commonprefix
4 -from typing import Dict, FrozenSet, Iterable, List, Optional
4 +from typing import Dict, FrozenSet, Iterable, List, Optional, Tuple
5
6 from vllm.core.block.common import (CopyOnWriteTracker,
7 get_all_blocks_recursively)
@@ -337,12 +337,12 @@ class PrefixCachingBlockAllocator(BlockAllocator):
337 """
338 return self._cow_tracker.cow_block_if_not_appendable(block)
339
340 - def clear_copy_on_writes(self) -> Dict[BlockId, List[BlockId]]:
340 + def clear_copy_on_writes(self) -> List[Tuple[BlockId, BlockId]]:
341 """Returns the copy-on-write source->destination mapping and clears it.
342
343 Returns:
344 - Dict[BlockId, List[BlockId]]: A dictionary mapping source
345 - block indices to lists of destination block indices.
344 + List[Tuple[BlockId, BlockId]]: A list mapping source
345 + block indices to destination block indices.
346 """
347 return self._cow_tracker.clear_cows()
348
vllm/core/block_manager_v1.py
+5 -5
@@ -5,7 +5,7 @@ from itertools import count, takewhile
5 from os.path import commonprefix
6 from typing import Dict, List, Optional
7 from typing import Sequence as GenericSequence
8 -from typing import Set
8 +from typing import Set, Tuple
9
10 from vllm.block import BlockTable, PhysicalTokenBlock
11 from vllm.core.evictor_v1 import EvictionPolicy, Evictor, make_evictor
@@ -386,7 +386,7 @@ class BlockSpaceManagerV1(BlockSpaceManager):
386 self,
387 seq: Sequence,
388 num_lookahead_slots: int = 0,
389 - ) -> Dict[int, List[int]]:
389 + ) -> List[Tuple[int, int]]:
390 """Allocate a physical slot for a new token."""
391 logical_blocks = seq.logical_token_blocks
392 block_table = self.block_tables[seq.seq_id]
@@ -405,7 +405,7 @@ class BlockSpaceManagerV1(BlockSpaceManager):
405 # Allocate a new physical block.
406 new_block = self._allocate_last_physical_block(seq)
407 block_table.append(new_block)
408 - return {}
408 + return []
409
410 # We want to append the token to the last physical block.
411 last_block = block_table[-1]
@@ -418,7 +418,7 @@ class BlockSpaceManagerV1(BlockSpaceManager):
418 maybe_new_block = self._maybe_promote_last_block(
419 seq, last_block)
420 block_table[-1] = maybe_new_block
421 - return {}
421 + return []
422 else:
423 # The last block is shared with other sequences.
424 # Copy on Write: Allocate a new block and copy the tokens.
@@ -426,7 +426,7 @@ class BlockSpaceManagerV1(BlockSpaceManager):
426
427 block_table[-1] = new_block
428 self.gpu_allocator.free(last_block)
429 - return {last_block.block_number: [new_block.block_number]}
429 + return [(last_block.block_number, new_block.block_number)]
430
431 def fork(self, parent_seq: Sequence, child_seq: Sequence) -> None:
432 # NOTE: fork does not allocate a new physical block.
vllm/core/block_manager_v2.py
+2 -1
@@ -1,6 +1,7 @@
1 """A block manager that manages token blocks."""
2 from typing import Dict, List, Optional
3 from typing import Sequence as GenericSequence
4 +from typing import Tuple
5
6 from vllm.core.block.block_table import BlockTable
7 from vllm.core.block.cpu_gpu_block_allocator import CpuGpuBlockAllocator
@@ -166,7 +167,7 @@ class BlockSpaceManagerV2(BlockSpaceManager):
167 self,
168 seq: Sequence,
169 num_lookahead_slots: int,
169 - ) -> Dict[int, List[int]]:
170 + ) -> List[Tuple[int, int]]:
171
172 block_table = self.block_tables[seq.seq_id]
173
vllm/core/interfaces.py
+2 -1
@@ -2,6 +2,7 @@ import enum
2 from abc import ABC, abstractmethod
3 from typing import Dict, List
4 from typing import Sequence as GenericSequence
5 +from typing import Tuple
6
7 from vllm.sequence import Sequence, SequenceGroup
8
@@ -54,7 +55,7 @@ class BlockSpaceManager(ABC):
55 self,
56 seq: Sequence,
57 num_lookahead_slots: int,
57 - ) -> Dict[int, List[int]]:
58 + ) -> List[Tuple[int, int]]:
59 pass
60
61 @abstractmethod
vllm/core/scheduler.py
+1 -4
@@ -1027,10 +1027,7 @@ class Scheduler:
1027
1028 for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
1029 cows = self.block_manager.append_slots(seq, num_lookahead_slots)
1030 -
1031 - for src, dests in cows.items():
1032 - for dest in dests:
1033 - blocks_to_copy.append((src, dest))
1030 + blocks_to_copy.extend(cows)
1031
1032 def _preempt(
1033 self,