@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cf8cac8c70
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[mypy][6/N] Fix all the core subdirectory typing (#4450)

Co-authored-by: Cade Daniel <edacih@gmail.com>

SangBin Cho committed May 2, 2024 at 12:01 UTC cf8cac8c701079a3fda068ffd1cd6f72a490aa6d
10 files changed +275 -83
.github/workflows/mypy.yaml
+2 -4
@@ -33,6 +33,7 @@ jobs:
33 - name: Mypy
34 run: |
35 mypy vllm/attention --config-file pyproject.toml
36 + mypy vllm/core --config-file pyproject.toml
37 mypy vllm/distributed --config-file pyproject.toml
38 mypy vllm/entrypoints --config-file pyproject.toml
39 mypy vllm/executor --config-file pyproject.toml
@@ -42,9 +43,6 @@ jobs:
43 mypy vllm/engine --config-file pyproject.toml
44 mypy vllm/worker --config-file pyproject.toml
45 mypy vllm/spec_decode --config-file pyproject.toml
45 - mypy vllm/lora --config-file pyproject.toml
46 mypy vllm/model_executor --config-file pyproject.toml
47 -
48 - # TODO(sang): Fix nested dir
49 - mypy vllm/core/*.py --follow-imports=skip --config-file pyproject.toml
47 + mypy vllm/lora --config-file pyproject.toml
48
format.sh
+1 -1
@@ -95,7 +95,7 @@ echo 'vLLM yapf: Done'
95 # Run mypy
96 echo 'vLLM mypy:'
97 mypy vllm/attention --config-file pyproject.toml
98 -mypy vllm/core/*.py --follow-imports=skip --config-file pyproject.toml
98 +mypy vllm/core --config-file pyproject.toml
99 mypy vllm/distributed --config-file pyproject.toml
100 mypy vllm/entrypoints --config-file pyproject.toml
101 mypy vllm/executor --config-file pyproject.toml
vllm/core/block/block_table.py
+10 -6
@@ -40,7 +40,9 @@ class BlockTable:
40 ):
41 self._block_size = block_size
42 self._allocator = block_allocator
43 - self._blocks: Optional[List[Block]] = _blocks
43 + if _blocks is None:
44 + _blocks = []
45 + self._blocks: List[Block] = _blocks
46
47 # Use helper method instead of directly calculating, as blocks
48 # may not be allocated.
@@ -104,7 +106,7 @@ class BlockTable:
106 token_ids (List[int]): The sequence of token IDs to be appended.
107 """
108 assert self._is_allocated
107 - assert self._blocks is not None
109 + assert len(self._blocks) > 0
110
111 self.ensure_num_empty_slots(num_empty_slots=len(token_ids) +
112 num_lookahead_slots)
@@ -141,6 +143,7 @@ class BlockTable:
143 blocks_to_allocate = cdiv(slots_to_allocate, self._block_size)
144
145 for _ in range(blocks_to_allocate):
146 + assert len(self._blocks) > 0
147 self._blocks.append(
148 self._allocator.allocate_mutable(prev_block=self._blocks[-1],
149 device=device))
@@ -159,6 +162,7 @@ class BlockTable:
162 the current instance.
163 """
164 assert self._is_allocated
165 + assert len(self._blocks) > 0
166 forked_blocks = self._allocator.fork(self._blocks[-1])
167 return BlockTable(
168 block_size=self._block_size,
@@ -177,10 +181,10 @@ class BlockTable:
181 assert self._is_allocated
182 for block in self._blocks:
183 self._allocator.free(block)
180 - self._blocks = None
184 + self._blocks = []
185
186 @property
183 - def physical_block_ids(self) -> List[int]:
187 + def physical_block_ids(self) -> List[Optional[int]]:
188 """Returns a list of physical block indices for the blocks in the
189 BlockTable.
190
@@ -235,7 +239,7 @@ class BlockTable:
239
240 def _get_all_token_ids(self) -> List[int]:
241 # NOTE: This function is O(seq_len); use sparingly.
238 - token_ids = []
242 + token_ids: List[int] = []
243
244 if not self._is_allocated:
245 return token_ids
@@ -247,7 +251,7 @@ class BlockTable:
251
252 @property
253 def _is_allocated(self) -> bool:
250 - return self._blocks is not None
254 + return len(self._blocks) > 0
255
256 @property
257 def _num_empty_slots(self) -> int:
vllm/core/block/common.py
+16 -4
@@ -1,5 +1,5 @@
1 from collections import defaultdict
2 -from typing import Dict, Iterable, List, Optional
2 +from typing import Dict, Iterable, List, Optional, Protocol
3
4 from vllm.core.block.interfaces import Block, BlockAllocator
5
@@ -7,7 +7,19 @@ BlockId = int
7 RefCount = int
8
9
10 -class RefCounter:
10 +class RefCounterProtocol(Protocol):
11 +
12 + def incr(self, block_id: BlockId) -> RefCount:
13 + raise NotImplementedError
14 +
15 + def decr(self, block_id: BlockId) -> RefCount:
16 + raise NotImplementedError
17 +
18 + def get(self, block_id: BlockId) -> RefCount:
19 + raise NotImplementedError
20 +
21 +
22 +class RefCounter(RefCounterProtocol):
23 """A class for managing reference counts for a set of block indices.
24
25 The RefCounter class maintains a dictionary that maps block indices to their
@@ -54,7 +66,7 @@ class RefCounter:
66 return ReadOnlyRefCounter(self)
67
68
57 -class ReadOnlyRefCounter:
69 +class ReadOnlyRefCounter(RefCounterProtocol):
70 """A read-only view of the RefCounter class.
71
72 The ReadOnlyRefCounter class provides a read-only interface to access the
@@ -96,7 +108,7 @@ class CopyOnWriteTracker:
108
109 def __init__(
110 self,
99 - refcounter: RefCounter,
111 + refcounter: RefCounterProtocol,
112 allocator: BlockAllocator,
113 ):
114 self._copy_on_writes: Dict[BlockId, List[BlockId]] = defaultdict(list)
vllm/core/block/cpu_gpu_block_allocator.py
+33 -16
@@ -1,6 +1,6 @@
1 -from typing import Dict, List, Optional
1 +from typing import Dict, FrozenSet, List, Optional
2
3 -from vllm.core.block.interfaces import (Block, BlockAllocator,
3 +from vllm.core.block.interfaces import (Block, BlockAllocator, BlockId,
4 DeviceAwareBlockAllocator)
5 from vllm.core.block.naive_block import NaiveBlock, NaiveBlockAllocator
6 from vllm.core.block.prefix_caching_block import PrefixCachingBlockAllocator
@@ -57,15 +57,15 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
57 cpu_block_ids = block_ids[num_gpu_blocks:]
58
59 if allocator_type == "naive":
60 - gpu_allocator = NaiveBlockAllocator(
61 - create_block=NaiveBlock,
60 + gpu_allocator: BlockAllocator = NaiveBlockAllocator(
61 + create_block=NaiveBlock, # type: ignore
62 num_blocks=num_gpu_blocks,
63 block_size=block_size,
64 block_ids=gpu_block_ids,
65 )
66
67 - cpu_allocator = NaiveBlockAllocator(
68 - create_block=NaiveBlock,
67 + cpu_allocator: BlockAllocator = NaiveBlockAllocator(
68 + create_block=NaiveBlock, # type: ignore
69 num_blocks=num_cpu_blocks,
70 block_size=block_size,
71 block_ids=cpu_block_ids,
@@ -105,13 +105,14 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
105 Device.GPU: gpu_block_allocator,
106 }
107
108 - self._block_ids_to_allocator = {}
108 + self._block_ids_to_allocator: Dict[int, BlockAllocator] = {}
109 for _, allocator in self._allocators.items():
110 for block_id in allocator.all_block_ids:
111 self._block_ids_to_allocator[block_id] = allocator
112
113 - def allocate_mutable(self, prev_block: Optional[Block],
114 - device: Device) -> Block:
113 + def allocate_mutable(self,
114 + prev_block: Optional[Block],
115 + device: Optional[Device] = None) -> Block:
116 """Allocates a new mutable block on the specified device.
117
118 Args:
@@ -122,10 +123,13 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
123 Returns:
124 Block: The newly allocated mutable block.
125 """
126 + assert device is not None
127 return self._allocators[device].allocate_mutable(prev_block)
128
127 - def allocate_immutable(self, prev_block: Optional[Block],
128 - token_ids: List[int], device: Device) -> Block:
129 + def allocate_immutable(self,
130 + prev_block: Optional[Block],
131 + token_ids: List[int],
132 + device: Optional[Device] = None) -> Block:
133 """Allocates a new immutable block with the provided token IDs on the
134 specified device.
135
@@ -140,6 +144,7 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
144 Block: The newly allocated immutable block containing the provided
145 token IDs.
146 """
147 + assert device is not None
148 return self._allocators[device].allocate_immutable(
149 prev_block, token_ids)
150
@@ -149,7 +154,9 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
154 Args:
155 block (Block): The block to be freed.
156 """
152 - allocator = self._block_ids_to_allocator[block.block_id]
157 + block_id = block.block_id
158 + assert block_id is not None
159 + allocator = self._block_ids_to_allocator[block_id]
160 return allocator.free(block)
161
162 def fork(self, last_block: Block) -> List[Block]:
@@ -163,19 +170,22 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
170 List[Block]: A new list of blocks that shares the same memory as the
171 original sequence.
172 """
166 - allocator = self._block_ids_to_allocator[last_block.block_id]
173 + block_id = last_block.block_id
174 + assert block_id is not None
175 + allocator = self._block_ids_to_allocator[block_id]
176 return allocator.fork(last_block)
177
169 - def get_num_free_blocks(self, device: Device) -> int:
178 + def get_num_free_blocks(self, device: Optional[Device] = None) -> int:
179 """Returns the number of free blocks available on the specified device.
180
181 Args:
182 device (Device): The device for which to query the number of free
174 - blocks.
183 + blocks. AssertionError is raised if None is passed.
184
185 Returns:
186 int: The number of free blocks available on the specified device.
187 """
188 + assert device is not None
189 return self._allocators[device].get_num_free_blocks()
190
191 def clear_copy_on_writes(self) -> Dict[int, List[int]]:
@@ -210,5 +220,12 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
220 return self._allocators[device].get_common_computed_block_ids(
221 seq_block_ids)
222
213 - def all_block_ids(self) -> frozenset[int]:
223 + @property
224 + def all_block_ids(self) -> FrozenSet[int]:
225 return frozenset(self._block_ids_to_allocator.keys())
226 +
227 + def promote_to_immutable_block(self, block: Block) -> BlockId:
228 + raise NotImplementedError
229 +
230 + def cow_block_if_not_appendable(self, block: Block) -> Optional[BlockId]:
231 + raise NotImplementedError
vllm/core/block/interfaces.py
+95 -9
@@ -3,6 +3,8 @@ from typing import Dict, FrozenSet, List, Optional, Protocol
3
4 from vllm.utils import Device
5
6 +BlockId = int
7 +
8
9 class Block(ABC):
10
@@ -15,6 +17,12 @@ class Block(ABC):
17 def block_id(self) -> Optional[int]:
18 pass
19
20 + @block_id.setter
21 + @abstractmethod
22 + def block_id(self, value: Optional[int]) -> None:
23 + """NOTE: Do not use this API outside Block."""
24 + self._block_id = value
25 +
26 @property
27 @abstractmethod
28 def token_ids(self) -> List[int]:
@@ -35,6 +43,27 @@ class Block(ABC):
43 def prev_block(self) -> Optional["Block"]:
44 pass
45
46 + @property
47 + @abstractmethod
48 + def computed(self) -> bool:
49 + raise NotImplementedError
50 +
51 + @computed.setter
52 + @abstractmethod
53 + def computed(self, value) -> bool:
54 + """Should be only used by PrefixCacingAllocator"""
55 + raise NotImplementedError
56 +
57 + @property
58 + @abstractmethod
59 + def last_accessed(self) -> float:
60 + raise NotImplementedError
61 +
62 + @last_accessed.setter
63 + @abstractmethod
64 + def last_accessed(self, last_accessed_ts: float):
65 + raise NotImplementedError
66 +
67 class Factory(Protocol):
68
69 @abstractmethod
@@ -48,6 +77,17 @@ class Block(ABC):
77 ) -> "Block":
78 pass
79
80 + @property
81 + @abstractmethod
82 + def content_hash(self) -> Optional[int]:
83 + """Return the content-based hash of the current block, or None if it is
84 + not yet defined or not supported.
85 +
86 + For the content-based hash to be defined, the current block must be
87 + full.
88 + """
89 + return None
90 +
91
92 class BlockAllocator(ABC):
93
@@ -57,7 +97,7 @@ class BlockAllocator(ABC):
97
98 @abstractmethod
99 def allocate_immutable(self, prev_block: Optional[Block],
60 - token_ids: List[int], device: Device) -> Block:
100 + token_ids: List[int]) -> Block:
101 pass
102
103 @abstractmethod
@@ -69,7 +109,7 @@ class BlockAllocator(ABC):
109 pass
110
111 @abstractmethod
72 - def get_num_free_blocks(self, device: Device) -> int:
112 + def get_num_free_blocks(self) -> int:
113 pass
114
115 @property
@@ -82,11 +122,12 @@ class BlockAllocator(ABC):
122 pass
123
124 @abstractmethod
85 - def mark_blocks_as_accessed(self) -> None:
125 + def mark_blocks_as_accessed(self, block_ids: List[int],
126 + now: float) -> None:
127 pass
128
129 @abstractmethod
89 - def mark_blocks_as_computed(self) -> None:
130 + def mark_blocks_as_computed(self, block_ids: List[int]) -> None:
131 pass
132
133 @abstractmethod
@@ -94,21 +135,66 @@ class BlockAllocator(ABC):
135 self, seq_block_ids: List[List[int]]) -> List[int]:
136 pass
137
138 + @abstractmethod
139 + def cow_block_if_not_appendable(self, block: Block) -> Optional["BlockId"]:
140 + """NOTE: This should not be used besides Block"""
141 + pass
142 +
143 + @abstractmethod
144 + def promote_to_immutable_block(self, block: Block) -> BlockId:
145 + """NOTE: This should not be used besides Block"""
146 + pass
147 +
148 class NoFreeBlocksError(ValueError):
149 pass
150
151
101 -class DeviceAwareBlockAllocator(BlockAllocator):
152 +class DeviceAwareBlockAllocator(ABC):
153
154 @abstractmethod
104 - def allocate_mutable(self, prev_block: Optional[Block]) -> Block:
155 + def allocate_mutable(self,
156 + prev_block: Optional[Block],
157 + device: Optional[Device] = None) -> Block:
158 pass
159
160 @abstractmethod
108 - def allocate_immutable(self, prev_block: Optional[Block],
109 - token_ids: List[int], device: Device) -> Block:
161 + def allocate_immutable(self,
162 + prev_block: Optional[Block],
163 + token_ids: List[int],
164 + device: Optional[Device] = None) -> Block:
165 pass
166
167 @abstractmethod
113 - def get_num_free_blocks(self, device: Device) -> int:
168 + def get_num_free_blocks(self, device: Optional[Device] = None) -> int:
169 + pass
170 +
171 + @abstractmethod
172 + def free(self, block: Block) -> None:
173 + pass
174 +
175 + @abstractmethod
176 + def fork(self, last_block: Block) -> List[Block]:
177 + pass
178 +
179 + @property
180 + @abstractmethod
181 + def all_block_ids(self) -> FrozenSet[int]:
182 + pass
183 +
184 + @abstractmethod
185 + def clear_copy_on_writes(self) -> Dict[int, List[int]]:
186 + pass
187 +
188 + @abstractmethod
189 + def mark_blocks_as_accessed(self, block_ids: List[int],
190 + now: float) -> None:
191 + pass
192 +
193 + @abstractmethod
194 + def mark_blocks_as_computed(self, block_ids: List[int]) -> None:
195 + pass
196 +
197 + @abstractmethod
198 + def get_common_computed_block_ids(
199 + self, seq_block_ids: List[List[int]]) -> List[int]:
200 pass
vllm/core/block/naive_block.py
+42 -10
@@ -1,10 +1,9 @@
1 -from typing import Dict, Iterable, List, Optional, Set
1 +from typing import Dict, FrozenSet, Iterable, List, Optional, Set
2
3 from vllm.core.block.common import (CopyOnWriteTracker, RefCounter,
4 get_all_blocks_recursively)
5 -from vllm.core.block.interfaces import Block, BlockAllocator
5 +from vllm.core.block.interfaces import Block, BlockAllocator, BlockId, Device
6
7 -BlockId = int
7 Refcount = int
8
9
@@ -49,8 +48,10 @@ class NaiveBlockAllocator(BlockAllocator):
48 allocator=self,
49 )
50
52 - def allocate_immutable(self, prev_block: Optional[Block],
53 - token_ids: List[int]) -> Block:
51 + def allocate_immutable(self,
52 + prev_block: Optional[Block],
53 + token_ids: List[int],
54 + device: Optional[Device] = None) -> Block:
55 """Allocates a new immutable block with the given token IDs, linked to
56 the previous block.
57
@@ -63,11 +64,14 @@ class NaiveBlockAllocator(BlockAllocator):
64 Returns:
65 Block: The newly allocated immutable block.
66 """
67 + assert device is None
68 block = self.allocate_mutable(prev_block=prev_block)
69 block.append_token_ids(token_ids)
70 return block
71
70 - def allocate_mutable(self, prev_block: Optional[Block]) -> Block:
72 + def allocate_mutable(self,
73 + prev_block: Optional[Block],
74 + device: Optional[Device] = None) -> Block:
75 """Allocates a new mutable block, linked to the previous block.
76
77 Args:
@@ -78,6 +82,7 @@ class NaiveBlockAllocator(BlockAllocator):
82 Returns:
83 Block: The newly allocated mutable block.
84 """
85 + assert device is None
86 block_id = self._allocate_new_block_id()
87 return self._create_block(
88 prev_block=prev_block,
@@ -88,6 +93,7 @@ class NaiveBlockAllocator(BlockAllocator):
93 )
94
95 def free(self, block: Block) -> None:
96 + assert block.block_id is not None
97 self._free_block_id(block.block_id)
98
99 # Mark the block as having no allocation.
@@ -111,6 +117,7 @@ class NaiveBlockAllocator(BlockAllocator):
117 for block in source_blocks:
118
119 # Increment refcount for each block.
120 + assert block.block_id is not None
121 refcount = self._refcounter.incr(block.block_id)
122 assert refcount != 1, "can't fork free'd block"
123
@@ -126,7 +133,8 @@ class NaiveBlockAllocator(BlockAllocator):
133
134 return forked_blocks
135
129 - def get_num_free_blocks(self) -> int:
136 + def get_num_free_blocks(self, device: Optional[Device] = None) -> int:
137 + assert device is None
138 return len(self._free_block_indices)
139
140 def _allocate_new_block_id(self) -> BlockId:
@@ -148,7 +156,7 @@ class NaiveBlockAllocator(BlockAllocator):
156 return self._refcounter
157
158 @property
151 - def all_block_ids(self):
159 + def all_block_ids(self) -> FrozenSet[int]:
160 return self._all_block_indices
161
162 def cow_block_if_not_appendable(self, block: Block) -> Optional[BlockId]:
@@ -200,6 +208,9 @@ class NaiveBlockAllocator(BlockAllocator):
208 """
209 return []
210
211 + def promote_to_immutable_block(self, block: Block) -> BlockId:
212 + raise NotImplementedError
213 +
214
215 class NaiveBlock(Block):
216 """An implementation of the Block class that does not support prefix
@@ -224,13 +235,13 @@ class NaiveBlock(Block):
235 """
236
237 def __init__(self,
227 - prev_block: Block,
238 + prev_block: Optional[Block],
239 token_ids: List[int],
240 block_size: int,
241 allocator: BlockAllocator,
242 block_id: Optional[int] = None,
243 _cow_target: Optional[Block] = None):
233 - self._token_ids = []
244 + self._token_ids: List[int] = []
245 self._block_size = block_size
246 self._prev_block = prev_block
247 self._block_id = block_id
@@ -256,6 +267,22 @@ class NaiveBlock(Block):
267 assert self.num_empty_slots >= len(token_ids)
268 self._token_ids.extend(token_ids)
269
270 + @property
271 + def computed(self) -> bool:
272 + raise NotImplementedError
273 +
274 + @computed.setter
275 + def computed(self, value) -> None:
276 + raise NotImplementedError
277 +
278 + @property
279 + def last_accessed(self) -> float:
280 + raise NotImplementedError
281 +
282 + @last_accessed.setter
283 + def last_accessed(self, last_accessed_ts: float):
284 + raise NotImplementedError
285 +
286 @property
287 def block_id(self) -> Optional[int]:
288 return self._block_id
@@ -276,9 +303,14 @@ class NaiveBlock(Block):
303 def token_ids(self) -> List[int]:
304 return self._token_ids
305
306 + @property
307 def block_size(self) -> int:
308 return self._block_size
309
310 @property
311 def prev_block(self) -> Optional["Block"]:
312 return self._prev_block
313 +
314 + @property
315 + def content_hash(self) -> Optional[int]:
316 + return None
vllm/core/block/prefix_caching_block.py
+60 -25
@@ -1,16 +1,15 @@
1 """Token blocks."""
2 from itertools import takewhile
3 from os.path import commonprefix
4 -from typing import Dict, Iterable, List, Optional
4 +from typing import Dict, FrozenSet, Iterable, List, Optional
5
6 from vllm.core.block.common import (CopyOnWriteTracker,
7 get_all_blocks_recursively)
8 -from vllm.core.block.interfaces import Block, BlockAllocator
8 +from vllm.core.block.interfaces import Block, BlockAllocator, BlockId, Device
9 from vllm.core.block.naive_block import NaiveBlock, NaiveBlockAllocator
10 from vllm.core.evictor_v2 import EvictionPolicy, Evictor, make_evictor
11
12 PrefixHash = int
13 -BlockId = int
13
14 # By default, we init our block access time as _DEFAULT_LAST_ACCESSED_TIME
15 # so that if we find one block is still hold _DEFAULT_LAST_ACCESSED_TIME,
@@ -38,7 +37,7 @@ class PrefixCachingBlockAllocator(BlockAllocator):
37 num_blocks: int,
38 block_size: int,
39 block_ids: Optional[Iterable[int]] = None,
41 - eviction_policy: Optional[EvictionPolicy] = EvictionPolicy.LRU,
40 + eviction_policy: EvictionPolicy = EvictionPolicy.LRU,
41 ):
42 # A mapping of prefix hash to block index. All blocks which have a
43 # prefix hash will be in this dict, even if they have refcount 0.
@@ -49,7 +48,7 @@ class PrefixCachingBlockAllocator(BlockAllocator):
48
49 # An allocator for blocks that do not have prefix hashes.
50 self._hashless_allocator = NaiveBlockAllocator(
52 - create_block=self._create_block,
51 + create_block=self._create_block, # type: ignore
52 num_blocks=num_blocks,
53 block_size=block_size,
54 block_ids=block_ids,
@@ -79,7 +78,7 @@ class PrefixCachingBlockAllocator(BlockAllocator):
78 block_size: int,
79 allocator: BlockAllocator,
80 block_id: Optional[int] = None,
82 - computed: Optional[bool] = False,
81 + computed: bool = False,
82 ) -> Block:
83 # Bind block to self.
84 allocator = self
@@ -93,8 +92,10 @@ class PrefixCachingBlockAllocator(BlockAllocator):
92 computed=computed,
93 )
94
96 - def allocate_immutable(self, prev_block: Optional[Block],
97 - token_ids: List[int]) -> Block:
95 + def allocate_immutable(self,
96 + prev_block: Optional[Block],
97 + token_ids: List[int],
98 + device: Optional[Device] = None) -> Block:
99 """Allocates an immutable block with the given token IDs, reusing cached
100 blocks if possible.
101
@@ -105,6 +106,7 @@ class PrefixCachingBlockAllocator(BlockAllocator):
106 Returns:
107 Block: The allocated immutable block.
108 """
109 + assert device is None
110 assert_prefix_caching_block_or_none(prev_block)
111
112 block = self._create_block(
@@ -127,16 +129,20 @@ class PrefixCachingBlockAllocator(BlockAllocator):
129
130 return block
131
130 - def allocate_mutable(self, prev_block: Block) -> Block:
132 + def allocate_mutable(self,
133 + prev_block: Optional[Block],
134 + device: Optional[Device] = None) -> Block:
135 """Allocates a mutable block. If there are no free blocks, this will
136 evict unused cached blocks.
137
138 Args:
139 prev_block (Block): The previous block in the sequence.
140 + None is not allowed unlike it is super class.
141
142 Returns:
143 Block: The allocated mutable block.
144 """
145 + assert device is None
146 assert_prefix_caching_block_or_none(prev_block)
147
148 try:
@@ -144,6 +150,7 @@ class PrefixCachingBlockAllocator(BlockAllocator):
150 prev_block=prev_block)
151
152 assert block.block_id not in self._blocks
153 + assert block.block_id is not None
154 self._blocks[block.block_id] = block
155 return block
156 except BlockAllocator.NoFreeBlocksError:
@@ -183,6 +190,7 @@ class PrefixCachingBlockAllocator(BlockAllocator):
190 assert block.content_hash is None
191
192 assert block.block_id not in self._blocks
193 + assert block.block_id is not None
194 self._blocks[block.block_id] = block
195 return block
196
@@ -225,6 +233,7 @@ class PrefixCachingBlockAllocator(BlockAllocator):
233 # We have fork case where block would get more than one ref,
234 # so we cannot free it from tracking if ref cnt large than 1
235 if refcount <= 1:
236 + assert block.block_id is not None
237 del self._blocks[block.block_id]
238 return self._hashless_allocator.free(block)
239
@@ -233,6 +242,7 @@ class PrefixCachingBlockAllocator(BlockAllocator):
242 # If no longer used, add the block to the evictor.
243 if refcount == 0:
244 assert block.content_hash in self._cached_blocks
245 + assert block.block_id is not None
246 del self._blocks[block.block_id]
247 self.evictor.add(block.block_id, block.content_hash,
248 block.num_tokens_total, block.last_accessed)
@@ -268,18 +278,18 @@ class PrefixCachingBlockAllocator(BlockAllocator):
278
279 return forked_blocks
280
271 - def get_num_free_blocks(self) -> int:
281 + def get_num_free_blocks(self, device: Optional[Device] = None) -> int:
282 + assert device is None
283 # The number of free blocks is the number of hashless free blocks
284 # plus the number of blocks evictor could free from its list.
285 return self._hashless_allocator.get_num_free_blocks(
286 ) + self.evictor.num_blocks
287
288 @property
278 - def all_block_ids(self) -> frozenset[int]:
289 + def all_block_ids(self) -> FrozenSet[int]:
290 return self._hashless_allocator.all_block_ids
291
281 - def promote_to_immutable_block(self,
282 - block: "PrefixCachingBlock") -> BlockId:
292 + def promote_to_immutable_block(self, block: Block) -> BlockId:
293 """Once a mutable block is full, it can be promoted to an immutable
294 block. This means that its content can be referenced by future blocks
295 having the same prefix.
@@ -289,7 +299,7 @@ class PrefixCachingBlockAllocator(BlockAllocator):
299 block.
300
301 Args:
292 - block (PrefixCachingBlock): The mutable block to be promoted.
302 + block: The mutable block to be promoted.
303
304 Returns:
305 BlockId: Either the original block index, or the block index of
@@ -385,8 +395,11 @@ class PrefixCachingBlockAllocator(BlockAllocator):
395 takewhile(lambda block_id: self.block_is_computed(block_id),
396 seq[:-1])) for seq in seq_block_ids
397 ]
388 - res = commonprefix([ids for ids in ids_list if ids != []])
389 - return res
398 + # It returns a list of int although type annotation says list of string.
399 + return commonprefix([
400 + ids for ids in ids_list # type: ignore
401 + if ids != []
402 + ])
403
404
405 class PrefixCachingBlock(Block):
@@ -403,7 +416,7 @@ class PrefixCachingBlock(Block):
416 token_ids (List[int]): The initial token IDs to be stored in the block.
417 block_size (int): The maximum number of token IDs that can be stored in
418 the block.
406 - prefix_caching_allocator (PrefixCachingBlockAllocator): The prefix
419 + prefix_caching_allocator (BlockAllocator): The prefix
420 caching block allocator associated with this block.
421 block_id (Optional[int], optional): The physical block index
422 of this block. Defaults to None.
@@ -411,21 +424,25 @@ class PrefixCachingBlock(Block):
424
425 def __init__(
426 self,
414 - prev_block: Optional["PrefixCachingBlock"],
427 + prev_block: Optional[Block],
428 token_ids: List[int],
429 block_size: int,
417 - prefix_caching_allocator: PrefixCachingBlockAllocator,
430 + prefix_caching_allocator: BlockAllocator,
431 block_id: Optional[int] = None,
419 - computed: Optional[bool] = False,
432 + computed: bool = False,
433 ):
434 + assert isinstance(prefix_caching_allocator,
435 + PrefixCachingBlockAllocator), (
436 + "Currently this class is only tested with "
437 + "PrefixCachingBlockAllocator.")
438 assert_prefix_caching_block_or_none(prev_block)
439
440 self._prev_block = prev_block
441 self._cached_content_hash: Optional[int] = None
442 self._cached_num_tokens_total: Optional[int] = None
443 self._prefix_caching_allocator = prefix_caching_allocator
427 - self.last_accessed = _DEFAULT_LAST_ACCESSED_TIME
428 - self.computed = computed
444 + self._last_accessed: float = _DEFAULT_LAST_ACCESSED_TIME
445 + self._computed = computed
446
447 self._block = NaiveBlock(
448 prev_block=prev_block,
@@ -436,6 +453,22 @@ class PrefixCachingBlock(Block):
453 _cow_target=self,
454 )
455
456 + @property
457 + def computed(self) -> bool:
458 + return self._computed
459 +
460 + @computed.setter
461 + def computed(self, value) -> None:
462 + self._computed = value
463 +
464 + @property
465 + def last_accessed(self) -> float:
466 + return self._last_accessed
467 +
468 + @last_accessed.setter
469 + def last_accessed(self, last_accessed_ts: float):
470 + self._last_accessed = last_accessed_ts
471 +
472 def append_token_ids(self, token_ids: List[int]) -> None:
473 """Appends the given token IDs to the block and registers the block as
474 immutable if the block becomes full.
@@ -483,7 +516,7 @@ class PrefixCachingBlock(Block):
516 if self._cached_num_tokens_total is not None:
517 return self._cached_num_tokens_total
518
486 - _block = self
519 + _block: Optional[Block] = self
520 self._cached_num_tokens_total = 0
521
522 # TODO: current implement here take O(N^2), we expect future
@@ -524,8 +557,10 @@ class PrefixCachingBlock(Block):
557 return None
558
559 is_first_block = self._prev_block is None
527 - prev_block_hash = (None if is_first_block else
528 - self._prev_block.content_hash)
560 + prev_block_hash = (
561 + None if is_first_block else
562 + self._prev_block.content_hash # type: ignore
563 + )
564
565 # Previous block exists but does not yet have a hash.
566 # Return no hash in this case.
vllm/core/block_manager_v2.py
+6 -3
@@ -190,7 +190,7 @@ class BlockSpaceManagerV2(BlockSpaceManager):
190 assert seq.seq_id in self.block_tables
191 block_ids = self.block_tables[seq.seq_id].physical_block_ids
192 assert all(b is not None for b in block_ids)
193 - return block_ids
193 + return block_ids # type: ignore
194
195 def access_all_blocks_in_seq(self, seq: Sequence, now: float):
196 # Update the last accessed time of all the blocks accessed
@@ -204,7 +204,9 @@ class BlockSpaceManagerV2(BlockSpaceManager):
204 block_ids = []
205 for block_id in block_table.physical_block_ids:
206 block_ids.append(block_id)
207 - self.block_allocator.mark_blocks_as_accessed(block_ids, now)
207 + self.block_allocator.mark_blocks_as_accessed(
208 + block_ids, # type: ignore
209 + now)
210
211 def mark_blocks_as_computed(self, seq_group: SequenceGroup):
212 # The only need for mark block as computed is for prefix caching,
@@ -227,8 +229,9 @@ class BlockSpaceManagerV2(BlockSpaceManager):
229 seq_block_ids = [
230 self.block_tables[seq.seq_id].physical_block_ids for seq in seqs
231 ]
232 + # NOTE(sang): This assumes seq_block_ids doesn't contain any None.
233 return self.block_allocator.get_common_computed_block_ids(
231 - seq_block_ids)
234 + seq_block_ids) # type: ignore
235
236 def fork(self, parent_seq: Sequence, child_seq: Sequence) -> None:
237 src_block_table = self.block_tables[parent_seq.seq_id]
vllm/core/evictor_v2.py
+10 -5
@@ -32,15 +32,20 @@ class Evictor(ABC):
32
33 @abstractmethod
34 def add(self, block_id: int, content_hash: int, num_hashed_tokens: int,
35 - last_accessed: int):
35 + last_accessed: float):
36 """Adds block to the evictor, making it a candidate for eviction"""
37 pass
38
39 @abstractmethod
40 - def update(self, block_id: int, last_accessed: int):
40 + def update(self, block_id: int, last_accessed: float):
41 """Update corresponding block's access time in metadata"""
42 pass
43
44 + @abstractmethod
45 + def remove(self, block_id: int):
46 + """Remove a given block id from the cache."""
47 + pass
48 +
49 @abstractproperty
50 def num_blocks(self) -> int:
51 pass
@@ -55,7 +60,7 @@ class BlockMetaData():
60 """
61
62 def __init__(self, content_hash: int, num_hashed_tokens: int,
58 - last_accessed: int):
63 + last_accessed: float):
64 self.content_hash = content_hash
65 self.num_hashed_tokens = num_hashed_tokens
66 self.last_accessed = last_accessed
@@ -96,12 +101,12 @@ class LRUEvictor(Evictor):
101 return evicted_block_id, evicted_block.content_hash
102
103 def add(self, block_id: int, content_hash: int, num_hashed_tokens: int,
99 - last_accessed: int):
104 + last_accessed: float):
105 self.free_table[block_id] = BlockMetaData(content_hash,
106 num_hashed_tokens,
107 last_accessed)
108
104 - def update(self, block_id: int, last_accessed: int):
109 + def update(self, block_id: int, last_accessed: float):
110 self.free_table[block_id].last_accessed = last_accessed
111
112 def remove(self, block_id: int):