A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[mypy][6/N] Fix all the core subdirectory typing (#4450)
Co-authored-by: Cade Daniel <edacih@gmail.com>
SangBin Cho committed
May 2, 2024 at 12:01 UTC
cf8cac8c701079a3fda068ffd1cd6f72a490aa6d
10 files changed
+275
-83
.github/workflows/mypy.yaml
+2
-4
@@ -33,6 +33,7 @@ jobs:
33
- name: Mypy
34
run: |
35
mypy vllm/attention --config-file pyproject.toml
36
+ mypy vllm/core --config-file pyproject.toml
37
mypy vllm/distributed --config-file pyproject.toml
38
mypy vllm/entrypoints --config-file pyproject.toml
39
mypy vllm/executor --config-file pyproject.toml
@@ -42,9 +43,6 @@ jobs:
43
mypy vllm/engine --config-file pyproject.toml
44
mypy vllm/worker --config-file pyproject.toml
45
mypy vllm/spec_decode --config-file pyproject.toml
45
- mypy vllm/lora --config-file pyproject.toml
46
mypy vllm/model_executor --config-file pyproject.toml
47
-
48
- # TODO(sang): Fix nested dir
49
- mypy vllm/core/*.py --follow-imports=skip --config-file pyproject.toml
47
+ mypy vllm/lora --config-file pyproject.toml
48
format.sh
+1
-1
@@ -95,7 +95,7 @@ echo 'vLLM yapf: Done'
95
# Run mypy
96
echo 'vLLM mypy:'
97
mypy vllm/attention --config-file pyproject.toml
98
-mypy vllm/core/*.py --follow-imports=skip --config-file pyproject.toml
98
+mypy vllm/core --config-file pyproject.toml
99
mypy vllm/distributed --config-file pyproject.toml
100
mypy vllm/entrypoints --config-file pyproject.toml
101
mypy vllm/executor --config-file pyproject.toml
vllm/core/block/block_table.py
+10
-6
@@ -40,7 +40,9 @@ class BlockTable:
40
):
41
self._block_size = block_size
42
self._allocator = block_allocator
43
- self._blocks: Optional[List[Block]] = _blocks
43
+ if _blocks is None:
44
+ _blocks = []
45
+ self._blocks: List[Block] = _blocks
46
47
# Use helper method instead of directly calculating, as blocks
48
# may not be allocated.
@@ -104,7 +106,7 @@ class BlockTable:
106
token_ids (List[int]): The sequence of token IDs to be appended.
107
"""
108
assert self._is_allocated
107
- assert self._blocks is not None
109
+ assert len(self._blocks) > 0
110
111
self.ensure_num_empty_slots(num_empty_slots=len(token_ids) +
112
num_lookahead_slots)
@@ -141,6 +143,7 @@ class BlockTable:
143
blocks_to_allocate = cdiv(slots_to_allocate, self._block_size)
144
145
for _ in range(blocks_to_allocate):
146
+ assert len(self._blocks) > 0
147
self._blocks.append(
148
self._allocator.allocate_mutable(prev_block=self._blocks[-1],
149
device=device))
@@ -159,6 +162,7 @@ class BlockTable:
162
the current instance.
163
"""
164
assert self._is_allocated
165
+ assert len(self._blocks) > 0
166
forked_blocks = self._allocator.fork(self._blocks[-1])
167
return BlockTable(
168
block_size=self._block_size,
@@ -177,10 +181,10 @@ class BlockTable:
181
assert self._is_allocated
182
for block in self._blocks:
183
self._allocator.free(block)
180
- self._blocks = None
184
+ self._blocks = []
185
186
@property
183
- def physical_block_ids(self) -> List[int]:
187
+ def physical_block_ids(self) -> List[Optional[int]]:
188
"""Returns a list of physical block indices for the blocks in the
189
BlockTable.
190
@@ -235,7 +239,7 @@ class BlockTable:
239
240
def _get_all_token_ids(self) -> List[int]:
241
# NOTE: This function is O(seq_len); use sparingly.
238
- token_ids = []
242
+ token_ids: List[int] = []
243
244
if not self._is_allocated:
245
return token_ids
@@ -247,7 +251,7 @@ class BlockTable:
251
252
@property
253
def _is_allocated(self) -> bool:
250
- return self._blocks is not None
254
+ return len(self._blocks) > 0
255
256
@property
257
def _num_empty_slots(self) -> int:
vllm/core/block/common.py
+16
-4
@@ -1,5 +1,5 @@
1
from collections import defaultdict
2
-from typing import Dict, Iterable, List, Optional
2
+from typing import Dict, Iterable, List, Optional, Protocol
3
4
from vllm.core.block.interfaces import Block, BlockAllocator
5
@@ -7,7 +7,19 @@ BlockId = int
7
RefCount = int
8
9
10
-class RefCounter:
10
+class RefCounterProtocol(Protocol):
11
+
12
+ def incr(self, block_id: BlockId) -> RefCount:
13
+ raise NotImplementedError
14
+
15
+ def decr(self, block_id: BlockId) -> RefCount:
16
+ raise NotImplementedError
17
+
18
+ def get(self, block_id: BlockId) -> RefCount:
19
+ raise NotImplementedError
20
+
21
+
22
+class RefCounter(RefCounterProtocol):
23
"""A class for managing reference counts for a set of block indices.
24
25
The RefCounter class maintains a dictionary that maps block indices to their
@@ -54,7 +66,7 @@ class RefCounter:
66
return ReadOnlyRefCounter(self)
67
68
57
-class ReadOnlyRefCounter:
69
+class ReadOnlyRefCounter(RefCounterProtocol):
70
"""A read-only view of the RefCounter class.
71
72
The ReadOnlyRefCounter class provides a read-only interface to access the
@@ -96,7 +108,7 @@ class CopyOnWriteTracker:
108
109
def __init__(
110
self,
99
- refcounter: RefCounter,
111
+ refcounter: RefCounterProtocol,
112
allocator: BlockAllocator,
113
):
114
self._copy_on_writes: Dict[BlockId, List[BlockId]] = defaultdict(list)
vllm/core/block/cpu_gpu_block_allocator.py
+33
-16
@@ -1,6 +1,6 @@
1
-from typing import Dict, List, Optional
1
+from typing import Dict, FrozenSet, List, Optional
2
3
-from vllm.core.block.interfaces import (Block, BlockAllocator,
3
+from vllm.core.block.interfaces import (Block, BlockAllocator, BlockId,
4
DeviceAwareBlockAllocator)
5
from vllm.core.block.naive_block import NaiveBlock, NaiveBlockAllocator
6
from vllm.core.block.prefix_caching_block import PrefixCachingBlockAllocator
@@ -57,15 +57,15 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
57
cpu_block_ids = block_ids[num_gpu_blocks:]
58
59
if allocator_type == "naive":
60
- gpu_allocator = NaiveBlockAllocator(
61
- create_block=NaiveBlock,
60
+ gpu_allocator: BlockAllocator = NaiveBlockAllocator(
61
+ create_block=NaiveBlock, # type: ignore
62
num_blocks=num_gpu_blocks,
63
block_size=block_size,
64
block_ids=gpu_block_ids,
65
)
66
67
- cpu_allocator = NaiveBlockAllocator(
68
- create_block=NaiveBlock,
67
+ cpu_allocator: BlockAllocator = NaiveBlockAllocator(
68
+ create_block=NaiveBlock, # type: ignore
69
num_blocks=num_cpu_blocks,
70
block_size=block_size,
71
block_ids=cpu_block_ids,
@@ -105,13 +105,14 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
105
Device.GPU: gpu_block_allocator,
106
}
107
108
- self._block_ids_to_allocator = {}
108
+ self._block_ids_to_allocator: Dict[int, BlockAllocator] = {}
109
for _, allocator in self._allocators.items():
110
for block_id in allocator.all_block_ids:
111
self._block_ids_to_allocator[block_id] = allocator
112
113
- def allocate_mutable(self, prev_block: Optional[Block],
114
- device: Device) -> Block:
113
+ def allocate_mutable(self,
114
+ prev_block: Optional[Block],
115
+ device: Optional[Device] = None) -> Block:
116
"""Allocates a new mutable block on the specified device.
117
118
Args:
@@ -122,10 +123,13 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
123
Returns:
124
Block: The newly allocated mutable block.
125
"""
126
+ assert device is not None
127
return self._allocators[device].allocate_mutable(prev_block)
128
127
- def allocate_immutable(self, prev_block: Optional[Block],
128
- token_ids: List[int], device: Device) -> Block:
129
+ def allocate_immutable(self,
130
+ prev_block: Optional[Block],
131
+ token_ids: List[int],
132
+ device: Optional[Device] = None) -> Block:
133
"""Allocates a new immutable block with the provided token IDs on the
134
specified device.
135
@@ -140,6 +144,7 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
144
Block: The newly allocated immutable block containing the provided
145
token IDs.
146
"""
147
+ assert device is not None
148
return self._allocators[device].allocate_immutable(
149
prev_block, token_ids)
150
@@ -149,7 +154,9 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
154
Args:
155
block (Block): The block to be freed.
156
"""
152
- allocator = self._block_ids_to_allocator[block.block_id]
157
+ block_id = block.block_id
158
+ assert block_id is not None
159
+ allocator = self._block_ids_to_allocator[block_id]
160
return allocator.free(block)
161
162
def fork(self, last_block: Block) -> List[Block]:
@@ -163,19 +170,22 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
170
List[Block]: A new list of blocks that shares the same memory as the
171
original sequence.
172
"""
166
- allocator = self._block_ids_to_allocator[last_block.block_id]
173
+ block_id = last_block.block_id
174
+ assert block_id is not None
175
+ allocator = self._block_ids_to_allocator[block_id]
176
return allocator.fork(last_block)
177
169
- def get_num_free_blocks(self, device: Device) -> int:
178
+ def get_num_free_blocks(self, device: Optional[Device] = None) -> int:
179
"""Returns the number of free blocks available on the specified device.
180
181
Args:
182
device (Device): The device for which to query the number of free
174
- blocks.
183
+ blocks. AssertionError is raised if None is passed.
184
185
Returns:
186
int: The number of free blocks available on the specified device.
187
"""
188
+ assert device is not None
189
return self._allocators[device].get_num_free_blocks()
190
191
def clear_copy_on_writes(self) -> Dict[int, List[int]]:
@@ -210,5 +220,12 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
220
return self._allocators[device].get_common_computed_block_ids(
221
seq_block_ids)
222
213
- def all_block_ids(self) -> frozenset[int]:
223
+ @property
224
+ def all_block_ids(self) -> FrozenSet[int]:
225
return frozenset(self._block_ids_to_allocator.keys())
226
+
227
+ def promote_to_immutable_block(self, block: Block) -> BlockId:
228
+ raise NotImplementedError
229
+
230
+ def cow_block_if_not_appendable(self, block: Block) -> Optional[BlockId]:
231
+ raise NotImplementedError
vllm/core/block/interfaces.py
+95
-9
@@ -3,6 +3,8 @@ from typing import Dict, FrozenSet, List, Optional, Protocol
3
4
from vllm.utils import Device
5
6
+BlockId = int
7
+
8
9
class Block(ABC):
10
@@ -15,6 +17,12 @@ class Block(ABC):
17
def block_id(self) -> Optional[int]:
18
pass
19
20
+ @block_id.setter
21
+ @abstractmethod
22
+ def block_id(self, value: Optional[int]) -> None:
23
+ """NOTE: Do not use this API outside Block."""
24
+ self._block_id = value
25
+
26
@property
27
@abstractmethod
28
def token_ids(self) -> List[int]:
@@ -35,6 +43,27 @@ class Block(ABC):
43
def prev_block(self) -> Optional["Block"]:
44
pass
45
46
+ @property
47
+ @abstractmethod
48
+ def computed(self) -> bool:
49
+ raise NotImplementedError
50
+
51
+ @computed.setter
52
+ @abstractmethod
53
+ def computed(self, value) -> bool:
54
+ """Should be only used by PrefixCacingAllocator"""
55
+ raise NotImplementedError
56
+
57
+ @property
58
+ @abstractmethod
59
+ def last_accessed(self) -> float:
60
+ raise NotImplementedError
61
+
62
+ @last_accessed.setter
63
+ @abstractmethod
64
+ def last_accessed(self, last_accessed_ts: float):
65
+ raise NotImplementedError
66
+
67
class Factory(Protocol):
68
69
@abstractmethod
@@ -48,6 +77,17 @@ class Block(ABC):
77
) -> "Block":
78
pass
79
80
+ @property
81
+ @abstractmethod
82
+ def content_hash(self) -> Optional[int]:
83
+ """Return the content-based hash of the current block, or None if it is
84
+ not yet defined or not supported.
85
+
86
+ For the content-based hash to be defined, the current block must be
87
+ full.
88
+ """
89
+ return None
90
+
91
92
class BlockAllocator(ABC):
93
@@ -57,7 +97,7 @@ class BlockAllocator(ABC):
97
98
@abstractmethod
99
def allocate_immutable(self, prev_block: Optional[Block],
60
- token_ids: List[int], device: Device) -> Block:
100
+ token_ids: List[int]) -> Block:
101
pass
102
103
@abstractmethod
@@ -69,7 +109,7 @@ class BlockAllocator(ABC):
109
pass
110
111
@abstractmethod
72
- def get_num_free_blocks(self, device: Device) -> int:
112
+ def get_num_free_blocks(self) -> int:
113
pass
114
115
@property
@@ -82,11 +122,12 @@ class BlockAllocator(ABC):
122
pass
123
124
@abstractmethod
85
- def mark_blocks_as_accessed(self) -> None:
125
+ def mark_blocks_as_accessed(self, block_ids: List[int],
126
+ now: float) -> None:
127
pass
128
129
@abstractmethod
89
- def mark_blocks_as_computed(self) -> None:
130
+ def mark_blocks_as_computed(self, block_ids: List[int]) -> None:
131
pass
132
133
@abstractmethod
@@ -94,21 +135,66 @@ class BlockAllocator(ABC):
135
self, seq_block_ids: List[List[int]]) -> List[int]:
136
pass
137
138
+ @abstractmethod
139
+ def cow_block_if_not_appendable(self, block: Block) -> Optional["BlockId"]:
140
+ """NOTE: This should not be used besides Block"""
141
+ pass
142
+
143
+ @abstractmethod
144
+ def promote_to_immutable_block(self, block: Block) -> BlockId:
145
+ """NOTE: This should not be used besides Block"""
146
+ pass
147
+
148
class NoFreeBlocksError(ValueError):
149
pass
150
151
101
-class DeviceAwareBlockAllocator(BlockAllocator):
152
+class DeviceAwareBlockAllocator(ABC):
153
154
@abstractmethod
104
- def allocate_mutable(self, prev_block: Optional[Block]) -> Block:
155
+ def allocate_mutable(self,
156
+ prev_block: Optional[Block],
157
+ device: Optional[Device] = None) -> Block:
158
pass
159
160
@abstractmethod
108
- def allocate_immutable(self, prev_block: Optional[Block],
109
- token_ids: List[int], device: Device) -> Block:
161
+ def allocate_immutable(self,
162
+ prev_block: Optional[Block],
163
+ token_ids: List[int],
164
+ device: Optional[Device] = None) -> Block:
165
pass
166
167
@abstractmethod
113
- def get_num_free_blocks(self, device: Device) -> int:
168
+ def get_num_free_blocks(self, device: Optional[Device] = None) -> int:
169
+ pass
170
+
171
+ @abstractmethod
172
+ def free(self, block: Block) -> None:
173
+ pass
174
+
175
+ @abstractmethod
176
+ def fork(self, last_block: Block) -> List[Block]:
177
+ pass
178
+
179
+ @property
180
+ @abstractmethod
181
+ def all_block_ids(self) -> FrozenSet[int]:
182
+ pass
183
+
184
+ @abstractmethod
185
+ def clear_copy_on_writes(self) -> Dict[int, List[int]]:
186
+ pass
187
+
188
+ @abstractmethod
189
+ def mark_blocks_as_accessed(self, block_ids: List[int],
190
+ now: float) -> None:
191
+ pass
192
+
193
+ @abstractmethod
194
+ def mark_blocks_as_computed(self, block_ids: List[int]) -> None:
195
+ pass
196
+
197
+ @abstractmethod
198
+ def get_common_computed_block_ids(
199
+ self, seq_block_ids: List[List[int]]) -> List[int]:
200
pass
vllm/core/block/naive_block.py
+42
-10
@@ -1,10 +1,9 @@
1
-from typing import Dict, Iterable, List, Optional, Set
1
+from typing import Dict, FrozenSet, Iterable, List, Optional, Set
2
3
from vllm.core.block.common import (CopyOnWriteTracker, RefCounter,
4
get_all_blocks_recursively)
5
-from vllm.core.block.interfaces import Block, BlockAllocator
5
+from vllm.core.block.interfaces import Block, BlockAllocator, BlockId, Device
6
7
-BlockId = int
7
Refcount = int
8
9
@@ -49,8 +48,10 @@ class NaiveBlockAllocator(BlockAllocator):
48
allocator=self,
49
)
50
52
- def allocate_immutable(self, prev_block: Optional[Block],
53
- token_ids: List[int]) -> Block:
51
+ def allocate_immutable(self,
52
+ prev_block: Optional[Block],
53
+ token_ids: List[int],
54
+ device: Optional[Device] = None) -> Block:
55
"""Allocates a new immutable block with the given token IDs, linked to
56
the previous block.
57
@@ -63,11 +64,14 @@ class NaiveBlockAllocator(BlockAllocator):
64
Returns:
65
Block: The newly allocated immutable block.
66
"""
67
+ assert device is None
68
block = self.allocate_mutable(prev_block=prev_block)
69
block.append_token_ids(token_ids)
70
return block
71
70
- def allocate_mutable(self, prev_block: Optional[Block]) -> Block:
72
+ def allocate_mutable(self,
73
+ prev_block: Optional[Block],
74
+ device: Optional[Device] = None) -> Block:
75
"""Allocates a new mutable block, linked to the previous block.
76
77
Args:
@@ -78,6 +82,7 @@ class NaiveBlockAllocator(BlockAllocator):
82
Returns:
83
Block: The newly allocated mutable block.
84
"""
85
+ assert device is None
86
block_id = self._allocate_new_block_id()
87
return self._create_block(
88
prev_block=prev_block,
@@ -88,6 +93,7 @@ class NaiveBlockAllocator(BlockAllocator):
93
)
94
95
def free(self, block: Block) -> None:
96
+ assert block.block_id is not None
97
self._free_block_id(block.block_id)
98
99
# Mark the block as having no allocation.
@@ -111,6 +117,7 @@ class NaiveBlockAllocator(BlockAllocator):
117
for block in source_blocks:
118
119
# Increment refcount for each block.
120
+ assert block.block_id is not None
121
refcount = self._refcounter.incr(block.block_id)
122
assert refcount != 1, "can't fork free'd block"
123
@@ -126,7 +133,8 @@ class NaiveBlockAllocator(BlockAllocator):
133
134
return forked_blocks
135
129
- def get_num_free_blocks(self) -> int:
136
+ def get_num_free_blocks(self, device: Optional[Device] = None) -> int:
137
+ assert device is None
138
return len(self._free_block_indices)
139
140
def _allocate_new_block_id(self) -> BlockId:
@@ -148,7 +156,7 @@ class NaiveBlockAllocator(BlockAllocator):
156
return self._refcounter
157
158
@property
151
- def all_block_ids(self):
159
+ def all_block_ids(self) -> FrozenSet[int]:
160
return self._all_block_indices
161
162
def cow_block_if_not_appendable(self, block: Block) -> Optional[BlockId]:
@@ -200,6 +208,9 @@ class NaiveBlockAllocator(BlockAllocator):
208
"""
209
return []
210
211
+ def promote_to_immutable_block(self, block: Block) -> BlockId:
212
+ raise NotImplementedError
213
+
214
215
class NaiveBlock(Block):
216
"""An implementation of the Block class that does not support prefix
@@ -224,13 +235,13 @@ class NaiveBlock(Block):
235
"""
236
237
def __init__(self,
227
- prev_block: Block,
238
+ prev_block: Optional[Block],
239
token_ids: List[int],
240
block_size: int,
241
allocator: BlockAllocator,
242
block_id: Optional[int] = None,
243
_cow_target: Optional[Block] = None):
233
- self._token_ids = []
244
+ self._token_ids: List[int] = []
245
self._block_size = block_size
246
self._prev_block = prev_block
247
self._block_id = block_id
@@ -256,6 +267,22 @@ class NaiveBlock(Block):
267
assert self.num_empty_slots >= len(token_ids)
268
self._token_ids.extend(token_ids)
269
270
+ @property
271
+ def computed(self) -> bool:
272
+ raise NotImplementedError
273
+
274
+ @computed.setter
275
+ def computed(self, value) -> None:
276
+ raise NotImplementedError
277
+
278
+ @property
279
+ def last_accessed(self) -> float:
280
+ raise NotImplementedError
281
+
282
+ @last_accessed.setter
283
+ def last_accessed(self, last_accessed_ts: float):
284
+ raise NotImplementedError
285
+
286
@property
287
def block_id(self) -> Optional[int]:
288
return self._block_id
@@ -276,9 +303,14 @@ class NaiveBlock(Block):
303
def token_ids(self) -> List[int]:
304
return self._token_ids
305
306
+ @property
307
def block_size(self) -> int:
308
return self._block_size
309
310
@property
311
def prev_block(self) -> Optional["Block"]:
312
return self._prev_block
313
+
314
+ @property
315
+ def content_hash(self) -> Optional[int]:
316
+ return None
vllm/core/block/prefix_caching_block.py
+60
-25
@@ -1,16 +1,15 @@
1
"""Token blocks."""
2
from itertools import takewhile
3
from os.path import commonprefix
4
-from typing import Dict, Iterable, List, Optional
4
+from typing import Dict, FrozenSet, Iterable, List, Optional
5
6
from vllm.core.block.common import (CopyOnWriteTracker,
7
get_all_blocks_recursively)
8
-from vllm.core.block.interfaces import Block, BlockAllocator
8
+from vllm.core.block.interfaces import Block, BlockAllocator, BlockId, Device
9
from vllm.core.block.naive_block import NaiveBlock, NaiveBlockAllocator
10
from vllm.core.evictor_v2 import EvictionPolicy, Evictor, make_evictor
11
12
PrefixHash = int
13
-BlockId = int
13
14
# By default, we init our block access time as _DEFAULT_LAST_ACCESSED_TIME
15
# so that if we find one block is still hold _DEFAULT_LAST_ACCESSED_TIME,
@@ -38,7 +37,7 @@ class PrefixCachingBlockAllocator(BlockAllocator):
37
num_blocks: int,
38
block_size: int,
39
block_ids: Optional[Iterable[int]] = None,
41
- eviction_policy: Optional[EvictionPolicy] = EvictionPolicy.LRU,
40
+ eviction_policy: EvictionPolicy = EvictionPolicy.LRU,
41
):
42
# A mapping of prefix hash to block index. All blocks which have a
43
# prefix hash will be in this dict, even if they have refcount 0.
@@ -49,7 +48,7 @@ class PrefixCachingBlockAllocator(BlockAllocator):
48
49
# An allocator for blocks that do not have prefix hashes.
50
self._hashless_allocator = NaiveBlockAllocator(
52
- create_block=self._create_block,
51
+ create_block=self._create_block, # type: ignore
52
num_blocks=num_blocks,
53
block_size=block_size,
54
block_ids=block_ids,
@@ -79,7 +78,7 @@ class PrefixCachingBlockAllocator(BlockAllocator):
78
block_size: int,
79
allocator: BlockAllocator,
80
block_id: Optional[int] = None,
82
- computed: Optional[bool] = False,
81
+ computed: bool = False,
82
) -> Block:
83
# Bind block to self.
84
allocator = self
@@ -93,8 +92,10 @@ class PrefixCachingBlockAllocator(BlockAllocator):
92
computed=computed,
93
)
94
96
- def allocate_immutable(self, prev_block: Optional[Block],
97
- token_ids: List[int]) -> Block:
95
+ def allocate_immutable(self,
96
+ prev_block: Optional[Block],
97
+ token_ids: List[int],
98
+ device: Optional[Device] = None) -> Block:
99
"""Allocates an immutable block with the given token IDs, reusing cached
100
blocks if possible.
101
@@ -105,6 +106,7 @@ class PrefixCachingBlockAllocator(BlockAllocator):
106
Returns:
107
Block: The allocated immutable block.
108
"""
109
+ assert device is None
110
assert_prefix_caching_block_or_none(prev_block)
111
112
block = self._create_block(
@@ -127,16 +129,20 @@ class PrefixCachingBlockAllocator(BlockAllocator):
129
130
return block
131
130
- def allocate_mutable(self, prev_block: Block) -> Block:
132
+ def allocate_mutable(self,
133
+ prev_block: Optional[Block],
134
+ device: Optional[Device] = None) -> Block:
135
"""Allocates a mutable block. If there are no free blocks, this will
136
evict unused cached blocks.
137
138
Args:
139
prev_block (Block): The previous block in the sequence.
140
+ None is not allowed unlike it is super class.
141
142
Returns:
143
Block: The allocated mutable block.
144
"""
145
+ assert device is None
146
assert_prefix_caching_block_or_none(prev_block)
147
148
try:
@@ -144,6 +150,7 @@ class PrefixCachingBlockAllocator(BlockAllocator):
150
prev_block=prev_block)
151
152
assert block.block_id not in self._blocks
153
+ assert block.block_id is not None
154
self._blocks[block.block_id] = block
155
return block
156
except BlockAllocator.NoFreeBlocksError:
@@ -183,6 +190,7 @@ class PrefixCachingBlockAllocator(BlockAllocator):
190
assert block.content_hash is None
191
192
assert block.block_id not in self._blocks
193
+ assert block.block_id is not None
194
self._blocks[block.block_id] = block
195
return block
196
@@ -225,6 +233,7 @@ class PrefixCachingBlockAllocator(BlockAllocator):
233
# We have fork case where block would get more than one ref,
234
# so we cannot free it from tracking if ref cnt large than 1
235
if refcount <= 1:
236
+ assert block.block_id is not None
237
del self._blocks[block.block_id]
238
return self._hashless_allocator.free(block)
239
@@ -233,6 +242,7 @@ class PrefixCachingBlockAllocator(BlockAllocator):
242
# If no longer used, add the block to the evictor.
243
if refcount == 0:
244
assert block.content_hash in self._cached_blocks
245
+ assert block.block_id is not None
246
del self._blocks[block.block_id]
247
self.evictor.add(block.block_id, block.content_hash,
248
block.num_tokens_total, block.last_accessed)
@@ -268,18 +278,18 @@ class PrefixCachingBlockAllocator(BlockAllocator):
278
279
return forked_blocks
280
271
- def get_num_free_blocks(self) -> int:
281
+ def get_num_free_blocks(self, device: Optional[Device] = None) -> int:
282
+ assert device is None
283
# The number of free blocks is the number of hashless free blocks
284
# plus the number of blocks evictor could free from its list.
285
return self._hashless_allocator.get_num_free_blocks(
286
) + self.evictor.num_blocks
287
288
@property
278
- def all_block_ids(self) -> frozenset[int]:
289
+ def all_block_ids(self) -> FrozenSet[int]:
290
return self._hashless_allocator.all_block_ids
291
281
- def promote_to_immutable_block(self,
282
- block: "PrefixCachingBlock") -> BlockId:
292
+ def promote_to_immutable_block(self, block: Block) -> BlockId:
293
"""Once a mutable block is full, it can be promoted to an immutable
294
block. This means that its content can be referenced by future blocks
295
having the same prefix.
@@ -289,7 +299,7 @@ class PrefixCachingBlockAllocator(BlockAllocator):
299
block.
300
301
Args:
292
- block (PrefixCachingBlock): The mutable block to be promoted.
302
+ block: The mutable block to be promoted.
303
304
Returns:
305
BlockId: Either the original block index, or the block index of
@@ -385,8 +395,11 @@ class PrefixCachingBlockAllocator(BlockAllocator):
395
takewhile(lambda block_id: self.block_is_computed(block_id),
396
seq[:-1])) for seq in seq_block_ids
397
]
388
- res = commonprefix([ids for ids in ids_list if ids != []])
389
- return res
398
+ # It returns a list of int although type annotation says list of string.
399
+ return commonprefix([
400
+ ids for ids in ids_list # type: ignore
401
+ if ids != []
402
+ ])
403
404
405
class PrefixCachingBlock(Block):
@@ -403,7 +416,7 @@ class PrefixCachingBlock(Block):
416
token_ids (List[int]): The initial token IDs to be stored in the block.
417
block_size (int): The maximum number of token IDs that can be stored in
418
the block.
406
- prefix_caching_allocator (PrefixCachingBlockAllocator): The prefix
419
+ prefix_caching_allocator (BlockAllocator): The prefix
420
caching block allocator associated with this block.
421
block_id (Optional[int], optional): The physical block index
422
of this block. Defaults to None.
@@ -411,21 +424,25 @@ class PrefixCachingBlock(Block):
424
425
def __init__(
426
self,
414
- prev_block: Optional["PrefixCachingBlock"],
427
+ prev_block: Optional[Block],
428
token_ids: List[int],
429
block_size: int,
417
- prefix_caching_allocator: PrefixCachingBlockAllocator,
430
+ prefix_caching_allocator: BlockAllocator,
431
block_id: Optional[int] = None,
419
- computed: Optional[bool] = False,
432
+ computed: bool = False,
433
):
434
+ assert isinstance(prefix_caching_allocator,
435
+ PrefixCachingBlockAllocator), (
436
+ "Currently this class is only tested with "
437
+ "PrefixCachingBlockAllocator.")
438
assert_prefix_caching_block_or_none(prev_block)
439
440
self._prev_block = prev_block
441
self._cached_content_hash: Optional[int] = None
442
self._cached_num_tokens_total: Optional[int] = None
443
self._prefix_caching_allocator = prefix_caching_allocator
427
- self.last_accessed = _DEFAULT_LAST_ACCESSED_TIME
428
- self.computed = computed
444
+ self._last_accessed: float = _DEFAULT_LAST_ACCESSED_TIME
445
+ self._computed = computed
446
447
self._block = NaiveBlock(
448
prev_block=prev_block,
@@ -436,6 +453,22 @@ class PrefixCachingBlock(Block):
453
_cow_target=self,
454
)
455
456
+ @property
457
+ def computed(self) -> bool:
458
+ return self._computed
459
+
460
+ @computed.setter
461
+ def computed(self, value) -> None:
462
+ self._computed = value
463
+
464
+ @property
465
+ def last_accessed(self) -> float:
466
+ return self._last_accessed
467
+
468
+ @last_accessed.setter
469
+ def last_accessed(self, last_accessed_ts: float):
470
+ self._last_accessed = last_accessed_ts
471
+
472
def append_token_ids(self, token_ids: List[int]) -> None:
473
"""Appends the given token IDs to the block and registers the block as
474
immutable if the block becomes full.
@@ -483,7 +516,7 @@ class PrefixCachingBlock(Block):
516
if self._cached_num_tokens_total is not None:
517
return self._cached_num_tokens_total
518
486
- _block = self
519
+ _block: Optional[Block] = self
520
self._cached_num_tokens_total = 0
521
522
# TODO: current implement here take O(N^2), we expect future
@@ -524,8 +557,10 @@ class PrefixCachingBlock(Block):
557
return None
558
559
is_first_block = self._prev_block is None
527
- prev_block_hash = (None if is_first_block else
528
- self._prev_block.content_hash)
560
+ prev_block_hash = (
561
+ None if is_first_block else
562
+ self._prev_block.content_hash # type: ignore
563
+ )
564
565
# Previous block exists but does not yet have a hash.
566
# Return no hash in this case.
vllm/core/block_manager_v2.py
+6
-3
@@ -190,7 +190,7 @@ class BlockSpaceManagerV2(BlockSpaceManager):
190
assert seq.seq_id in self.block_tables
191
block_ids = self.block_tables[seq.seq_id].physical_block_ids
192
assert all(b is not None for b in block_ids)
193
- return block_ids
193
+ return block_ids # type: ignore
194
195
def access_all_blocks_in_seq(self, seq: Sequence, now: float):
196
# Update the last accessed time of all the blocks accessed
@@ -204,7 +204,9 @@ class BlockSpaceManagerV2(BlockSpaceManager):
204
block_ids = []
205
for block_id in block_table.physical_block_ids:
206
block_ids.append(block_id)
207
- self.block_allocator.mark_blocks_as_accessed(block_ids, now)
207
+ self.block_allocator.mark_blocks_as_accessed(
208
+ block_ids, # type: ignore
209
+ now)
210
211
def mark_blocks_as_computed(self, seq_group: SequenceGroup):
212
# The only need for mark block as computed is for prefix caching,
@@ -227,8 +229,9 @@ class BlockSpaceManagerV2(BlockSpaceManager):
229
seq_block_ids = [
230
self.block_tables[seq.seq_id].physical_block_ids for seq in seqs
231
]
232
+ # NOTE(sang): This assumes seq_block_ids doesn't contain any None.
233
return self.block_allocator.get_common_computed_block_ids(
231
- seq_block_ids)
234
+ seq_block_ids) # type: ignore
235
236
def fork(self, parent_seq: Sequence, child_seq: Sequence) -> None:
237
src_block_table = self.block_tables[parent_seq.seq_id]
vllm/core/evictor_v2.py
+10
-5
@@ -32,15 +32,20 @@ class Evictor(ABC):
32
33
@abstractmethod
34
def add(self, block_id: int, content_hash: int, num_hashed_tokens: int,
35
- last_accessed: int):
35
+ last_accessed: float):
36
"""Adds block to the evictor, making it a candidate for eviction"""
37
pass
38
39
@abstractmethod
40
- def update(self, block_id: int, last_accessed: int):
40
+ def update(self, block_id: int, last_accessed: float):
41
"""Update corresponding block's access time in metadata"""
42
pass
43
44
+ @abstractmethod
45
+ def remove(self, block_id: int):
46
+ """Remove a given block id from the cache."""
47
+ pass
48
+
49
@abstractproperty
50
def num_blocks(self) -> int:
51
pass
@@ -55,7 +60,7 @@ class BlockMetaData():
60
"""
61
62
def __init__(self, content_hash: int, num_hashed_tokens: int,
58
- last_accessed: int):
63
+ last_accessed: float):
64
self.content_hash = content_hash
65
self.num_hashed_tokens = num_hashed_tokens
66
self.last_accessed = last_accessed
@@ -96,12 +101,12 @@ class LRUEvictor(Evictor):
101
return evicted_block_id, evicted_block.content_hash
102
103
def add(self, block_id: int, content_hash: int, num_hashed_tokens: int,
99
- last_accessed: int):
104
+ last_accessed: float):
105
self.free_table[block_id] = BlockMetaData(content_hash,
106
num_hashed_tokens,
107
last_accessed)
108
104
- def update(self, block_id: int, last_accessed: int):
109
+ def update(self, block_id: int, last_accessed: float):
110
self.free_table[block_id].last_accessed = last_accessed
111
112
def remove(self, block_id: int):