A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Use the type BlockTable (#1791)
explainerauthors committed
Nov 28, 2023 at 16:34 UTC
6ed068a71a58110f41c9cba76035f4c086840eb1
1 file changed
+5
-5
vllm/core/block_manager.py
+5
-5
@@ -7,6 +7,10 @@ from vllm.sequence import Sequence, SequenceGroup, SequenceStatus
7
from vllm.utils import Device
8
9
10
+# Mapping: logical block number -> physical block.
11
+BlockTable = List[PhysicalTokenBlock]
12
+
13
+
14
class BlockAllocator:
15
"""Manages free physical token blocks for a device.
16
@@ -26,7 +30,7 @@ class BlockAllocator:
30
self.num_blocks = num_blocks
31
32
# Initialize the free blocks.
29
- self.free_blocks: List[PhysicalTokenBlock] = []
33
+ self.free_blocks: BlockTable = []
34
for i in range(num_blocks):
35
block = PhysicalTokenBlock(device=device,
36
block_number=i,
@@ -51,10 +55,6 @@ class BlockAllocator:
55
return len(self.free_blocks)
56
57
54
-# Mapping: logical block number -> physical block.
55
-BlockTable = List[PhysicalTokenBlock]
56
-
57
-
58
class AllocStatus(enum.Enum):
59
"""Result for BlockSpaceManager.can_allocate
60