@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6ed068a71a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Use the type BlockTable (#1791)

explainerauthors committed Nov 28, 2023 at 16:34 UTC 6ed068a71a58110f41c9cba76035f4c086840eb1
1 file changed +5 -5
vllm/core/block_manager.py
+5 -5
@@ -7,6 +7,10 @@ from vllm.sequence import Sequence, SequenceGroup, SequenceStatus
7 from vllm.utils import Device
8
9
10 +# Mapping: logical block number -> physical block.
11 +BlockTable = List[PhysicalTokenBlock]
12 +
13 +
14 class BlockAllocator:
15 """Manages free physical token blocks for a device.
16
@@ -26,7 +30,7 @@ class BlockAllocator:
30 self.num_blocks = num_blocks
31
32 # Initialize the free blocks.
29 - self.free_blocks: List[PhysicalTokenBlock] = []
33 + self.free_blocks: BlockTable = []
34 for i in range(num_blocks):
35 block = PhysicalTokenBlock(device=device,
36 block_number=i,
@@ -51,10 +55,6 @@ class BlockAllocator:
55 return len(self.free_blocks)
56
57
54 -# Mapping: logical block number -> physical block.
55 -BlockTable = List[PhysicalTokenBlock]
56 -
57 -
58 class AllocStatus(enum.Enum):
59 """Result for BlockSpaceManager.can_allocate
60