A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Add blocks
Woosuk Kwon committed
Feb 9, 2023 at 11:26 UTC
6680129ba2d9c779050a18a97aeb57930a0b2f35
1 file changed
+51
cacheflow/block.py
new
+51
@@ -0,0 +1,51 @@
1
+from typing import List
2
+
3
+from cacheflow.utils import Device
4
+
5
+BLANK_TOKEN_ID = -1
6
+
7
+
8
+class LogicalTokenBlock:
9
+
10
+ def __int__(
11
+ self,
12
+ block_number: int,
13
+ block_size: int,
14
+ ) -> None:
15
+ self.block_number = block_number
16
+ self.block_size = block_size
17
+
18
+ self.token_ids = [BLANK_TOKEN_ID] * block_size
19
+ self.num_tokens = 0
20
+
21
+ def is_empty(self) -> bool:
22
+ return self.num_tokens == 0
23
+
24
+ def get_num_empty_slots(self) -> int:
25
+ return self.block_size - self.num_tokens
26
+
27
+ def is_full(self) -> bool:
28
+ return self.num_tokens == self.block_size
29
+
30
+ def append(self, token_ids: List[int]) -> None:
31
+ assert len(token_ids) <= self.get_num_empty_slots()
32
+ self.token_ids[self.num_tokens:self.num_tokens + len(token_ids)] = token_ids
33
+ self.num_tokens += len(token_ids)
34
+
35
+ def get_token_ids(self) -> List[int]:
36
+ return self.token_ids[:self.num_tokens]
37
+
38
+
39
+class PhysicalTokenBlock:
40
+
41
+ def __init__(
42
+ self,
43
+ device: Device,
44
+ block_number: int,
45
+ block_size: int,
46
+ ) -> None:
47
+ self.device = device
48
+ self.block_number = block_number
49
+ self.block_size = block_size
50
+
51
+ self.ref_count = 0