@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6680129ba2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Add blocks

Woosuk Kwon committed Feb 9, 2023 at 11:26 UTC 6680129ba2d9c779050a18a97aeb57930a0b2f35
1 file changed +51
cacheflow/block.py new
+51
@@ -0,0 +1,51 @@
1 +from typing import List
2 +
3 +from cacheflow.utils import Device
4 +
5 +BLANK_TOKEN_ID = -1
6 +
7 +
8 +class LogicalTokenBlock:
9 +
10 + def __int__(
11 + self,
12 + block_number: int,
13 + block_size: int,
14 + ) -> None:
15 + self.block_number = block_number
16 + self.block_size = block_size
17 +
18 + self.token_ids = [BLANK_TOKEN_ID] * block_size
19 + self.num_tokens = 0
20 +
21 + def is_empty(self) -> bool:
22 + return self.num_tokens == 0
23 +
24 + def get_num_empty_slots(self) -> int:
25 + return self.block_size - self.num_tokens
26 +
27 + def is_full(self) -> bool:
28 + return self.num_tokens == self.block_size
29 +
30 + def append(self, token_ids: List[int]) -> None:
31 + assert len(token_ids) <= self.get_num_empty_slots()
32 + self.token_ids[self.num_tokens:self.num_tokens + len(token_ids)] = token_ids
33 + self.num_tokens += len(token_ids)
34 +
35 + def get_token_ids(self) -> List[int]:
36 + return self.token_ids[:self.num_tokens]
37 +
38 +
39 +class PhysicalTokenBlock:
40 +
41 + def __init__(
42 + self,
43 + device: Device,
44 + block_number: int,
45 + block_size: int,
46 + ) -> None:
47 + self.device = device
48 + self.block_number = block_number
49 + self.block_size = block_size
50 +
51 + self.ref_count = 0