A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Add sequence
Woosuk Kwon committed
Feb 9, 2023 at 11:26 UTC
d904350a2c9a9fb2e476b45a486cc72fa6c2bd8f
1 file changed
+78
cacheflow/sequence.py
new
+78
@@ -0,0 +1,78 @@
1
+import enum
2
+from typing import List, Optional
3
+
4
+from cacheflow.block import LogicalTokenBlock
5
+from cacheflow.decoding import DecodingParams
6
+
7
+
8
+class SequenceStatus(enum.Enum):
9
+ PENDING = enum.auto()
10
+ SERVING = enum.auto()
11
+ SWAPPED = enum.auto()
12
+ FINISHED = enum.auto()
13
+
14
+
15
+class Sequence:
16
+
17
+ def __init__(
18
+ self,
19
+ seq_id: int,
20
+ token_ids: List[int],
21
+ max_context_len: int,
22
+ block_size: int,
23
+ ) -> None:
24
+ self.seq_id = seq_id
25
+ self.max_context_len = max_context_len
26
+ self.block_size = block_size
27
+
28
+ self.logical_token_blocks: List[LogicalTokenBlock] = []
29
+ # Initialize the logical token blocks with the given token ids.
30
+ self.append(token_ids)
31
+
32
+ self.status = SequenceStatus.PENDING
33
+
34
+ def add_block(self) -> None:
35
+ block = LogicalTokenBlock(
36
+ block_number=len(self.logical_token_blocks),
37
+ block_size=self.block_size,
38
+ )
39
+ self.logical_token_blocks.append(block)
40
+
41
+ def append(self, token_ids: List[int]) -> None:
42
+ while token_ids:
43
+ if not self.logical_token_blocks:
44
+ self.add_block()
45
+
46
+ last_block = self.logical_token_blocks[-1]
47
+ if last_block.is_full():
48
+ self.add_block()
49
+ last_block = self.logical_token_blocks[-1]
50
+
51
+ num_empty_slots = last_block.get_num_empty_slots()
52
+ last_block.append(token_ids[:num_empty_slots])
53
+ token_ids = token_ids[num_empty_slots:]
54
+
55
+ def get_token_ids(self) -> List[int]:
56
+ token_ids: List[int] = []
57
+ for block in self.logical_token_blocks:
58
+ token_ids.extend(block.get_token_ids())
59
+ return token_ids
60
+
61
+
62
+class SequenceGroup:
63
+
64
+ def __init__(
65
+ self,
66
+ group_id: int,
67
+ seqs: List[Sequence],
68
+ decoding_params: DecodingParams,
69
+ ) -> None:
70
+ self.group_id = group_id
71
+ self.seqs = seqs
72
+ self.decoding_params = decoding_params
73
+
74
+ def num_seqs(self, status: Optional[SequenceStatus] = None) -> int:
75
+ if status is None:
76
+ return len(self.seqs)
77
+ else:
78
+ return len([seq for seq in self.seqs if seq.status == status])