@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d904350a2c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Add sequence

Woosuk Kwon committed Feb 9, 2023 at 11:26 UTC d904350a2c9a9fb2e476b45a486cc72fa6c2bd8f
1 file changed +78
cacheflow/sequence.py new
+78
@@ -0,0 +1,78 @@
1 +import enum
2 +from typing import List, Optional
3 +
4 +from cacheflow.block import LogicalTokenBlock
5 +from cacheflow.decoding import DecodingParams
6 +
7 +
8 +class SequenceStatus(enum.Enum):
9 + PENDING = enum.auto()
10 + SERVING = enum.auto()
11 + SWAPPED = enum.auto()
12 + FINISHED = enum.auto()
13 +
14 +
15 +class Sequence:
16 +
17 + def __init__(
18 + self,
19 + seq_id: int,
20 + token_ids: List[int],
21 + max_context_len: int,
22 + block_size: int,
23 + ) -> None:
24 + self.seq_id = seq_id
25 + self.max_context_len = max_context_len
26 + self.block_size = block_size
27 +
28 + self.logical_token_blocks: List[LogicalTokenBlock] = []
29 + # Initialize the logical token blocks with the given token ids.
30 + self.append(token_ids)
31 +
32 + self.status = SequenceStatus.PENDING
33 +
34 + def add_block(self) -> None:
35 + block = LogicalTokenBlock(
36 + block_number=len(self.logical_token_blocks),
37 + block_size=self.block_size,
38 + )
39 + self.logical_token_blocks.append(block)
40 +
41 + def append(self, token_ids: List[int]) -> None:
42 + while token_ids:
43 + if not self.logical_token_blocks:
44 + self.add_block()
45 +
46 + last_block = self.logical_token_blocks[-1]
47 + if last_block.is_full():
48 + self.add_block()
49 + last_block = self.logical_token_blocks[-1]
50 +
51 + num_empty_slots = last_block.get_num_empty_slots()
52 + last_block.append(token_ids[:num_empty_slots])
53 + token_ids = token_ids[num_empty_slots:]
54 +
55 + def get_token_ids(self) -> List[int]:
56 + token_ids: List[int] = []
57 + for block in self.logical_token_blocks:
58 + token_ids.extend(block.get_token_ids())
59 + return token_ids
60 +
61 +
62 +class SequenceGroup:
63 +
64 + def __init__(
65 + self,
66 + group_id: int,
67 + seqs: List[Sequence],
68 + decoding_params: DecodingParams,
69 + ) -> None:
70 + self.group_id = group_id
71 + self.seqs = seqs
72 + self.decoding_params = decoding_params
73 +
74 + def num_seqs(self, status: Optional[SequenceStatus] = None) -> int:
75 + if status is None:
76 + return len(self.seqs)
77 + else:
78 + return len([seq for seq in self.seqs if seq.status == status])