A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Add scheduler
Woosuk Kwon committed
Feb 13, 2023 at 02:39 UTC
5a309bb5887eeba8d47f07854c476870f3b40f03
1 file changed
+170
cacheflow/master/scheduler.py
new
+170
@@ -0,0 +1,170 @@
1
+from typing import Dict, List, Tuple
2
+
3
+from cacheflow.master.block_manager import BlockSpaceManager
4
+from cacheflow.sequence import Sequence
5
+from cacheflow.sequence import SequenceGroup
6
+from cacheflow.sequence import SequenceStatus
7
+
8
+
9
+class Scheduler:
10
+
11
+ def __int__(
12
+ self,
13
+ block_size: int,
14
+ num_gpu_blocks: int,
15
+ num_cpu_blocks: int,
16
+ ) -> None:
17
+ self.block_manager = BlockSpaceManager(
18
+ block_size=block_size,
19
+ num_gpu_blocks=num_gpu_blocks,
20
+ num_cpu_blocks=num_cpu_blocks,
21
+ )
22
+
23
+ # Serving sequence groups (FIFO).
24
+ self.serving: List[SequenceGroup] = []
25
+ # Mapping: group_id -> num_steps.
26
+ self.num_steps: Dict[int, int] = {}
27
+ # Mapping: group_id -> max_num_steps.
28
+ self.max_num_steps: Dict[int, int] = {}
29
+ # Mapping: group_id -> stop_token_ids.
30
+ self.stop_token_ids: Dict[int, List[int]] = {}
31
+
32
+ # Swapped sequence groups (LIFO).
33
+ self.swapped: List[SequenceGroup] = []
34
+
35
+ # Pending sequence groups (FIFO).
36
+ self.queue: List[SequenceGroup] = []
37
+
38
+ def _free_seq(self, seq: Sequence) -> None:
39
+ seq.status = SequenceStatus.FINISHED
40
+ self.block_manager.free(seq)
41
+
42
+ def _allocate(self, seq_group: SequenceGroup) -> None:
43
+ self.block_manager.allocate(seq_group)
44
+ for seq in seq_group.seqs:
45
+ seq.status = SequenceStatus.RUNNING
46
+ self.serving.append(seq_group)
47
+
48
+ def _append(self, seq_group: SequenceGroup) -> None:
49
+ for seq in seq_group.seqs:
50
+ if seq.status == SequenceStatus.FINISHED:
51
+ continue
52
+ ret = self.block_manager.append(seq)
53
+ if ret is not None:
54
+ src_block, dst_block = ret
55
+ # TODO: Issue COPY commands to the workers.
56
+
57
+ def _swap_in(self, seq_group: SequenceGroup) -> None:
58
+ # TODO: Issue SWAP_IN commands to the workers.
59
+ self.block_manager.swap_in(seq_group)
60
+ self.block_manager.append(seq_group)
61
+ for seq in seq_group.seqs:
62
+ if seq.status == SequenceStatus.SWAPPED:
63
+ seq.status = SequenceStatus.RUNNING
64
+ self.serving.append(seq_group)
65
+
66
+ def _swap_out(self, seq_group: SequenceGroup) -> None:
67
+ assert self.block_manager.can_swap_out(seq_group)
68
+ # TODO: Issue SWAP_OUT commands to the workers.
69
+ self.block_manager.swap_out(seq_group)
70
+ for seq in seq_group.seqs:
71
+ if seq.status == SequenceStatus.RUNNING:
72
+ seq.status = SequenceStatus.SWAPPED
73
+ self.swapped.append(seq_group)
74
+
75
+ def step(self) -> None:
76
+ # 1. Prepare new slots for the running sequences.
77
+ # NOTE: Here we implicitly assume FCFS scheduling.
78
+ # That is, the most recently added sequence group is the first
79
+ # to be swapped out.
80
+ victim_idx = len(self.serving) - 1
81
+ for i, seq_group in enumerate(self.serving):
82
+ if i > victim_idx:
83
+ # The i-th sequence group has already been swapped out.
84
+ break
85
+ # OOM. Swap out the victim sequence groups.
86
+ while not self.block_manager.can_append(seq_group):
87
+ victim_seq_group = self.serving[victim_idx]
88
+ self._swap_out(victim_seq_group)
89
+ victim_idx -= 1
90
+ if i > victim_idx:
91
+ # No other sequence groups can be swapped out.
92
+ break
93
+ else:
94
+ self._append(seq_group)
95
+ self.serving = self.serving[:victim_idx + 1]
96
+
97
+ # 2. Swap in the swapped sequences if possible.
98
+ # NOTE: Here we implicitly assume FCFS scheduling.
99
+ # The swapped sequences are in LIFO order.
100
+ for i, seq_group in enumerate(reversed(self.swapped)):
101
+ if self.block_manager.can_swap_in(seq_group):
102
+ self._swap_in(seq_group)
103
+ else:
104
+ # OOM. Stop swapping.
105
+ self.swapped = self.swapped[:len(self.swapped) - i]
106
+ break
107
+ else:
108
+ # All swapped sequences are swapped in.
109
+ self.swapped.clear()
110
+
111
+ # 3. Join new sequences if possible.
112
+ # NOTE: Here we implicitly assume FCFS scheduling.
113
+ # TODO(woosuk): Add a heuristic to control the maximum batch size.
114
+ if not self.swapped:
115
+ for i, seq_group in enumerate(self.queue):
116
+ if self.block_manager.can_allocate(seq_group):
117
+ self._allocate(seq_group)
118
+ else:
119
+ # FIXME: Consider the race condition.
120
+ self.queue = self.queue[i:]
121
+ break
122
+
123
+ def post_step(
124
+ self,
125
+ next_tokens: Dict[int, Tuple[int, int]],
126
+ ) -> None:
127
+ # Update the running sequences and free blocks.
128
+ for seq_group in self.serving:
129
+ group_id = seq_group.group_id
130
+ self.num_steps[group_id] += 1
131
+ stop_token_ids = self.stop_token_ids[group_id]
132
+
133
+ for seq in seq_group.seqs:
134
+ if seq.status == SequenceStatus.FINISHED:
135
+ continue
136
+
137
+ parent_seq_id, next_token = next_tokens[seq.seq_id]
138
+ if seq.seq_id != parent_seq_id:
139
+ # The sequence is a fork of the parent sequence (beam search).
140
+ # Free the current sequence.
141
+ self.block_manager.free(seq)
142
+ # Fork the parent sequence.
143
+ parent_seq = seq_group.find(parent_seq_id)
144
+ seq.logical_token_blocks = parent_seq.logical_token_blocks.copy()
145
+ self.block_manager.fork(parent_seq, seq)
146
+
147
+ # Append a new token to the sequence.
148
+ seq.append(next_token)
149
+
150
+ # Check if the sequence has generated a stop token.
151
+ if next_token in stop_token_ids:
152
+ self._free_seq(seq)
153
+ continue
154
+
155
+ # Check if the sequence has reached the maximum number of steps.
156
+ if self.num_steps[group_id] == self.max_num_steps[group_id]:
157
+ self._free_seq(seq)
158
+ continue
159
+
160
+ # Update the serving states.
161
+ serving: List[SequenceGroup] = []
162
+ for seq_group in self.serving:
163
+ if seq_group.num_seqs(status=SequenceStatus.RUNNING) == 0:
164
+ del self.num_steps[seq_group.group_id]
165
+ del self.max_num_steps[seq_group.group_id]
166
+ del self.stop_token_ids[seq_group.group_id]
167
+ # TODO: Return the seq_group to the client.
168
+ else:
169
+ serving.append(seq_group)
170
+ self.serving = serving