@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 331fa0b042
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Implement scheduler.step & Add a threshold for batch size

Woosuk Kwon committed Feb 23, 2023 at 07:54 UTC 331fa0b042b5cbcdadb03f25dc8592aba9c12b53
1 file changed +59 -21
cacheflow/master/scheduler.py
+59 -21
@@ -5,6 +5,8 @@ from cacheflow.sequence import Sequence
5 from cacheflow.sequence import SequenceGroup
6 from cacheflow.sequence import SequenceStatus
7
8 +_MAX_NUM_BATCHED_TOKENS = 2048
9 +
10
11 class Scheduler:
12
@@ -27,8 +29,8 @@ class Scheduler:
29 num_cpu_blocks=num_cpu_blocks,
30 )
31
30 - # Serving sequence groups (FIFO).
31 - self.serving: List[SequenceGroup] = []
32 + # Running sequence groups (FIFO).
33 + self.running: List[SequenceGroup] = []
34 # Mapping: group_id -> num_steps.
35 self.num_steps: Dict[int, int] = {}
36 # Mapping: group_id -> max_num_steps.
@@ -54,7 +56,7 @@ class Scheduler:
56 self.block_manager.allocate(seq_group)
57 for seq in seq_group.seqs:
58 seq.status = SequenceStatus.RUNNING
57 - self.serving.append(seq_group)
59 + self.running.append(seq_group)
60 # FIXME
61 self.num_steps[seq_group.group_id] = 0
62
@@ -73,7 +75,7 @@ class Scheduler:
75 for seq in seq_group.seqs:
76 if seq.status == SequenceStatus.SWAPPED:
77 seq.status = SequenceStatus.RUNNING
76 - self.serving.append(seq_group)
78 + self.running.append(seq_group)
79
80 def _swap_out(self, seq_group: SequenceGroup) -> None:
81 assert self.block_manager.can_swap_out(seq_group)
@@ -89,14 +91,14 @@ class Scheduler:
91 # NOTE: Here we implicitly assume FCFS scheduling.
92 # That is, the most recently added sequence group is the first
93 # to be swapped out.
92 - victim_idx = len(self.serving) - 1
93 - for i, seq_group in enumerate(self.serving):
94 + victim_idx = len(self.running) - 1
95 + for i, seq_group in enumerate(self.running):
96 if i > victim_idx:
97 # The i-th sequence group has already been swapped out.
98 break
99 # OOM. Swap out the victim sequence groups.
100 while not self.block_manager.can_append(seq_group):
99 - victim_seq_group = self.serving[victim_idx]
101 + victim_seq_group = self.running[victim_idx]
102 self._swap_out(victim_seq_group)
103 victim_idx -= 1
104 if i > victim_idx:
@@ -104,7 +106,7 @@ class Scheduler:
106 break
107 else:
108 self._append(seq_group)
107 - self.serving = self.serving[:victim_idx + 1]
109 + self.running = self.running[:victim_idx + 1]
110
111 # 2. Swap in the swapped sequences if possible.
112 # NOTE: Here we implicitly assume FCFS scheduling.
@@ -121,18 +123,27 @@ class Scheduler:
123 # All swapped sequences are swapped in.
124 self.swapped.clear()
125
126 + num_batched_tokens = sum(
127 + seq_group.num_seqs(status=SequenceStatus.RUNNING)
128 + for seq_group in self.running
129 + )
130 +
131 # 3. Join new sequences if possible.
132 # NOTE: Here we implicitly assume FCFS scheduling.
126 - # TODO(woosuk): Add a heuristic to control the maximum batch size.
133 + # TODO(woosuk): Add a batching policy to control the batch size.
134 if not self.swapped:
128 - # FIXME: Acquire a lock.
135 + # FIXME(woosuk): Acquire a lock to protect pending.
136 for i, seq_group in enumerate(self.pending):
137 + num_prompt_tokens = seq_group.seqs[0].get_len()
138 if self.block_manager.can_allocate(seq_group):
131 - self._allocate(seq_group)
132 - else:
133 - # FIXME: Consider the race condition.
134 - self.pending = self.pending[i:]
135 - break
139 + if (num_batched_tokens + num_prompt_tokens
140 + <= _MAX_NUM_BATCHED_TOKENS):
141 + self._allocate(seq_group)
142 + num_batched_tokens += num_prompt_tokens
143 + continue
144 +
145 + self.pending = self.pending[i:]
146 + break
147 else:
148 self.pending.clear()
149
@@ -141,8 +152,35 @@ class Scheduler:
152 if self.blocks_to_swap_in:
153 assert not self.blocks_to_swap_out
154
155 + # Create input data structures.
156 + prompt_tokens: Dict[int, List[int]] = {}
157 + generation_tokens: Dict[int, int] = {}
158 + context_lens: Dict[int, int] = {}
159 + block_tables: Dict[int, List[int]] = {}
160 + for seq_group in self.running:
161 + group_id = seq_group.group_id
162 + num_steps = self.num_steps[group_id]
163 + # NOTE(woosuk): We assume that the number of steps is 0
164 + # for the prompt sequences.
165 + is_prompt = num_steps == 0
166 + for seq in seq_group.seqs:
167 + if seq.status != SequenceStatus.RUNNING:
168 + continue
169 +
170 + seq_id = seq.seq_id
171 + block_tables[seq_id] = self.block_manager.get_block_table(seq)
172 + if is_prompt:
173 + prompt_tokens[seq_id] = seq.get_token_ids()
174 + else:
175 + generation_tokens[seq_id] = seq.get_token_ids()[-1]
176 + context_lens[seq_id] = seq.get_len()
177 +
178 # Execute the first stage of the pipeline.
179 self.controllers[0].execute_stage(
180 + prompt_tokens,
181 + generation_tokens,
182 + context_lens,
183 + block_tables,
184 self.blocks_to_swap_in.copy(),
185 self.blocks_to_swap_out.copy(),
186 self.blocks_to_copy.copy(),
@@ -158,7 +196,7 @@ class Scheduler:
196 next_tokens: Dict[int, Tuple[int, int]],
197 ) -> None:
198 # Update the running sequences and free blocks.
161 - for seq_group in self.serving:
199 + for seq_group in self.running:
200 group_id = seq_group.group_id
201 self.num_steps[group_id] += 1
202 stop_token_ids = self.stop_token_ids[group_id]
@@ -190,14 +228,14 @@ class Scheduler:
228 self._free_seq(seq)
229 continue
230
193 - # Update the serving states.
194 - serving: List[SequenceGroup] = []
195 - for seq_group in self.serving:
231 + # Update the running sequences.
232 + running: List[SequenceGroup] = []
233 + for seq_group in self.running:
234 if all(seq.status == SequenceStatus.FINISHED for seq in seq_group.seqs):
235 del self.num_steps[seq_group.group_id]
236 del self.max_num_steps[seq_group.group_id]
237 del self.stop_token_ids[seq_group.group_id]
238 # TODO: Return the seq_group to the client.
239 else:
202 - serving.append(seq_group)
203 - self.serving = serving
240 + running.append(seq_group)
241 + self.running = running