@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 53f70e7334
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Reduce the number of states in scheduler

Woosuk Kwon committed Feb 24, 2023 at 10:22 UTC 53f70e73344a67f61b80feab03834a770bfb671b
1 file changed +44 -29
cacheflow/master/scheduler.py
+44 -29
@@ -43,11 +43,6 @@ class Scheduler:
43 # Pending sequence groups (FIFO).
44 self.pending: List[SequenceGroup] = []
45
46 - # Blocks that need to be swaped or copied before model execution.
47 - self.blocks_to_swap_in: Dict[int, int] = {}
48 - self.blocks_to_swap_out: Dict[int, int] = {}
49 - self.blocks_to_copy: Dict[int, int] = {}
50 -
46 def _free_seq(self, seq: Sequence) -> None:
47 seq.status = SequenceStatus.FINISHED
48 self.block_manager.free(seq)
@@ -57,36 +52,53 @@ class Scheduler:
52 for seq in seq_group.seqs:
53 seq.status = SequenceStatus.RUNNING
54 self.running.append(seq_group)
60 - # FIXME
55 + # FIXME(woosuk): Support interactive generation.
56 self.num_steps[seq_group.group_id] = 0
57
63 - def _append(self, seq_group: SequenceGroup) -> None:
58 + def _append(
59 + self,
60 + seq_group: SequenceGroup,
61 + blocks_to_copy: Dict[int, int],
62 + ) -> None:
63 for seq in seq_group.seqs:
64 if seq.status == SequenceStatus.FINISHED:
65 continue
66 ret = self.block_manager.append(seq)
67 if ret is not None:
68 src_block, dst_block = ret
70 - self.blocks_to_copy[src_block] = dst_block
69 + blocks_to_copy[src_block] = dst_block
70
72 - def _swap_in(self, seq_group: SequenceGroup) -> None:
71 + def _swap_in(
72 + self,
73 + seq_group: SequenceGroup,
74 + blocks_to_swap_in: Dict[int, int],
75 + ) -> None:
76 mapping = self.block_manager.swap_in(seq_group)
74 - self.blocks_to_swap_in.update(mapping)
77 + blocks_to_swap_in.update(mapping)
78 for seq in seq_group.seqs:
79 if seq.status == SequenceStatus.SWAPPED:
80 seq.status = SequenceStatus.RUNNING
81 self.running.append(seq_group)
82
80 - def _swap_out(self, seq_group: SequenceGroup) -> None:
83 + def _swap_out(
84 + self,
85 + seq_group: SequenceGroup,
86 + blocks_to_swap_out: Dict[int, int],
87 + ) -> None:
88 assert self.block_manager.can_swap_out(seq_group)
89 mapping = self.block_manager.swap_out(seq_group)
83 - self.blocks_to_swap_out.update(mapping)
90 + blocks_to_swap_out.update(mapping)
91 for seq in seq_group.seqs:
92 if seq.status == SequenceStatus.RUNNING:
93 seq.status = SequenceStatus.SWAPPED
94 self.swapped.append(seq_group)
95
89 - def prepare(self) -> None:
96 + def pre_step(self) -> None:
97 + # Blocks that need to be swaped or copied before model execution.
98 + blocks_to_swap_in: Dict[int, int] = {}
99 + blocks_to_swap_out: Dict[int, int] = {}
100 + blocks_to_copy: Dict[int, int] = {}
101 +
102 # 1. Prepare new slots for the running sequences.
103 # NOTE: Here we implicitly assume FCFS scheduling.
104 # That is, the most recently added sequence group is the first
@@ -99,13 +111,13 @@ class Scheduler:
111 # OOM. Swap out the victim sequence groups.
112 while not self.block_manager.can_append(seq_group):
113 victim_seq_group = self.running[victim_idx]
102 - self._swap_out(victim_seq_group)
114 + self._swap_out(victim_seq_group, blocks_to_swap_out)
115 victim_idx -= 1
116 if i > victim_idx:
117 # No other sequence groups can be swapped out.
118 break
119 else:
108 - self._append(seq_group)
120 + self._append(seq_group, blocks_to_copy)
121 self.running = self.running[:victim_idx + 1]
122
123 # 2. Swap in the swapped sequences if possible.
@@ -113,8 +125,8 @@ class Scheduler:
125 # The swapped sequences are in LIFO order.
126 for i, seq_group in enumerate(reversed(self.swapped)):
127 if self.block_manager.can_swap_in(seq_group):
116 - self._swap_in(seq_group)
117 - self._append(seq_group)
128 + self._swap_in(seq_group, blocks_to_swap_in)
129 + self._append(seq_group, blocks_to_copy)
130 else:
131 # OOM. Stop swapping.
132 self.swapped = self.swapped[:len(self.swapped) - i]
@@ -147,10 +159,18 @@ class Scheduler:
159 else:
160 self.pending.clear()
161
150 - def step(self) -> None:
151 - # Ensure that either swap-in or swap-out is performed.
152 - if self.blocks_to_swap_in:
153 - assert not self.blocks_to_swap_out
162 + # Execute step.
163 + self.step(blocks_to_swap_in, blocks_to_swap_out, blocks_to_copy)
164 +
165 + def step(
166 + self,
167 + blocks_to_swap_in: Dict[int, int],
168 + blocks_to_swap_out: Dict[int, int],
169 + blocks_to_copy: Dict[int, int],
170 + ) -> None:
171 + # Ensure that swap-in and swap-out never happen at the same timestep.
172 + if blocks_to_swap_in:
173 + assert not blocks_to_swap_out
174
175 # Create input data structures.
176 prompt_tokens: Dict[int, List[int]] = {}
@@ -181,16 +201,11 @@ class Scheduler:
201 generation_tokens,
202 context_lens,
203 block_tables,
184 - self.blocks_to_swap_in.copy(),
185 - self.blocks_to_swap_out.copy(),
186 - self.blocks_to_copy.copy(),
204 + blocks_to_swap_in,
205 + blocks_to_swap_out,
206 + blocks_to_copy,
207 )
208
189 - # Clear for the next step.
190 - self.blocks_to_swap_in.clear()
191 - self.blocks_to_swap_out.clear()
192 - self.blocks_to_copy.clear()
193 -
209 def post_step(
210 self,
211 next_tokens: Dict[int, Tuple[int, int]],