A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Reduce the number of states in scheduler
Woosuk Kwon committed
Feb 24, 2023 at 10:22 UTC
53f70e73344a67f61b80feab03834a770bfb671b
1 file changed
+44
-29
cacheflow/master/scheduler.py
+44
-29
@@ -43,11 +43,6 @@ class Scheduler:
43
# Pending sequence groups (FIFO).
44
self.pending: List[SequenceGroup] = []
45
46
- # Blocks that need to be swaped or copied before model execution.
47
- self.blocks_to_swap_in: Dict[int, int] = {}
48
- self.blocks_to_swap_out: Dict[int, int] = {}
49
- self.blocks_to_copy: Dict[int, int] = {}
50
-
46
def _free_seq(self, seq: Sequence) -> None:
47
seq.status = SequenceStatus.FINISHED
48
self.block_manager.free(seq)
@@ -57,36 +52,53 @@ class Scheduler:
52
for seq in seq_group.seqs:
53
seq.status = SequenceStatus.RUNNING
54
self.running.append(seq_group)
60
- # FIXME
55
+ # FIXME(woosuk): Support interactive generation.
56
self.num_steps[seq_group.group_id] = 0
57
63
- def _append(self, seq_group: SequenceGroup) -> None:
58
+ def _append(
59
+ self,
60
+ seq_group: SequenceGroup,
61
+ blocks_to_copy: Dict[int, int],
62
+ ) -> None:
63
for seq in seq_group.seqs:
64
if seq.status == SequenceStatus.FINISHED:
65
continue
66
ret = self.block_manager.append(seq)
67
if ret is not None:
68
src_block, dst_block = ret
70
- self.blocks_to_copy[src_block] = dst_block
69
+ blocks_to_copy[src_block] = dst_block
70
72
- def _swap_in(self, seq_group: SequenceGroup) -> None:
71
+ def _swap_in(
72
+ self,
73
+ seq_group: SequenceGroup,
74
+ blocks_to_swap_in: Dict[int, int],
75
+ ) -> None:
76
mapping = self.block_manager.swap_in(seq_group)
74
- self.blocks_to_swap_in.update(mapping)
77
+ blocks_to_swap_in.update(mapping)
78
for seq in seq_group.seqs:
79
if seq.status == SequenceStatus.SWAPPED:
80
seq.status = SequenceStatus.RUNNING
81
self.running.append(seq_group)
82
80
- def _swap_out(self, seq_group: SequenceGroup) -> None:
83
+ def _swap_out(
84
+ self,
85
+ seq_group: SequenceGroup,
86
+ blocks_to_swap_out: Dict[int, int],
87
+ ) -> None:
88
assert self.block_manager.can_swap_out(seq_group)
89
mapping = self.block_manager.swap_out(seq_group)
83
- self.blocks_to_swap_out.update(mapping)
90
+ blocks_to_swap_out.update(mapping)
91
for seq in seq_group.seqs:
92
if seq.status == SequenceStatus.RUNNING:
93
seq.status = SequenceStatus.SWAPPED
94
self.swapped.append(seq_group)
95
89
- def prepare(self) -> None:
96
+ def pre_step(self) -> None:
97
+ # Blocks that need to be swaped or copied before model execution.
98
+ blocks_to_swap_in: Dict[int, int] = {}
99
+ blocks_to_swap_out: Dict[int, int] = {}
100
+ blocks_to_copy: Dict[int, int] = {}
101
+
102
# 1. Prepare new slots for the running sequences.
103
# NOTE: Here we implicitly assume FCFS scheduling.
104
# That is, the most recently added sequence group is the first
@@ -99,13 +111,13 @@ class Scheduler:
111
# OOM. Swap out the victim sequence groups.
112
while not self.block_manager.can_append(seq_group):
113
victim_seq_group = self.running[victim_idx]
102
- self._swap_out(victim_seq_group)
114
+ self._swap_out(victim_seq_group, blocks_to_swap_out)
115
victim_idx -= 1
116
if i > victim_idx:
117
# No other sequence groups can be swapped out.
118
break
119
else:
108
- self._append(seq_group)
120
+ self._append(seq_group, blocks_to_copy)
121
self.running = self.running[:victim_idx + 1]
122
123
# 2. Swap in the swapped sequences if possible.
@@ -113,8 +125,8 @@ class Scheduler:
125
# The swapped sequences are in LIFO order.
126
for i, seq_group in enumerate(reversed(self.swapped)):
127
if self.block_manager.can_swap_in(seq_group):
116
- self._swap_in(seq_group)
117
- self._append(seq_group)
128
+ self._swap_in(seq_group, blocks_to_swap_in)
129
+ self._append(seq_group, blocks_to_copy)
130
else:
131
# OOM. Stop swapping.
132
self.swapped = self.swapped[:len(self.swapped) - i]
@@ -147,10 +159,18 @@ class Scheduler:
159
else:
160
self.pending.clear()
161
150
- def step(self) -> None:
151
- # Ensure that either swap-in or swap-out is performed.
152
- if self.blocks_to_swap_in:
153
- assert not self.blocks_to_swap_out
162
+ # Execute step.
163
+ self.step(blocks_to_swap_in, blocks_to_swap_out, blocks_to_copy)
164
+
165
+ def step(
166
+ self,
167
+ blocks_to_swap_in: Dict[int, int],
168
+ blocks_to_swap_out: Dict[int, int],
169
+ blocks_to_copy: Dict[int, int],
170
+ ) -> None:
171
+ # Ensure that swap-in and swap-out never happen at the same timestep.
172
+ if blocks_to_swap_in:
173
+ assert not blocks_to_swap_out
174
175
# Create input data structures.
176
prompt_tokens: Dict[int, List[int]] = {}
@@ -181,16 +201,11 @@ class Scheduler:
201
generation_tokens,
202
context_lens,
203
block_tables,
184
- self.blocks_to_swap_in.copy(),
185
- self.blocks_to_swap_out.copy(),
186
- self.blocks_to_copy.copy(),
204
+ blocks_to_swap_in,
205
+ blocks_to_swap_out,
206
+ blocks_to_copy,
207
)
208
189
- # Clear for the next step.
190
- self.blocks_to_swap_in.clear()
191
- self.blocks_to_swap_out.clear()
192
- self.blocks_to_copy.clear()
193
-
209
def post_step(
210
self,
211
next_tokens: Dict[int, Tuple[int, int]],