A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Fix bugs in scheduler
Woosuk Kwon committed
Feb 14, 2023 at 02:25 UTC
7e5f604e68fb3eb69c62297984a4da355b7f8eba
1 file changed
+12
-6
cacheflow/master/scheduler.py
+12
-6
@@ -42,9 +42,9 @@ class Scheduler:
42
self.pending: List[SequenceGroup] = []
43
44
# Blocks that need to be swaped or copied before model execution.
45
- self.blocks_to_swap_in: Dict[int, int] = []
46
- self.blocks_to_swap_out: Dict[int, int] = []
47
- self.blocks_to_copy: Dict[int, int] = []
45
+ self.blocks_to_swap_in: Dict[int, int] = {}
46
+ self.blocks_to_swap_out: Dict[int, int] = {}
47
+ self.blocks_to_copy: Dict[int, int] = {}
48
49
def _free_seq(self, seq: Sequence) -> None:
50
seq.status = SequenceStatus.FINISHED
@@ -55,6 +55,8 @@ class Scheduler:
55
for seq in seq_group.seqs:
56
seq.status = SequenceStatus.RUNNING
57
self.serving.append(seq_group)
58
+ # FIXME
59
+ self.num_steps[seq_group.group_id] = 0
60
61
def _append(self, seq_group: SequenceGroup) -> None:
62
for seq in seq_group.seqs:
@@ -123,6 +125,7 @@ class Scheduler:
125
# NOTE: Here we implicitly assume FCFS scheduling.
126
# TODO(woosuk): Add a heuristic to control the maximum batch size.
127
if not self.swapped:
128
+ # FIXME: Acquire a lock.
129
for i, seq_group in enumerate(self.pending):
130
if self.block_manager.can_allocate(seq_group):
131
self._allocate(seq_group)
@@ -130,11 +133,13 @@ class Scheduler:
133
# FIXME: Consider the race condition.
134
self.pending = self.pending[i:]
135
break
136
+ else:
137
+ self.pending.clear()
138
139
def step(self) -> None:
140
# Ensure that either swap-in or swap-out is performed.
136
- if self.blocks_to_swap_in is not None:
137
- assert self.blocks_to_swap_out is None
141
+ if self.blocks_to_swap_in:
142
+ assert not self.blocks_to_swap_out
143
144
# Execute the first stage of the pipeline.
145
self.controllers[0].execute_stage(
@@ -142,6 +147,7 @@ class Scheduler:
147
self.blocks_to_swap_out.copy(),
148
self.blocks_to_copy.copy(),
149
)
150
+
151
# Clear for the next step.
152
self.blocks_to_swap_in.clear()
153
self.blocks_to_swap_out.clear()
@@ -187,7 +193,7 @@ class Scheduler:
193
# Update the serving states.
194
serving: List[SequenceGroup] = []
195
for seq_group in self.serving:
190
- if seq_group.num_seqs(status=SequenceStatus.RUNNING) == 0:
196
+ if all(seq.status == SequenceStatus.FINISHED for seq in seq_group.seqs):
197
del self.num_steps[seq_group.group_id]
198
del self.max_num_steps[seq_group.group_id]
199
del self.stop_token_ids[seq_group.group_id]