@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7e5f604e68
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Fix bugs in scheduler

Woosuk Kwon committed Feb 14, 2023 at 02:25 UTC 7e5f604e68fb3eb69c62297984a4da355b7f8eba
1 file changed +12 -6
cacheflow/master/scheduler.py
+12 -6
@@ -42,9 +42,9 @@ class Scheduler:
42 self.pending: List[SequenceGroup] = []
43
44 # Blocks that need to be swaped or copied before model execution.
45 - self.blocks_to_swap_in: Dict[int, int] = []
46 - self.blocks_to_swap_out: Dict[int, int] = []
47 - self.blocks_to_copy: Dict[int, int] = []
45 + self.blocks_to_swap_in: Dict[int, int] = {}
46 + self.blocks_to_swap_out: Dict[int, int] = {}
47 + self.blocks_to_copy: Dict[int, int] = {}
48
49 def _free_seq(self, seq: Sequence) -> None:
50 seq.status = SequenceStatus.FINISHED
@@ -55,6 +55,8 @@ class Scheduler:
55 for seq in seq_group.seqs:
56 seq.status = SequenceStatus.RUNNING
57 self.serving.append(seq_group)
58 + # FIXME
59 + self.num_steps[seq_group.group_id] = 0
60
61 def _append(self, seq_group: SequenceGroup) -> None:
62 for seq in seq_group.seqs:
@@ -123,6 +125,7 @@ class Scheduler:
125 # NOTE: Here we implicitly assume FCFS scheduling.
126 # TODO(woosuk): Add a heuristic to control the maximum batch size.
127 if not self.swapped:
128 + # FIXME: Acquire a lock.
129 for i, seq_group in enumerate(self.pending):
130 if self.block_manager.can_allocate(seq_group):
131 self._allocate(seq_group)
@@ -130,11 +133,13 @@ class Scheduler:
133 # FIXME: Consider the race condition.
134 self.pending = self.pending[i:]
135 break
136 + else:
137 + self.pending.clear()
138
139 def step(self) -> None:
140 # Ensure that either swap-in or swap-out is performed.
136 - if self.blocks_to_swap_in is not None:
137 - assert self.blocks_to_swap_out is None
141 + if self.blocks_to_swap_in:
142 + assert not self.blocks_to_swap_out
143
144 # Execute the first stage of the pipeline.
145 self.controllers[0].execute_stage(
@@ -142,6 +147,7 @@ class Scheduler:
147 self.blocks_to_swap_out.copy(),
148 self.blocks_to_copy.copy(),
149 )
150 +
151 # Clear for the next step.
152 self.blocks_to_swap_in.clear()
153 self.blocks_to_swap_out.clear()
@@ -187,7 +193,7 @@ class Scheduler:
193 # Update the serving states.
194 serving: List[SequenceGroup] = []
195 for seq_group in self.serving:
190 - if seq_group.num_seqs(status=SequenceStatus.RUNNING) == 0:
196 + if all(seq.status == SequenceStatus.FINISHED for seq in seq_group.seqs):
197 del self.num_steps[seq_group.group_id]
198 del self.max_num_steps[seq_group.group_id]
199 del self.stop_token_ids[seq_group.group_id]