@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d3d317665b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Fix scheduler

Woosuk Kwon committed Feb 13, 2023 at 09:37 UTC d3d317665b594e752422039f925ff8a0770723f1
1 file changed +38 -11
cacheflow/master/scheduler.py
+38 -11
@@ -10,10 +10,17 @@ class Scheduler:
10
11 def __int__(
12 self,
13 + controllers: List,
14 block_size: int,
15 num_gpu_blocks: int,
16 num_cpu_blocks: int,
17 ) -> None:
18 + self.controllers = controllers
19 + self.block_size = block_size
20 + self.num_gpu_blocks = num_gpu_blocks
21 + self.num_cpu_blocks = num_cpu_blocks
22 +
23 + # Create the block space manager.
24 self.block_manager = BlockSpaceManager(
25 block_size=block_size,
26 num_gpu_blocks=num_gpu_blocks,
@@ -31,9 +38,13 @@ class Scheduler:
38
39 # Swapped sequence groups (LIFO).
40 self.swapped: List[SequenceGroup] = []
34 -
41 # Pending sequence groups (FIFO).
36 - self.queue: List[SequenceGroup] = []
42 + self.pending: List[SequenceGroup] = []
43 +
44 + # Blocks that need to be swaped or copied before model execution.
45 + self.blocks_to_swap_in: Dict[int, int] = []
46 + self.blocks_to_swap_out: Dict[int, int] = []
47 + self.blocks_to_copy: Dict[int, int] = []
48
49 def _free_seq(self, seq: Sequence) -> None:
50 seq.status = SequenceStatus.FINISHED
@@ -52,12 +63,11 @@ class Scheduler:
63 ret = self.block_manager.append(seq)
64 if ret is not None:
65 src_block, dst_block = ret
55 - # TODO: Issue COPY commands to the workers.
66 + self.blocks_to_copy[src_block] = dst_block
67
68 def _swap_in(self, seq_group: SequenceGroup) -> None:
58 - # TODO: Issue SWAP_IN commands to the workers.
59 - self.block_manager.swap_in(seq_group)
60 - self.block_manager.append(seq_group)
69 + mapping = self.block_manager.swap_in(seq_group)
70 + self.blocks_to_swap_in.update(mapping)
71 for seq in seq_group.seqs:
72 if seq.status == SequenceStatus.SWAPPED:
73 seq.status = SequenceStatus.RUNNING
@@ -65,14 +75,14 @@ class Scheduler:
75
76 def _swap_out(self, seq_group: SequenceGroup) -> None:
77 assert self.block_manager.can_swap_out(seq_group)
68 - # TODO: Issue SWAP_OUT commands to the workers.
69 - self.block_manager.swap_out(seq_group)
78 + mapping = self.block_manager.swap_out(seq_group)
79 + self.blocks_to_swap_out.update(mapping)
80 for seq in seq_group.seqs:
81 if seq.status == SequenceStatus.RUNNING:
82 seq.status = SequenceStatus.SWAPPED
83 self.swapped.append(seq_group)
84
75 - def step(self) -> None:
85 + def prepare(self) -> None:
86 # 1. Prepare new slots for the running sequences.
87 # NOTE: Here we implicitly assume FCFS scheduling.
88 # That is, the most recently added sequence group is the first
@@ -100,6 +110,7 @@ class Scheduler:
110 for i, seq_group in enumerate(reversed(self.swapped)):
111 if self.block_manager.can_swap_in(seq_group):
112 self._swap_in(seq_group)
113 + self._append(seq_group)
114 else:
115 # OOM. Stop swapping.
116 self.swapped = self.swapped[:len(self.swapped) - i]
@@ -112,14 +123,30 @@ class Scheduler:
123 # NOTE: Here we implicitly assume FCFS scheduling.
124 # TODO(woosuk): Add a heuristic to control the maximum batch size.
125 if not self.swapped:
115 - for i, seq_group in enumerate(self.queue):
126 + for i, seq_group in enumerate(self.pending):
127 if self.block_manager.can_allocate(seq_group):
128 self._allocate(seq_group)
129 else:
130 # FIXME: Consider the race condition.
120 - self.queue = self.queue[i:]
131 + self.pending = self.pending[i:]
132 break
133
134 + def step(self) -> None:
135 + # Ensure that either swap-in or swap-out is performed.
136 + if self.blocks_to_swap_in is not None:
137 + assert self.blocks_to_swap_out is None
138 +
139 + # Execute the first stage of the pipeline.
140 + self.controllers[0].execute_stage(
141 + self.blocks_to_swap_in.copy(),
142 + self.blocks_to_swap_out.copy(),
143 + self.blocks_to_copy.copy(),
144 + )
145 + # Clear for the next step.
146 + self.blocks_to_swap_in.clear()
147 + self.blocks_to_swap_out.clear()
148 + self.blocks_to_copy.clear()
149 +
150 def post_step(
151 self,
152 next_tokens: Dict[int, Tuple[int, int]],