A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Fix scheduler
Woosuk Kwon committed
Feb 13, 2023 at 09:37 UTC
d3d317665b594e752422039f925ff8a0770723f1
1 file changed
+38
-11
cacheflow/master/scheduler.py
+38
-11
@@ -10,10 +10,17 @@ class Scheduler:
10
11
def __int__(
12
self,
13
+ controllers: List,
14
block_size: int,
15
num_gpu_blocks: int,
16
num_cpu_blocks: int,
17
) -> None:
18
+ self.controllers = controllers
19
+ self.block_size = block_size
20
+ self.num_gpu_blocks = num_gpu_blocks
21
+ self.num_cpu_blocks = num_cpu_blocks
22
+
23
+ # Create the block space manager.
24
self.block_manager = BlockSpaceManager(
25
block_size=block_size,
26
num_gpu_blocks=num_gpu_blocks,
@@ -31,9 +38,13 @@ class Scheduler:
38
39
# Swapped sequence groups (LIFO).
40
self.swapped: List[SequenceGroup] = []
34
-
41
# Pending sequence groups (FIFO).
36
- self.queue: List[SequenceGroup] = []
42
+ self.pending: List[SequenceGroup] = []
43
+
44
+ # Blocks that need to be swaped or copied before model execution.
45
+ self.blocks_to_swap_in: Dict[int, int] = []
46
+ self.blocks_to_swap_out: Dict[int, int] = []
47
+ self.blocks_to_copy: Dict[int, int] = []
48
49
def _free_seq(self, seq: Sequence) -> None:
50
seq.status = SequenceStatus.FINISHED
@@ -52,12 +63,11 @@ class Scheduler:
63
ret = self.block_manager.append(seq)
64
if ret is not None:
65
src_block, dst_block = ret
55
- # TODO: Issue COPY commands to the workers.
66
+ self.blocks_to_copy[src_block] = dst_block
67
68
def _swap_in(self, seq_group: SequenceGroup) -> None:
58
- # TODO: Issue SWAP_IN commands to the workers.
59
- self.block_manager.swap_in(seq_group)
60
- self.block_manager.append(seq_group)
69
+ mapping = self.block_manager.swap_in(seq_group)
70
+ self.blocks_to_swap_in.update(mapping)
71
for seq in seq_group.seqs:
72
if seq.status == SequenceStatus.SWAPPED:
73
seq.status = SequenceStatus.RUNNING
@@ -65,14 +75,14 @@ class Scheduler:
75
76
def _swap_out(self, seq_group: SequenceGroup) -> None:
77
assert self.block_manager.can_swap_out(seq_group)
68
- # TODO: Issue SWAP_OUT commands to the workers.
69
- self.block_manager.swap_out(seq_group)
78
+ mapping = self.block_manager.swap_out(seq_group)
79
+ self.blocks_to_swap_out.update(mapping)
80
for seq in seq_group.seqs:
81
if seq.status == SequenceStatus.RUNNING:
82
seq.status = SequenceStatus.SWAPPED
83
self.swapped.append(seq_group)
84
75
- def step(self) -> None:
85
+ def prepare(self) -> None:
86
# 1. Prepare new slots for the running sequences.
87
# NOTE: Here we implicitly assume FCFS scheduling.
88
# That is, the most recently added sequence group is the first
@@ -100,6 +110,7 @@ class Scheduler:
110
for i, seq_group in enumerate(reversed(self.swapped)):
111
if self.block_manager.can_swap_in(seq_group):
112
self._swap_in(seq_group)
113
+ self._append(seq_group)
114
else:
115
# OOM. Stop swapping.
116
self.swapped = self.swapped[:len(self.swapped) - i]
@@ -112,14 +123,30 @@ class Scheduler:
123
# NOTE: Here we implicitly assume FCFS scheduling.
124
# TODO(woosuk): Add a heuristic to control the maximum batch size.
125
if not self.swapped:
115
- for i, seq_group in enumerate(self.queue):
126
+ for i, seq_group in enumerate(self.pending):
127
if self.block_manager.can_allocate(seq_group):
128
self._allocate(seq_group)
129
else:
130
# FIXME: Consider the race condition.
120
- self.queue = self.queue[i:]
131
+ self.pending = self.pending[i:]
132
break
133
134
+ def step(self) -> None:
135
+ # Ensure that either swap-in or swap-out is performed.
136
+ if self.blocks_to_swap_in is not None:
137
+ assert self.blocks_to_swap_out is None
138
+
139
+ # Execute the first stage of the pipeline.
140
+ self.controllers[0].execute_stage(
141
+ self.blocks_to_swap_in.copy(),
142
+ self.blocks_to_swap_out.copy(),
143
+ self.blocks_to_copy.copy(),
144
+ )
145
+ # Clear for the next step.
146
+ self.blocks_to_swap_in.clear()
147
+ self.blocks_to_swap_out.clear()
148
+ self.blocks_to_copy.clear()
149
+
150
def post_step(
151
self,
152
next_tokens: Dict[int, Tuple[int, int]],