A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Implement scheduler.step & Add a threshold for batch size
Woosuk Kwon committed
Feb 23, 2023 at 07:54 UTC
331fa0b042b5cbcdadb03f25dc8592aba9c12b53
1 file changed
+59
-21
cacheflow/master/scheduler.py
+59
-21
@@ -5,6 +5,8 @@ from cacheflow.sequence import Sequence
5
from cacheflow.sequence import SequenceGroup
6
from cacheflow.sequence import SequenceStatus
7
8
+_MAX_NUM_BATCHED_TOKENS = 2048
9
+
10
11
class Scheduler:
12
@@ -27,8 +29,8 @@ class Scheduler:
29
num_cpu_blocks=num_cpu_blocks,
30
)
31
30
- # Serving sequence groups (FIFO).
31
- self.serving: List[SequenceGroup] = []
32
+ # Running sequence groups (FIFO).
33
+ self.running: List[SequenceGroup] = []
34
# Mapping: group_id -> num_steps.
35
self.num_steps: Dict[int, int] = {}
36
# Mapping: group_id -> max_num_steps.
@@ -54,7 +56,7 @@ class Scheduler:
56
self.block_manager.allocate(seq_group)
57
for seq in seq_group.seqs:
58
seq.status = SequenceStatus.RUNNING
57
- self.serving.append(seq_group)
59
+ self.running.append(seq_group)
60
# FIXME
61
self.num_steps[seq_group.group_id] = 0
62
@@ -73,7 +75,7 @@ class Scheduler:
75
for seq in seq_group.seqs:
76
if seq.status == SequenceStatus.SWAPPED:
77
seq.status = SequenceStatus.RUNNING
76
- self.serving.append(seq_group)
78
+ self.running.append(seq_group)
79
80
def _swap_out(self, seq_group: SequenceGroup) -> None:
81
assert self.block_manager.can_swap_out(seq_group)
@@ -89,14 +91,14 @@ class Scheduler:
91
# NOTE: Here we implicitly assume FCFS scheduling.
92
# That is, the most recently added sequence group is the first
93
# to be swapped out.
92
- victim_idx = len(self.serving) - 1
93
- for i, seq_group in enumerate(self.serving):
94
+ victim_idx = len(self.running) - 1
95
+ for i, seq_group in enumerate(self.running):
96
if i > victim_idx:
97
# The i-th sequence group has already been swapped out.
98
break
99
# OOM. Swap out the victim sequence groups.
100
while not self.block_manager.can_append(seq_group):
99
- victim_seq_group = self.serving[victim_idx]
101
+ victim_seq_group = self.running[victim_idx]
102
self._swap_out(victim_seq_group)
103
victim_idx -= 1
104
if i > victim_idx:
@@ -104,7 +106,7 @@ class Scheduler:
106
break
107
else:
108
self._append(seq_group)
107
- self.serving = self.serving[:victim_idx + 1]
109
+ self.running = self.running[:victim_idx + 1]
110
111
# 2. Swap in the swapped sequences if possible.
112
# NOTE: Here we implicitly assume FCFS scheduling.
@@ -121,18 +123,27 @@ class Scheduler:
123
# All swapped sequences are swapped in.
124
self.swapped.clear()
125
126
+ num_batched_tokens = sum(
127
+ seq_group.num_seqs(status=SequenceStatus.RUNNING)
128
+ for seq_group in self.running
129
+ )
130
+
131
# 3. Join new sequences if possible.
132
# NOTE: Here we implicitly assume FCFS scheduling.
126
- # TODO(woosuk): Add a heuristic to control the maximum batch size.
133
+ # TODO(woosuk): Add a batching policy to control the batch size.
134
if not self.swapped:
128
- # FIXME: Acquire a lock.
135
+ # FIXME(woosuk): Acquire a lock to protect pending.
136
for i, seq_group in enumerate(self.pending):
137
+ num_prompt_tokens = seq_group.seqs[0].get_len()
138
if self.block_manager.can_allocate(seq_group):
131
- self._allocate(seq_group)
132
- else:
133
- # FIXME: Consider the race condition.
134
- self.pending = self.pending[i:]
135
- break
139
+ if (num_batched_tokens + num_prompt_tokens
140
+ <= _MAX_NUM_BATCHED_TOKENS):
141
+ self._allocate(seq_group)
142
+ num_batched_tokens += num_prompt_tokens
143
+ continue
144
+
145
+ self.pending = self.pending[i:]
146
+ break
147
else:
148
self.pending.clear()
149
@@ -141,8 +152,35 @@ class Scheduler:
152
if self.blocks_to_swap_in:
153
assert not self.blocks_to_swap_out
154
155
+ # Create input data structures.
156
+ prompt_tokens: Dict[int, List[int]] = {}
157
+ generation_tokens: Dict[int, int] = {}
158
+ context_lens: Dict[int, int] = {}
159
+ block_tables: Dict[int, List[int]] = {}
160
+ for seq_group in self.running:
161
+ group_id = seq_group.group_id
162
+ num_steps = self.num_steps[group_id]
163
+ # NOTE(woosuk): We assume that the number of steps is 0
164
+ # for the prompt sequences.
165
+ is_prompt = num_steps == 0
166
+ for seq in seq_group.seqs:
167
+ if seq.status != SequenceStatus.RUNNING:
168
+ continue
169
+
170
+ seq_id = seq.seq_id
171
+ block_tables[seq_id] = self.block_manager.get_block_table(seq)
172
+ if is_prompt:
173
+ prompt_tokens[seq_id] = seq.get_token_ids()
174
+ else:
175
+ generation_tokens[seq_id] = seq.get_token_ids()[-1]
176
+ context_lens[seq_id] = seq.get_len()
177
+
178
# Execute the first stage of the pipeline.
179
self.controllers[0].execute_stage(
180
+ prompt_tokens,
181
+ generation_tokens,
182
+ context_lens,
183
+ block_tables,
184
self.blocks_to_swap_in.copy(),
185
self.blocks_to_swap_out.copy(),
186
self.blocks_to_copy.copy(),
@@ -158,7 +196,7 @@ class Scheduler:
196
next_tokens: Dict[int, Tuple[int, int]],
197
) -> None:
198
# Update the running sequences and free blocks.
161
- for seq_group in self.serving:
199
+ for seq_group in self.running:
200
group_id = seq_group.group_id
201
self.num_steps[group_id] += 1
202
stop_token_ids = self.stop_token_ids[group_id]
@@ -190,14 +228,14 @@ class Scheduler:
228
self._free_seq(seq)
229
continue
230
193
- # Update the serving states.
194
- serving: List[SequenceGroup] = []
195
- for seq_group in self.serving:
231
+ # Update the running sequences.
232
+ running: List[SequenceGroup] = []
233
+ for seq_group in self.running:
234
if all(seq.status == SequenceStatus.FINISHED for seq in seq_group.seqs):
235
del self.num_steps[seq_group.group_id]
236
del self.max_num_steps[seq_group.group_id]
237
del self.stop_token_ids[seq_group.group_id]
238
# TODO: Return the seq_group to the client.
239
else:
202
- serving.append(seq_group)
203
- self.serving = serving
240
+ running.append(seq_group)
241
+ self.running = running