A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Implement preemption via recomputation & Refactor scheduling logic (#12)
Woosuk Kwon committed
Mar 30, 2023 at 14:51 UTC
7a7929abe8e2fd6a4688487c471a1ee1fde0edd2
7 files changed
+279
-126
cacheflow/http_frontend/fastapi_frontend.py
+2
-1
@@ -84,8 +84,9 @@ class FastAPIFrontend:
84
seq = Sequence(seq_id, token_ids, block_size=self.block_size)
85
seqs.append(seq)
86
87
+ arrival_time = time.time()
88
group_id = next(self.seq_group_counter)
88
- seq_group = SequenceGroup(group_id, seqs)
89
+ seq_group = SequenceGroup(group_id, seqs, arrival_time)
90
group_event = asyncio.Event()
91
self.sequence_group_events[group_id] = group_event
92
await self.server.add_sequence_groups.remote([(seq_group, sampling_params)])
cacheflow/master/block_manager.py
+2
-1
@@ -76,7 +76,8 @@ class BlockSpaceManager:
76
self.block_tables: Dict[int, BlockTable] = {}
77
78
def can_allocate(self, seq_group: SequenceGroup) -> bool:
79
- # NOTE: Here we assume that all sequences in the group have the same prompt.
79
+ # FIXME(woosuk): Here we assume that all sequences in the group share
80
+ # the same prompt. This may not be true for preempted sequences.
81
seq = seq_group.seqs[0]
82
num_required_blocks = len(seq.logical_token_blocks)
83
num_free_gpu_blocks = self.gpu_allocator.get_num_free_blocks()
cacheflow/master/policy.py
new
+45
@@ -0,0 +1,45 @@
1
+from typing import List
2
+
3
+from cacheflow.sequence import SequenceGroup
4
+
5
+
6
+class Policy:
7
+
8
+ def get_priority(
9
+ self,
10
+ now: float,
11
+ seq_group: SequenceGroup,
12
+ ) -> float:
13
+ raise NotImplementedError
14
+
15
+ def sort_by_priority(
16
+ self,
17
+ now: float,
18
+ seq_groups: List[SequenceGroup],
19
+ ) -> List[SequenceGroup]:
20
+ return sorted(
21
+ seq_groups,
22
+ key=lambda seq_group: self.get_priority(now, seq_group),
23
+ reverse=True,
24
+ )
25
+
26
+
27
+class FCFS(Policy):
28
+
29
+ def get_priority(
30
+ self,
31
+ now: float,
32
+ seq_group: SequenceGroup,
33
+ ) -> float:
34
+ return now - seq_group.arrival_time
35
+
36
+
37
+class PolicyFactory:
38
+
39
+ _POLICY_REGISTRY = {
40
+ 'fcfs': FCFS,
41
+ }
42
+
43
+ @classmethod
44
+ def get_policy(cls, policy_name: str, **kwargs) -> Policy:
45
+ return cls._POLICY_REGISTRY[policy_name](**kwargs)
cacheflow/master/scheduler.py
+221
-120
@@ -1,6 +1,9 @@
1
-from typing import Dict, List, Tuple
1
+import enum
2
+import time
3
+from typing import Dict, List, Optional, Tuple
4
5
from cacheflow.master.block_manager import BlockSpaceManager
6
+from cacheflow.master.policy import PolicyFactory
7
from cacheflow.sampling_params import SamplingParams
8
from cacheflow.sequence import Sequence
9
from cacheflow.sequence import SequenceGroup
@@ -9,6 +12,19 @@ from cacheflow.sequence import SequenceOutputs
12
from cacheflow.sequence import SequenceStatus
13
14
15
+class PreemptionMode(enum.Enum):
16
+ """Preemption modes.
17
+
18
+ 1. Swapping: Swap out the blocks of the preempted sequences to CPU memory
19
+ and swap them back in when the sequences are resumed.
20
+ 2. Recomputation: Discard the blocks of the preempted sequences and
21
+ recompute them when the sequences are resumed, treating the sequences as
22
+ new prompts.
23
+ """
24
+ SWAP = enum.auto()
25
+ RECOMPUTE = enum.auto()
26
+
27
+
28
class Scheduler:
29
30
def __init__(
@@ -25,6 +41,8 @@ class Scheduler:
41
self.num_cpu_blocks = num_cpu_blocks
42
self.max_num_batched_tokens = max_num_batched_tokens
43
44
+ # Instantiate the scheduling policy.
45
+ self.policy = PolicyFactory.get_policy(policy_name='fcfs')
46
# Create the block space manager.
47
self.block_manager = BlockSpaceManager(
48
block_size=block_size,
@@ -32,158 +50,140 @@ class Scheduler:
50
num_cpu_blocks=num_cpu_blocks,
51
)
52
35
- # Running sequence groups (FIFO).
53
+ # Sequence groups in the WAITING state.
54
+ self.waiting: List[SequenceGroup] = []
55
+ # Sequence groups in the RUNNING state.
56
self.running: List[SequenceGroup] = []
57
# Mapping: group_id -> num_steps.
58
self.num_steps: Dict[int, int] = {}
59
# Mapping: group_id -> sampling params.
60
self.sampling_params: Dict[int, SamplingParams] = {}
41
-
42
- # Swapped sequence groups (LIFO).
61
+ # Sequence groups in the SWAPPED state.
62
self.swapped: List[SequenceGroup] = []
44
- # Pending sequence groups (FIFO).
45
- self.pending: List[SequenceGroup] = []
63
64
def add_sequence_groups(
65
self,
49
- sequence_groups: List[Tuple[SequenceGroup, SamplingParams]],
66
+ seq_groups: List[Tuple[SequenceGroup, SamplingParams]],
67
) -> None:
51
- # Add sequence groups to the pending queue.
52
- for seq_group, sampling_params in sequence_groups:
53
- self.pending.append(seq_group)
68
+ # Add sequence groups to the waiting queue.
69
+ for seq_group, sampling_params in seq_groups:
70
+ self.waiting.append(seq_group)
71
self.sampling_params[seq_group.group_id] = sampling_params
72
56
- def _free_seq(self, seq: Sequence) -> None:
57
- seq.status = SequenceStatus.FINISHED
58
- self.block_manager.free(seq)
59
-
60
- def _allocate(self, seq_group: SequenceGroup) -> None:
61
- self.block_manager.allocate(seq_group)
62
- for seq in seq_group.seqs:
63
- seq.status = SequenceStatus.RUNNING
64
- self.running.append(seq_group)
65
- # FIXME(woosuk): Support interactive generation.
66
- self.num_steps[seq_group.group_id] = 0
67
-
68
- def _append(
73
+ def _schedule(
74
self,
70
- seq_group: SequenceGroup,
71
- blocks_to_copy: Dict[int, List[int]],
72
- ) -> None:
73
- for seq in seq_group.seqs:
74
- if seq.status == SequenceStatus.FINISHED:
75
- continue
76
- ret = self.block_manager.append(seq)
77
- if ret is not None:
78
- src_block, dst_block = ret
79
- if src_block in blocks_to_copy:
80
- blocks_to_copy[src_block].append(dst_block)
81
- else:
82
- blocks_to_copy[src_block] = [dst_block]
83
-
84
- def _swap_in(
85
- self,
86
- seq_group: SequenceGroup,
87
- blocks_to_swap_in: Dict[int, int],
88
- ) -> None:
89
- mapping = self.block_manager.swap_in(seq_group)
90
- blocks_to_swap_in.update(mapping)
91
- for seq in seq_group.get_seqs(status=SequenceStatus.SWAPPED):
92
- seq.status = SequenceStatus.RUNNING
93
- self.running.append(seq_group)
94
-
95
- def _swap_out(
96
- self,
97
- seq_group: SequenceGroup,
98
- blocks_to_swap_out: Dict[int, int],
99
- ) -> None:
100
- assert self.block_manager.can_swap_out(seq_group)
101
- mapping = self.block_manager.swap_out(seq_group)
102
- blocks_to_swap_out.update(mapping)
103
- for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
104
- seq.status = SequenceStatus.SWAPPED
105
- self.swapped.append(seq_group)
106
-
107
- def step(self) -> List[SequenceGroup]:
75
+ ) -> Tuple[Dict[int, int], Dict[int, int], Dict[int, List[int]], List[int]]:
76
# Blocks that need to be swaped or copied before model execution.
77
blocks_to_swap_in: Dict[int, int] = {}
78
blocks_to_swap_out: Dict[int, int] = {}
79
blocks_to_copy: Dict[int, List[int]] = {}
80
113
- # 1. Reserve new slots for the running sequences.
114
- # NOTE: Here we implicitly assume FCFS scheduling.
115
- # That is, the most recently added sequence group is the first
116
- # to be swapped out.
117
- victim_idx = len(self.running) - 1
118
- for i, seq_group in enumerate(self.running):
119
- if i > victim_idx:
120
- # The i-th sequence group has already been swapped out.
121
- break
122
- # OOM. Swap out the victim sequence groups.
81
+ # Fix the current time.
82
+ now = time.time()
83
+
84
+ # NOTE(woosuk): We prioritize the sequence groups in the RUNNING state
85
+ # in order to minimize the preemption overheads.
86
+ # Preemption happens only when there is no available slot to keep all
87
+ # the sequence groups in the RUNNING state.
88
+ # In this case, the policy is responsible for deciding which sequence
89
+ # groups to preempt.
90
+ self.running = self.policy.sort_by_priority(now, self.running)
91
+
92
+ # Reserve new token slots for the running sequence groups.
93
+ running: List[SequenceGroup] = []
94
+ preempted: List[SequenceGroup] = []
95
+ while self.running:
96
+ seq_group = self.running.pop(0)
97
while not self.block_manager.can_append(seq_group):
124
- victim_seq_group = self.running[victim_idx]
125
- self._swap_out(victim_seq_group, blocks_to_swap_out)
126
- victim_idx -= 1
127
- if i > victim_idx:
128
- # No other sequence groups can be swapped out.
98
+ if self.running:
99
+ # Preempt the lowest-priority sequence groups.
100
+ victim_seq_group = self.running.pop(-1)
101
+ self._preempt(victim_seq_group, blocks_to_swap_out)
102
+ preempted.append(victim_seq_group)
103
+ else:
104
+ # No other sequence groups can be preempted.
105
+ # Preempt the current sequence group.
106
+ self._preempt(seq_group, blocks_to_swap_out)
107
+ preempted.append(seq_group)
108
break
109
else:
110
+ # Append new slots to the sequence group.
111
self._append(seq_group, blocks_to_copy)
132
- self.running = self.running[:victim_idx + 1]
133
-
134
- # 2. Swap in the swapped sequences if possible.
135
- # NOTE: Here we implicitly assume FCFS scheduling.
136
- # The swapped sequences are in LIFO order.
137
- for i, seq_group in enumerate(reversed(self.swapped)):
138
- if self.block_manager.can_swap_in(seq_group):
139
- self._swap_in(seq_group, blocks_to_swap_in)
140
- self._append(seq_group, blocks_to_copy)
141
- else:
142
- # OOM. Stop swapping.
143
- self.swapped = self.swapped[:len(self.swapped) - i]
112
+ running.append(seq_group)
113
+ self.running = running
114
+
115
+ # Swap in the sequence groups in the SWAPPED state if possible.
116
+ self.swapped = self.policy.sort_by_priority(now, self.swapped)
117
+ while self.swapped:
118
+ seq_group = self.swapped[0]
119
+ # If the sequence group has been preempted in this step, stop.
120
+ if seq_group in preempted:
121
+ break
122
+ # If the sequence group cannot be swapped in, stop.
123
+ if not self.block_manager.can_swap_in(seq_group):
124
break
145
- else:
146
- # All swapped sequences are swapped in.
147
- self.swapped.clear()
125
149
- # Ensure that swap-in and swap-out never happen at the same timestep.
150
- if blocks_to_swap_in:
151
- assert not blocks_to_swap_out
126
+ seq_group = self.swapped.pop(0)
127
+ self._swap_in(seq_group, blocks_to_swap_in)
128
+ self._append(seq_group, blocks_to_copy)
129
+ self.running.append(seq_group)
130
131
num_batched_tokens = sum(
132
seq_group.num_seqs(status=SequenceStatus.RUNNING)
133
for seq_group in self.running
134
)
135
158
- # 3. Join new sequences if possible.
159
- # NOTE: Here we implicitly assume FCFS scheduling.
160
- # TODO(woosuk): Add a batching policy to control the batch size.
136
+ # Join waiting sequences if possible.
137
+ prompt_group_ids: List[int] = []
138
+ # NOTE(woosuk): The sequence groups in the SWAPPED state are strictly
139
+ # prioritized over the sequence groups in the WAITING state.
140
+ # This is because we want to bound the amount of CPU memory taken by
141
+ # the swapped sequence groups.
142
if not self.swapped:
162
- for i, seq_group in enumerate(self.pending):
143
+ self.waiting = self.policy.sort_by_priority(now, self.waiting)
144
+ while self.waiting:
145
+ seq_group = self.waiting[0]
146
+ # If the sequence group has been preempted in this step, stop.
147
+ if seq_group in preempted:
148
+ break
149
+ # If the sequence group cannot be allocated, stop.
150
+ if not self.block_manager.can_allocate(seq_group):
151
+ break
152
+
153
+ # If the number of batched tokens exceeds the limit, stop.
154
num_prompt_tokens = seq_group.seqs[0].get_len()
164
- if self.block_manager.can_allocate(seq_group):
165
- if (num_batched_tokens + num_prompt_tokens
166
- <= self.max_num_batched_tokens):
167
- self._allocate(seq_group)
168
- num_batched_tokens += num_prompt_tokens
169
- continue
170
-
171
- self.pending = self.pending[i:]
172
- break
173
- else:
174
- self.pending.clear()
155
+ if (num_batched_tokens + num_prompt_tokens
156
+ > self.max_num_batched_tokens):
157
+ break
158
+
159
+ seq_group = self.waiting.pop(0)
160
+ self._allocate(seq_group)
161
+ self.running.append(seq_group)
162
+ num_batched_tokens += num_prompt_tokens
163
+ prompt_group_ids.append(seq_group.group_id)
164
176
- # 4. Create input data structures.
165
+ return (blocks_to_swap_in,
166
+ blocks_to_swap_out,
167
+ blocks_to_copy,
168
+ prompt_group_ids)
169
+
170
+ def step(self) -> List[SequenceGroup]:
171
+ # Schedule sequence groups.
172
+ # This function call changes the internal states of the scheduler
173
+ # such as self.running, self.swapped, and self.waiting.
174
+ scheduler_output = self._schedule()
175
+ blocks_to_swap_in = scheduler_output[0]
176
+ blocks_to_swap_out = scheduler_output[1]
177
+ blocks_to_copy = scheduler_output[2]
178
+ prompt_group_ids = scheduler_output[3]
179
+
180
+ # Create input data structures.
181
input_seq_groups: List[SequenceGroupInputs] = []
182
updated_seq_groups: List[SequenceGroup] = self.running.copy()
183
184
for seq_group in self.running:
185
group_id = seq_group.group_id
182
- num_steps = self.num_steps[group_id]
183
-
184
- # NOTE(woosuk): We assume that the number of steps is 0
185
- # for the prompt sequences.
186
- is_prompt = num_steps == 0
186
+ is_prompt = group_id in prompt_group_ids
187
188
input_tokens: Dict[int, List[int]] = {}
189
seq_logprobs: Dict[int, float] = {}
@@ -211,13 +211,15 @@ class Scheduler:
211
)
212
input_seq_groups.append(input_seq_group)
213
214
- # 5. Execute the first stage of the pipeline.
215
- if (input_seq_groups or blocks_to_swap_in or blocks_to_swap_out):
214
+ # Execute the first stage of the pipeline.
215
+ if input_seq_groups or blocks_to_swap_in or blocks_to_swap_out:
216
+ # Swap in and swap out should never happen at the same time.
217
+ assert not (blocks_to_swap_in and blocks_to_swap_out)
218
self.controllers[0].execute_stage(
219
input_seq_groups,
218
- blocks_to_swap_in,
219
- blocks_to_swap_out,
220
- blocks_to_copy,
220
+ blocks_to_swap_in=blocks_to_swap_in,
221
+ blocks_to_swap_out=blocks_to_swap_out,
222
+ blocks_to_copy=blocks_to_copy,
223
)
224
225
return updated_seq_groups
@@ -276,7 +278,106 @@ class Scheduler:
278
running.append(seq_group)
279
self.running = running
280
281
+ def _allocate(self, seq_group: SequenceGroup) -> None:
282
+ self.block_manager.allocate(seq_group)
283
+ for seq in seq_group.seqs:
284
+ seq.status = SequenceStatus.RUNNING
285
+ # FIXME(woosuk): Support interactive generation.
286
+ if seq_group.group_id not in self.num_steps:
287
+ self.num_steps[seq_group.group_id] = 0
288
+
289
+ def _append(
290
+ self,
291
+ seq_group: SequenceGroup,
292
+ blocks_to_copy: Dict[int, List[int]],
293
+ ) -> None:
294
+ for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
295
+ ret = self.block_manager.append(seq)
296
+ if ret is not None:
297
+ src_block, dst_block = ret
298
+ if src_block in blocks_to_copy:
299
+ blocks_to_copy[src_block].append(dst_block)
300
+ else:
301
+ blocks_to_copy[src_block] = [dst_block]
302
+
303
+ def _preempt(
304
+ self,
305
+ seq_group: SequenceGroup,
306
+ blocks_to_swap_out: Dict[int, int],
307
+ preemption_mode: Optional[PreemptionMode] = None,
308
+ ) -> None:
309
+ # If preemption mode is not specified, we determine the mode as follows:
310
+ # We use recomputation by default since it incurs lower overhead than
311
+ # swapping. However, when the sequence group has multiple sequences
312
+ # (e.g., beam search), recomputation is not supported. In such a case,
313
+ # we use swapping instead.
314
+ # FIXME(woosuk): This makes our scheduling policy a bit bizarre.
315
+ # As swapped sequences are prioritized over waiting sequences,
316
+ # sequence groups with multiple sequences are implicitly prioritized
317
+ # over sequence groups with a single sequence.
318
+ # TODO(woosuk): Support recomputation for sequence groups with multiple
319
+ # sequences. This may require a more sophisticated CUDA kernel.
320
+ if preemption_mode is None:
321
+ seqs = seq_group.get_seqs(status=SequenceStatus.RUNNING)
322
+ if len(seqs) == 1:
323
+ preemption_mode = PreemptionMode.RECOMPUTE
324
+ else:
325
+ preemption_mode = PreemptionMode.SWAP
326
+ if preemption_mode == PreemptionMode.RECOMPUTE:
327
+ self._preempt_by_recompute(seq_group)
328
+ elif preemption_mode == PreemptionMode.SWAP:
329
+ self._preempt_by_swap(seq_group, blocks_to_swap_out)
330
+ else:
331
+ assert False, 'Invalid preemption mode.'
332
+
333
+ def _preempt_by_recompute(
334
+ self,
335
+ seq_group: SequenceGroup,
336
+ ) -> None:
337
+ seqs = seq_group.get_seqs(status=SequenceStatus.RUNNING)
338
+ assert len(seqs) == 1
339
+ for seq in seqs:
340
+ seq.status = SequenceStatus.WAITING
341
+ self.block_manager.free(seq)
342
+ self.waiting.append(seq_group)
343
+
344
+ def _preempt_by_swap(
345
+ self,
346
+ seq_group: SequenceGroup,
347
+ blocks_to_swap_out: Dict[int, int],
348
+ ) -> None:
349
+ seqs = seq_group.get_seqs(status=SequenceStatus.RUNNING)
350
+ for seq in seqs:
351
+ seq.status = SequenceStatus.SWAPPED
352
+ self._swap_out(seq_group, blocks_to_swap_out)
353
+ self.swapped.append(seq_group)
354
+
355
+ def _free_seq(self, seq: Sequence) -> None:
356
+ seq.status = SequenceStatus.FINISHED
357
+ self.block_manager.free(seq)
358
+
359
def _free_seq_group(self, seq_group: SequenceGroup) -> None:
360
group_id = seq_group.group_id
361
del self.num_steps[group_id]
362
del self.sampling_params[group_id]
363
+
364
+ def _swap_in(
365
+ self,
366
+ seq_group: SequenceGroup,
367
+ blocks_to_swap_in: Dict[int, int],
368
+ ) -> None:
369
+ mapping = self.block_manager.swap_in(seq_group)
370
+ blocks_to_swap_in.update(mapping)
371
+ for seq in seq_group.get_seqs(status=SequenceStatus.SWAPPED):
372
+ seq.status = SequenceStatus.RUNNING
373
+
374
+ def _swap_out(
375
+ self,
376
+ seq_group: SequenceGroup,
377
+ blocks_to_swap_out: Dict[int, int],
378
+ ) -> None:
379
+ assert self.block_manager.can_swap_out(seq_group)
380
+ mapping = self.block_manager.swap_out(seq_group)
381
+ blocks_to_swap_out.update(mapping)
382
+ for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
383
+ seq.status = SequenceStatus.SWAPPED
cacheflow/master/server.py
+2
-1
@@ -10,6 +10,7 @@ from cacheflow.worker.controller import Controller, DeviceID
10
from cacheflow.sequence import SequenceGroup
11
from cacheflow.sampling_params import SamplingParams
12
13
+
14
class Server:
15
def __init__(
16
self,
@@ -91,7 +92,7 @@ class Server:
92
return self.scheduler.step()
93
94
def has_unfinished_requests(self):
94
- return (self.scheduler.pending or self.scheduler.running or
95
+ return (self.scheduler.waiting or self.scheduler.running or
96
self.scheduler.swapped)
97
98
cacheflow/master/simple_frontend.py
+3
-1
@@ -1,3 +1,4 @@
1
+import time
2
from typing import List, Optional, Set, Tuple
3
4
from transformers import AutoTokenizer
@@ -39,6 +40,7 @@ class SimpleFrontend:
40
token_ids: List[int],
41
sampling_params: SamplingParams,
42
) -> None:
43
+ arrival_time = time.time()
44
seqs: List[Sequence] = []
45
for _ in range(sampling_params.n):
46
seq_id = next(self.seq_counter)
@@ -46,7 +48,7 @@ class SimpleFrontend:
48
seqs.append(seq)
49
50
group_id = next(self.seq_group_counter)
49
- seq_group = SequenceGroup(group_id, seqs)
51
+ seq_group = SequenceGroup(group_id, seqs, arrival_time)
52
self.inputs.append((seq_group, sampling_params))
53
54
def get_inputs(self) -> List[Tuple[SequenceGroup, SamplingParams]]:
cacheflow/sequence.py
+4
-2
@@ -7,7 +7,7 @@ from cacheflow.sampling_params import SamplingParams
7
8
9
class SequenceStatus(enum.Enum):
10
- PENDING = enum.auto()
10
+ WAITING = enum.auto()
11
RUNNING = enum.auto()
12
SWAPPED = enum.auto()
13
FINISHED = enum.auto()
@@ -28,7 +28,7 @@ class Sequence:
28
# Initialize the logical token blocks with the given token ids.
29
self.add(token_ids)
30
31
- self.status = SequenceStatus.PENDING
31
+ self.status = SequenceStatus.WAITING
32
self.output_logprobs: List[Dict[int, float]] = []
33
self.cumulative_logprobs = 0.0
34
@@ -88,9 +88,11 @@ class SequenceGroup:
88
self,
89
group_id: int,
90
seqs: List[Sequence],
91
+ arrival_time: float,
92
) -> None:
93
self.group_id = group_id
94
self.seqs = seqs
95
+ self.arrival_time = arrival_time
96
97
def get_seqs(
98
self,