@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7a7929abe8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Implement preemption via recomputation & Refactor scheduling logic (#12)

Woosuk Kwon committed Mar 30, 2023 at 14:51 UTC 7a7929abe8e2fd6a4688487c471a1ee1fde0edd2
7 files changed +279 -126
cacheflow/http_frontend/fastapi_frontend.py
+2 -1
@@ -84,8 +84,9 @@ class FastAPIFrontend:
84 seq = Sequence(seq_id, token_ids, block_size=self.block_size)
85 seqs.append(seq)
86
87 + arrival_time = time.time()
88 group_id = next(self.seq_group_counter)
88 - seq_group = SequenceGroup(group_id, seqs)
89 + seq_group = SequenceGroup(group_id, seqs, arrival_time)
90 group_event = asyncio.Event()
91 self.sequence_group_events[group_id] = group_event
92 await self.server.add_sequence_groups.remote([(seq_group, sampling_params)])
cacheflow/master/block_manager.py
+2 -1
@@ -76,7 +76,8 @@ class BlockSpaceManager:
76 self.block_tables: Dict[int, BlockTable] = {}
77
78 def can_allocate(self, seq_group: SequenceGroup) -> bool:
79 - # NOTE: Here we assume that all sequences in the group have the same prompt.
79 + # FIXME(woosuk): Here we assume that all sequences in the group share
80 + # the same prompt. This may not be true for preempted sequences.
81 seq = seq_group.seqs[0]
82 num_required_blocks = len(seq.logical_token_blocks)
83 num_free_gpu_blocks = self.gpu_allocator.get_num_free_blocks()
cacheflow/master/policy.py new
+45
@@ -0,0 +1,45 @@
1 +from typing import List
2 +
3 +from cacheflow.sequence import SequenceGroup
4 +
5 +
6 +class Policy:
7 +
8 + def get_priority(
9 + self,
10 + now: float,
11 + seq_group: SequenceGroup,
12 + ) -> float:
13 + raise NotImplementedError
14 +
15 + def sort_by_priority(
16 + self,
17 + now: float,
18 + seq_groups: List[SequenceGroup],
19 + ) -> List[SequenceGroup]:
20 + return sorted(
21 + seq_groups,
22 + key=lambda seq_group: self.get_priority(now, seq_group),
23 + reverse=True,
24 + )
25 +
26 +
27 +class FCFS(Policy):
28 +
29 + def get_priority(
30 + self,
31 + now: float,
32 + seq_group: SequenceGroup,
33 + ) -> float:
34 + return now - seq_group.arrival_time
35 +
36 +
37 +class PolicyFactory:
38 +
39 + _POLICY_REGISTRY = {
40 + 'fcfs': FCFS,
41 + }
42 +
43 + @classmethod
44 + def get_policy(cls, policy_name: str, **kwargs) -> Policy:
45 + return cls._POLICY_REGISTRY[policy_name](**kwargs)
cacheflow/master/scheduler.py
+221 -120
@@ -1,6 +1,9 @@
1 -from typing import Dict, List, Tuple
1 +import enum
2 +import time
3 +from typing import Dict, List, Optional, Tuple
4
5 from cacheflow.master.block_manager import BlockSpaceManager
6 +from cacheflow.master.policy import PolicyFactory
7 from cacheflow.sampling_params import SamplingParams
8 from cacheflow.sequence import Sequence
9 from cacheflow.sequence import SequenceGroup
@@ -9,6 +12,19 @@ from cacheflow.sequence import SequenceOutputs
12 from cacheflow.sequence import SequenceStatus
13
14
15 +class PreemptionMode(enum.Enum):
16 + """Preemption modes.
17 +
18 + 1. Swapping: Swap out the blocks of the preempted sequences to CPU memory
19 + and swap them back in when the sequences are resumed.
20 + 2. Recomputation: Discard the blocks of the preempted sequences and
21 + recompute them when the sequences are resumed, treating the sequences as
22 + new prompts.
23 + """
24 + SWAP = enum.auto()
25 + RECOMPUTE = enum.auto()
26 +
27 +
28 class Scheduler:
29
30 def __init__(
@@ -25,6 +41,8 @@ class Scheduler:
41 self.num_cpu_blocks = num_cpu_blocks
42 self.max_num_batched_tokens = max_num_batched_tokens
43
44 + # Instantiate the scheduling policy.
45 + self.policy = PolicyFactory.get_policy(policy_name='fcfs')
46 # Create the block space manager.
47 self.block_manager = BlockSpaceManager(
48 block_size=block_size,
@@ -32,158 +50,140 @@ class Scheduler:
50 num_cpu_blocks=num_cpu_blocks,
51 )
52
35 - # Running sequence groups (FIFO).
53 + # Sequence groups in the WAITING state.
54 + self.waiting: List[SequenceGroup] = []
55 + # Sequence groups in the RUNNING state.
56 self.running: List[SequenceGroup] = []
57 # Mapping: group_id -> num_steps.
58 self.num_steps: Dict[int, int] = {}
59 # Mapping: group_id -> sampling params.
60 self.sampling_params: Dict[int, SamplingParams] = {}
41 -
42 - # Swapped sequence groups (LIFO).
61 + # Sequence groups in the SWAPPED state.
62 self.swapped: List[SequenceGroup] = []
44 - # Pending sequence groups (FIFO).
45 - self.pending: List[SequenceGroup] = []
63
64 def add_sequence_groups(
65 self,
49 - sequence_groups: List[Tuple[SequenceGroup, SamplingParams]],
66 + seq_groups: List[Tuple[SequenceGroup, SamplingParams]],
67 ) -> None:
51 - # Add sequence groups to the pending queue.
52 - for seq_group, sampling_params in sequence_groups:
53 - self.pending.append(seq_group)
68 + # Add sequence groups to the waiting queue.
69 + for seq_group, sampling_params in seq_groups:
70 + self.waiting.append(seq_group)
71 self.sampling_params[seq_group.group_id] = sampling_params
72
56 - def _free_seq(self, seq: Sequence) -> None:
57 - seq.status = SequenceStatus.FINISHED
58 - self.block_manager.free(seq)
59 -
60 - def _allocate(self, seq_group: SequenceGroup) -> None:
61 - self.block_manager.allocate(seq_group)
62 - for seq in seq_group.seqs:
63 - seq.status = SequenceStatus.RUNNING
64 - self.running.append(seq_group)
65 - # FIXME(woosuk): Support interactive generation.
66 - self.num_steps[seq_group.group_id] = 0
67 -
68 - def _append(
73 + def _schedule(
74 self,
70 - seq_group: SequenceGroup,
71 - blocks_to_copy: Dict[int, List[int]],
72 - ) -> None:
73 - for seq in seq_group.seqs:
74 - if seq.status == SequenceStatus.FINISHED:
75 - continue
76 - ret = self.block_manager.append(seq)
77 - if ret is not None:
78 - src_block, dst_block = ret
79 - if src_block in blocks_to_copy:
80 - blocks_to_copy[src_block].append(dst_block)
81 - else:
82 - blocks_to_copy[src_block] = [dst_block]
83 -
84 - def _swap_in(
85 - self,
86 - seq_group: SequenceGroup,
87 - blocks_to_swap_in: Dict[int, int],
88 - ) -> None:
89 - mapping = self.block_manager.swap_in(seq_group)
90 - blocks_to_swap_in.update(mapping)
91 - for seq in seq_group.get_seqs(status=SequenceStatus.SWAPPED):
92 - seq.status = SequenceStatus.RUNNING
93 - self.running.append(seq_group)
94 -
95 - def _swap_out(
96 - self,
97 - seq_group: SequenceGroup,
98 - blocks_to_swap_out: Dict[int, int],
99 - ) -> None:
100 - assert self.block_manager.can_swap_out(seq_group)
101 - mapping = self.block_manager.swap_out(seq_group)
102 - blocks_to_swap_out.update(mapping)
103 - for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
104 - seq.status = SequenceStatus.SWAPPED
105 - self.swapped.append(seq_group)
106 -
107 - def step(self) -> List[SequenceGroup]:
75 + ) -> Tuple[Dict[int, int], Dict[int, int], Dict[int, List[int]], List[int]]:
76 # Blocks that need to be swaped or copied before model execution.
77 blocks_to_swap_in: Dict[int, int] = {}
78 blocks_to_swap_out: Dict[int, int] = {}
79 blocks_to_copy: Dict[int, List[int]] = {}
80
113 - # 1. Reserve new slots for the running sequences.
114 - # NOTE: Here we implicitly assume FCFS scheduling.
115 - # That is, the most recently added sequence group is the first
116 - # to be swapped out.
117 - victim_idx = len(self.running) - 1
118 - for i, seq_group in enumerate(self.running):
119 - if i > victim_idx:
120 - # The i-th sequence group has already been swapped out.
121 - break
122 - # OOM. Swap out the victim sequence groups.
81 + # Fix the current time.
82 + now = time.time()
83 +
84 + # NOTE(woosuk): We prioritize the sequence groups in the RUNNING state
85 + # in order to minimize the preemption overheads.
86 + # Preemption happens only when there is no available slot to keep all
87 + # the sequence groups in the RUNNING state.
88 + # In this case, the policy is responsible for deciding which sequence
89 + # groups to preempt.
90 + self.running = self.policy.sort_by_priority(now, self.running)
91 +
92 + # Reserve new token slots for the running sequence groups.
93 + running: List[SequenceGroup] = []
94 + preempted: List[SequenceGroup] = []
95 + while self.running:
96 + seq_group = self.running.pop(0)
97 while not self.block_manager.can_append(seq_group):
124 - victim_seq_group = self.running[victim_idx]
125 - self._swap_out(victim_seq_group, blocks_to_swap_out)
126 - victim_idx -= 1
127 - if i > victim_idx:
128 - # No other sequence groups can be swapped out.
98 + if self.running:
99 + # Preempt the lowest-priority sequence groups.
100 + victim_seq_group = self.running.pop(-1)
101 + self._preempt(victim_seq_group, blocks_to_swap_out)
102 + preempted.append(victim_seq_group)
103 + else:
104 + # No other sequence groups can be preempted.
105 + # Preempt the current sequence group.
106 + self._preempt(seq_group, blocks_to_swap_out)
107 + preempted.append(seq_group)
108 break
109 else:
110 + # Append new slots to the sequence group.
111 self._append(seq_group, blocks_to_copy)
132 - self.running = self.running[:victim_idx + 1]
133 -
134 - # 2. Swap in the swapped sequences if possible.
135 - # NOTE: Here we implicitly assume FCFS scheduling.
136 - # The swapped sequences are in LIFO order.
137 - for i, seq_group in enumerate(reversed(self.swapped)):
138 - if self.block_manager.can_swap_in(seq_group):
139 - self._swap_in(seq_group, blocks_to_swap_in)
140 - self._append(seq_group, blocks_to_copy)
141 - else:
142 - # OOM. Stop swapping.
143 - self.swapped = self.swapped[:len(self.swapped) - i]
112 + running.append(seq_group)
113 + self.running = running
114 +
115 + # Swap in the sequence groups in the SWAPPED state if possible.
116 + self.swapped = self.policy.sort_by_priority(now, self.swapped)
117 + while self.swapped:
118 + seq_group = self.swapped[0]
119 + # If the sequence group has been preempted in this step, stop.
120 + if seq_group in preempted:
121 + break
122 + # If the sequence group cannot be swapped in, stop.
123 + if not self.block_manager.can_swap_in(seq_group):
124 break
145 - else:
146 - # All swapped sequences are swapped in.
147 - self.swapped.clear()
125
149 - # Ensure that swap-in and swap-out never happen at the same timestep.
150 - if blocks_to_swap_in:
151 - assert not blocks_to_swap_out
126 + seq_group = self.swapped.pop(0)
127 + self._swap_in(seq_group, blocks_to_swap_in)
128 + self._append(seq_group, blocks_to_copy)
129 + self.running.append(seq_group)
130
131 num_batched_tokens = sum(
132 seq_group.num_seqs(status=SequenceStatus.RUNNING)
133 for seq_group in self.running
134 )
135
158 - # 3. Join new sequences if possible.
159 - # NOTE: Here we implicitly assume FCFS scheduling.
160 - # TODO(woosuk): Add a batching policy to control the batch size.
136 + # Join waiting sequences if possible.
137 + prompt_group_ids: List[int] = []
138 + # NOTE(woosuk): The sequence groups in the SWAPPED state are strictly
139 + # prioritized over the sequence groups in the WAITING state.
140 + # This is because we want to bound the amount of CPU memory taken by
141 + # the swapped sequence groups.
142 if not self.swapped:
162 - for i, seq_group in enumerate(self.pending):
143 + self.waiting = self.policy.sort_by_priority(now, self.waiting)
144 + while self.waiting:
145 + seq_group = self.waiting[0]
146 + # If the sequence group has been preempted in this step, stop.
147 + if seq_group in preempted:
148 + break
149 + # If the sequence group cannot be allocated, stop.
150 + if not self.block_manager.can_allocate(seq_group):
151 + break
152 +
153 + # If the number of batched tokens exceeds the limit, stop.
154 num_prompt_tokens = seq_group.seqs[0].get_len()
164 - if self.block_manager.can_allocate(seq_group):
165 - if (num_batched_tokens + num_prompt_tokens
166 - <= self.max_num_batched_tokens):
167 - self._allocate(seq_group)
168 - num_batched_tokens += num_prompt_tokens
169 - continue
170 -
171 - self.pending = self.pending[i:]
172 - break
173 - else:
174 - self.pending.clear()
155 + if (num_batched_tokens + num_prompt_tokens
156 + > self.max_num_batched_tokens):
157 + break
158 +
159 + seq_group = self.waiting.pop(0)
160 + self._allocate(seq_group)
161 + self.running.append(seq_group)
162 + num_batched_tokens += num_prompt_tokens
163 + prompt_group_ids.append(seq_group.group_id)
164
176 - # 4. Create input data structures.
165 + return (blocks_to_swap_in,
166 + blocks_to_swap_out,
167 + blocks_to_copy,
168 + prompt_group_ids)
169 +
170 + def step(self) -> List[SequenceGroup]:
171 + # Schedule sequence groups.
172 + # This function call changes the internal states of the scheduler
173 + # such as self.running, self.swapped, and self.waiting.
174 + scheduler_output = self._schedule()
175 + blocks_to_swap_in = scheduler_output[0]
176 + blocks_to_swap_out = scheduler_output[1]
177 + blocks_to_copy = scheduler_output[2]
178 + prompt_group_ids = scheduler_output[3]
179 +
180 + # Create input data structures.
181 input_seq_groups: List[SequenceGroupInputs] = []
182 updated_seq_groups: List[SequenceGroup] = self.running.copy()
183
184 for seq_group in self.running:
185 group_id = seq_group.group_id
182 - num_steps = self.num_steps[group_id]
183 -
184 - # NOTE(woosuk): We assume that the number of steps is 0
185 - # for the prompt sequences.
186 - is_prompt = num_steps == 0
186 + is_prompt = group_id in prompt_group_ids
187
188 input_tokens: Dict[int, List[int]] = {}
189 seq_logprobs: Dict[int, float] = {}
@@ -211,13 +211,15 @@ class Scheduler:
211 )
212 input_seq_groups.append(input_seq_group)
213
214 - # 5. Execute the first stage of the pipeline.
215 - if (input_seq_groups or blocks_to_swap_in or blocks_to_swap_out):
214 + # Execute the first stage of the pipeline.
215 + if input_seq_groups or blocks_to_swap_in or blocks_to_swap_out:
216 + # Swap in and swap out should never happen at the same time.
217 + assert not (blocks_to_swap_in and blocks_to_swap_out)
218 self.controllers[0].execute_stage(
219 input_seq_groups,
218 - blocks_to_swap_in,
219 - blocks_to_swap_out,
220 - blocks_to_copy,
220 + blocks_to_swap_in=blocks_to_swap_in,
221 + blocks_to_swap_out=blocks_to_swap_out,
222 + blocks_to_copy=blocks_to_copy,
223 )
224
225 return updated_seq_groups
@@ -276,7 +278,106 @@ class Scheduler:
278 running.append(seq_group)
279 self.running = running
280
281 + def _allocate(self, seq_group: SequenceGroup) -> None:
282 + self.block_manager.allocate(seq_group)
283 + for seq in seq_group.seqs:
284 + seq.status = SequenceStatus.RUNNING
285 + # FIXME(woosuk): Support interactive generation.
286 + if seq_group.group_id not in self.num_steps:
287 + self.num_steps[seq_group.group_id] = 0
288 +
289 + def _append(
290 + self,
291 + seq_group: SequenceGroup,
292 + blocks_to_copy: Dict[int, List[int]],
293 + ) -> None:
294 + for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
295 + ret = self.block_manager.append(seq)
296 + if ret is not None:
297 + src_block, dst_block = ret
298 + if src_block in blocks_to_copy:
299 + blocks_to_copy[src_block].append(dst_block)
300 + else:
301 + blocks_to_copy[src_block] = [dst_block]
302 +
303 + def _preempt(
304 + self,
305 + seq_group: SequenceGroup,
306 + blocks_to_swap_out: Dict[int, int],
307 + preemption_mode: Optional[PreemptionMode] = None,
308 + ) -> None:
309 + # If preemption mode is not specified, we determine the mode as follows:
310 + # We use recomputation by default since it incurs lower overhead than
311 + # swapping. However, when the sequence group has multiple sequences
312 + # (e.g., beam search), recomputation is not supported. In such a case,
313 + # we use swapping instead.
314 + # FIXME(woosuk): This makes our scheduling policy a bit bizarre.
315 + # As swapped sequences are prioritized over waiting sequences,
316 + # sequence groups with multiple sequences are implicitly prioritized
317 + # over sequence groups with a single sequence.
318 + # TODO(woosuk): Support recomputation for sequence groups with multiple
319 + # sequences. This may require a more sophisticated CUDA kernel.
320 + if preemption_mode is None:
321 + seqs = seq_group.get_seqs(status=SequenceStatus.RUNNING)
322 + if len(seqs) == 1:
323 + preemption_mode = PreemptionMode.RECOMPUTE
324 + else:
325 + preemption_mode = PreemptionMode.SWAP
326 + if preemption_mode == PreemptionMode.RECOMPUTE:
327 + self._preempt_by_recompute(seq_group)
328 + elif preemption_mode == PreemptionMode.SWAP:
329 + self._preempt_by_swap(seq_group, blocks_to_swap_out)
330 + else:
331 + assert False, 'Invalid preemption mode.'
332 +
333 + def _preempt_by_recompute(
334 + self,
335 + seq_group: SequenceGroup,
336 + ) -> None:
337 + seqs = seq_group.get_seqs(status=SequenceStatus.RUNNING)
338 + assert len(seqs) == 1
339 + for seq in seqs:
340 + seq.status = SequenceStatus.WAITING
341 + self.block_manager.free(seq)
342 + self.waiting.append(seq_group)
343 +
344 + def _preempt_by_swap(
345 + self,
346 + seq_group: SequenceGroup,
347 + blocks_to_swap_out: Dict[int, int],
348 + ) -> None:
349 + seqs = seq_group.get_seqs(status=SequenceStatus.RUNNING)
350 + for seq in seqs:
351 + seq.status = SequenceStatus.SWAPPED
352 + self._swap_out(seq_group, blocks_to_swap_out)
353 + self.swapped.append(seq_group)
354 +
355 + def _free_seq(self, seq: Sequence) -> None:
356 + seq.status = SequenceStatus.FINISHED
357 + self.block_manager.free(seq)
358 +
359 def _free_seq_group(self, seq_group: SequenceGroup) -> None:
360 group_id = seq_group.group_id
361 del self.num_steps[group_id]
362 del self.sampling_params[group_id]
363 +
364 + def _swap_in(
365 + self,
366 + seq_group: SequenceGroup,
367 + blocks_to_swap_in: Dict[int, int],
368 + ) -> None:
369 + mapping = self.block_manager.swap_in(seq_group)
370 + blocks_to_swap_in.update(mapping)
371 + for seq in seq_group.get_seqs(status=SequenceStatus.SWAPPED):
372 + seq.status = SequenceStatus.RUNNING
373 +
374 + def _swap_out(
375 + self,
376 + seq_group: SequenceGroup,
377 + blocks_to_swap_out: Dict[int, int],
378 + ) -> None:
379 + assert self.block_manager.can_swap_out(seq_group)
380 + mapping = self.block_manager.swap_out(seq_group)
381 + blocks_to_swap_out.update(mapping)
382 + for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
383 + seq.status = SequenceStatus.SWAPPED
cacheflow/master/server.py
+2 -1
@@ -10,6 +10,7 @@ from cacheflow.worker.controller import Controller, DeviceID
10 from cacheflow.sequence import SequenceGroup
11 from cacheflow.sampling_params import SamplingParams
12
13 +
14 class Server:
15 def __init__(
16 self,
@@ -91,7 +92,7 @@ class Server:
92 return self.scheduler.step()
93
94 def has_unfinished_requests(self):
94 - return (self.scheduler.pending or self.scheduler.running or
95 + return (self.scheduler.waiting or self.scheduler.running or
96 self.scheduler.swapped)
97
98
cacheflow/master/simple_frontend.py
+3 -1
@@ -1,3 +1,4 @@
1 +import time
2 from typing import List, Optional, Set, Tuple
3
4 from transformers import AutoTokenizer
@@ -39,6 +40,7 @@ class SimpleFrontend:
40 token_ids: List[int],
41 sampling_params: SamplingParams,
42 ) -> None:
43 + arrival_time = time.time()
44 seqs: List[Sequence] = []
45 for _ in range(sampling_params.n):
46 seq_id = next(self.seq_counter)
@@ -46,7 +48,7 @@ class SimpleFrontend:
48 seqs.append(seq)
49
50 group_id = next(self.seq_group_counter)
49 - seq_group = SequenceGroup(group_id, seqs)
51 + seq_group = SequenceGroup(group_id, seqs, arrival_time)
52 self.inputs.append((seq_group, sampling_params))
53
54 def get_inputs(self) -> List[Tuple[SequenceGroup, SamplingParams]]:
cacheflow/sequence.py
+4 -2
@@ -7,7 +7,7 @@ from cacheflow.sampling_params import SamplingParams
7
8
9 class SequenceStatus(enum.Enum):
10 - PENDING = enum.auto()
10 + WAITING = enum.auto()
11 RUNNING = enum.auto()
12 SWAPPED = enum.auto()
13 FINISHED = enum.auto()
@@ -28,7 +28,7 @@ class Sequence:
28 # Initialize the logical token blocks with the given token ids.
29 self.add(token_ids)
30
31 - self.status = SequenceStatus.PENDING
31 + self.status = SequenceStatus.WAITING
32 self.output_logprobs: List[Dict[int, float]] = []
33 self.cumulative_logprobs = 0.0
34
@@ -88,9 +88,11 @@ class SequenceGroup:
88 self,
89 group_id: int,
90 seqs: List[Sequence],
91 + arrival_time: float,
92 ) -> None:
93 self.group_id = group_id
94 self.seqs = seqs
95 + self.arrival_time = arrival_time
96
97 def get_seqs(
98 self,