@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5a309bb588
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Add scheduler

Woosuk Kwon committed Feb 13, 2023 at 02:39 UTC 5a309bb5887eeba8d47f07854c476870f3b40f03
1 file changed +170
cacheflow/master/scheduler.py new
+170
@@ -0,0 +1,170 @@
1 +from typing import Dict, List, Tuple
2 +
3 +from cacheflow.master.block_manager import BlockSpaceManager
4 +from cacheflow.sequence import Sequence
5 +from cacheflow.sequence import SequenceGroup
6 +from cacheflow.sequence import SequenceStatus
7 +
8 +
9 +class Scheduler:
10 +
11 + def __int__(
12 + self,
13 + block_size: int,
14 + num_gpu_blocks: int,
15 + num_cpu_blocks: int,
16 + ) -> None:
17 + self.block_manager = BlockSpaceManager(
18 + block_size=block_size,
19 + num_gpu_blocks=num_gpu_blocks,
20 + num_cpu_blocks=num_cpu_blocks,
21 + )
22 +
23 + # Serving sequence groups (FIFO).
24 + self.serving: List[SequenceGroup] = []
25 + # Mapping: group_id -> num_steps.
26 + self.num_steps: Dict[int, int] = {}
27 + # Mapping: group_id -> max_num_steps.
28 + self.max_num_steps: Dict[int, int] = {}
29 + # Mapping: group_id -> stop_token_ids.
30 + self.stop_token_ids: Dict[int, List[int]] = {}
31 +
32 + # Swapped sequence groups (LIFO).
33 + self.swapped: List[SequenceGroup] = []
34 +
35 + # Pending sequence groups (FIFO).
36 + self.queue: List[SequenceGroup] = []
37 +
38 + def _free_seq(self, seq: Sequence) -> None:
39 + seq.status = SequenceStatus.FINISHED
40 + self.block_manager.free(seq)
41 +
42 + def _allocate(self, seq_group: SequenceGroup) -> None:
43 + self.block_manager.allocate(seq_group)
44 + for seq in seq_group.seqs:
45 + seq.status = SequenceStatus.RUNNING
46 + self.serving.append(seq_group)
47 +
48 + def _append(self, seq_group: SequenceGroup) -> None:
49 + for seq in seq_group.seqs:
50 + if seq.status == SequenceStatus.FINISHED:
51 + continue
52 + ret = self.block_manager.append(seq)
53 + if ret is not None:
54 + src_block, dst_block = ret
55 + # TODO: Issue COPY commands to the workers.
56 +
57 + def _swap_in(self, seq_group: SequenceGroup) -> None:
58 + # TODO: Issue SWAP_IN commands to the workers.
59 + self.block_manager.swap_in(seq_group)
60 + self.block_manager.append(seq_group)
61 + for seq in seq_group.seqs:
62 + if seq.status == SequenceStatus.SWAPPED:
63 + seq.status = SequenceStatus.RUNNING
64 + self.serving.append(seq_group)
65 +
66 + def _swap_out(self, seq_group: SequenceGroup) -> None:
67 + assert self.block_manager.can_swap_out(seq_group)
68 + # TODO: Issue SWAP_OUT commands to the workers.
69 + self.block_manager.swap_out(seq_group)
70 + for seq in seq_group.seqs:
71 + if seq.status == SequenceStatus.RUNNING:
72 + seq.status = SequenceStatus.SWAPPED
73 + self.swapped.append(seq_group)
74 +
75 + def step(self) -> None:
76 + # 1. Prepare new slots for the running sequences.
77 + # NOTE: Here we implicitly assume FCFS scheduling.
78 + # That is, the most recently added sequence group is the first
79 + # to be swapped out.
80 + victim_idx = len(self.serving) - 1
81 + for i, seq_group in enumerate(self.serving):
82 + if i > victim_idx:
83 + # The i-th sequence group has already been swapped out.
84 + break
85 + # OOM. Swap out the victim sequence groups.
86 + while not self.block_manager.can_append(seq_group):
87 + victim_seq_group = self.serving[victim_idx]
88 + self._swap_out(victim_seq_group)
89 + victim_idx -= 1
90 + if i > victim_idx:
91 + # No other sequence groups can be swapped out.
92 + break
93 + else:
94 + self._append(seq_group)
95 + self.serving = self.serving[:victim_idx + 1]
96 +
97 + # 2. Swap in the swapped sequences if possible.
98 + # NOTE: Here we implicitly assume FCFS scheduling.
99 + # The swapped sequences are in LIFO order.
100 + for i, seq_group in enumerate(reversed(self.swapped)):
101 + if self.block_manager.can_swap_in(seq_group):
102 + self._swap_in(seq_group)
103 + else:
104 + # OOM. Stop swapping.
105 + self.swapped = self.swapped[:len(self.swapped) - i]
106 + break
107 + else:
108 + # All swapped sequences are swapped in.
109 + self.swapped.clear()
110 +
111 + # 3. Join new sequences if possible.
112 + # NOTE: Here we implicitly assume FCFS scheduling.
113 + # TODO(woosuk): Add a heuristic to control the maximum batch size.
114 + if not self.swapped:
115 + for i, seq_group in enumerate(self.queue):
116 + if self.block_manager.can_allocate(seq_group):
117 + self._allocate(seq_group)
118 + else:
119 + # FIXME: Consider the race condition.
120 + self.queue = self.queue[i:]
121 + break
122 +
123 + def post_step(
124 + self,
125 + next_tokens: Dict[int, Tuple[int, int]],
126 + ) -> None:
127 + # Update the running sequences and free blocks.
128 + for seq_group in self.serving:
129 + group_id = seq_group.group_id
130 + self.num_steps[group_id] += 1
131 + stop_token_ids = self.stop_token_ids[group_id]
132 +
133 + for seq in seq_group.seqs:
134 + if seq.status == SequenceStatus.FINISHED:
135 + continue
136 +
137 + parent_seq_id, next_token = next_tokens[seq.seq_id]
138 + if seq.seq_id != parent_seq_id:
139 + # The sequence is a fork of the parent sequence (beam search).
140 + # Free the current sequence.
141 + self.block_manager.free(seq)
142 + # Fork the parent sequence.
143 + parent_seq = seq_group.find(parent_seq_id)
144 + seq.logical_token_blocks = parent_seq.logical_token_blocks.copy()
145 + self.block_manager.fork(parent_seq, seq)
146 +
147 + # Append a new token to the sequence.
148 + seq.append(next_token)
149 +
150 + # Check if the sequence has generated a stop token.
151 + if next_token in stop_token_ids:
152 + self._free_seq(seq)
153 + continue
154 +
155 + # Check if the sequence has reached the maximum number of steps.
156 + if self.num_steps[group_id] == self.max_num_steps[group_id]:
157 + self._free_seq(seq)
158 + continue
159 +
160 + # Update the serving states.
161 + serving: List[SequenceGroup] = []
162 + for seq_group in self.serving:
163 + if seq_group.num_seqs(status=SequenceStatus.RUNNING) == 0:
164 + del self.num_steps[seq_group.group_id]
165 + del self.max_num_steps[seq_group.group_id]
166 + del self.stop_token_ids[seq_group.group_id]
167 + # TODO: Return the seq_group to the client.
168 + else:
169 + serving.append(seq_group)
170 + self.serving = serving