A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Merge pre_step and step
Woosuk Kwon committed
Feb 24, 2023 at 10:36 UTC
ef6098ec51fcd07da3b93704e2c48e1f861ff00b
1 file changed
+4
-13
cacheflow/master/scheduler.py
+4
-13
@@ -93,13 +93,13 @@ class Scheduler:
93
seq.status = SequenceStatus.SWAPPED
94
self.swapped.append(seq_group)
95
96
- def pre_step(self) -> None:
96
+ def step(self) -> None:
97
# Blocks that need to be swaped or copied before model execution.
98
blocks_to_swap_in: Dict[int, int] = {}
99
blocks_to_swap_out: Dict[int, int] = {}
100
blocks_to_copy: Dict[int, int] = {}
101
102
- # 1. Prepare new slots for the running sequences.
102
+ # 1. Reserve new slots for the running sequences.
103
# NOTE: Here we implicitly assume FCFS scheduling.
104
# That is, the most recently added sequence group is the first
105
# to be swapped out.
@@ -159,20 +159,11 @@ class Scheduler:
159
else:
160
self.pending.clear()
161
162
- # Execute step.
163
- self.step(blocks_to_swap_in, blocks_to_swap_out, blocks_to_copy)
164
-
165
- def step(
166
- self,
167
- blocks_to_swap_in: Dict[int, int],
168
- blocks_to_swap_out: Dict[int, int],
169
- blocks_to_copy: Dict[int, int],
170
- ) -> None:
162
# Ensure that swap-in and swap-out never happen at the same timestep.
163
if blocks_to_swap_in:
164
assert not blocks_to_swap_out
165
175
- # Create input data structures.
166
+ # 4. Create input data structures.
167
prompt_tokens: Dict[int, List[int]] = {}
168
generation_tokens: Dict[int, int] = {}
169
context_lens: Dict[int, int] = {}
@@ -195,7 +186,7 @@ class Scheduler:
186
generation_tokens[seq_id] = seq.get_token_ids()[-1]
187
context_lens[seq_id] = seq.get_len()
188
198
- # Execute the first stage of the pipeline.
189
+ # 5. Execute the first stage of the pipeline.
190
self.controllers[0].execute_stage(
191
prompt_tokens,
192
generation_tokens,