@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ef6098ec51
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Merge pre_step and step

Woosuk Kwon committed Feb 24, 2023 at 10:36 UTC ef6098ec51fcd07da3b93704e2c48e1f861ff00b
1 file changed +4 -13
cacheflow/master/scheduler.py
+4 -13
@@ -93,13 +93,13 @@ class Scheduler:
93 seq.status = SequenceStatus.SWAPPED
94 self.swapped.append(seq_group)
95
96 - def pre_step(self) -> None:
96 + def step(self) -> None:
97 # Blocks that need to be swaped or copied before model execution.
98 blocks_to_swap_in: Dict[int, int] = {}
99 blocks_to_swap_out: Dict[int, int] = {}
100 blocks_to_copy: Dict[int, int] = {}
101
102 - # 1. Prepare new slots for the running sequences.
102 + # 1. Reserve new slots for the running sequences.
103 # NOTE: Here we implicitly assume FCFS scheduling.
104 # That is, the most recently added sequence group is the first
105 # to be swapped out.
@@ -159,20 +159,11 @@ class Scheduler:
159 else:
160 self.pending.clear()
161
162 - # Execute step.
163 - self.step(blocks_to_swap_in, blocks_to_swap_out, blocks_to_copy)
164 -
165 - def step(
166 - self,
167 - blocks_to_swap_in: Dict[int, int],
168 - blocks_to_swap_out: Dict[int, int],
169 - blocks_to_copy: Dict[int, int],
170 - ) -> None:
162 # Ensure that swap-in and swap-out never happen at the same timestep.
163 if blocks_to_swap_in:
164 assert not blocks_to_swap_out
165
175 - # Create input data structures.
166 + # 4. Create input data structures.
167 prompt_tokens: Dict[int, List[int]] = {}
168 generation_tokens: Dict[int, int] = {}
169 context_lens: Dict[int, int] = {}
@@ -195,7 +186,7 @@ class Scheduler:
186 generation_tokens[seq_id] = seq.get_token_ids()[-1]
187 context_lens[seq_id] = seq.get_len()
188
198 - # Execute the first stage of the pipeline.
189 + # 5. Execute the first stage of the pipeline.
190 self.controllers[0].execute_stage(
191 prompt_tokens,
192 generation_tokens,