@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 05921a9a7a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Changed scheduler to use deques instead of lists (#2290)

Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Nadav Shmayovits committed Jan 7, 2024 at 19:48 UTC 05921a9a7ab2bd5694d28dd5a986b8aa00ddbbd1
3 files changed +28 -24
vllm/core/policy.py
+10 -8
@@ -1,4 +1,5 @@
1 -from typing import List
1 +from collections import deque
2 +from typing import Deque
3
4 from vllm.sequence import SequenceGroup
5
@@ -15,13 +16,14 @@ class Policy:
16 def sort_by_priority(
17 self,
18 now: float,
18 - seq_groups: List[SequenceGroup],
19 - ) -> List[SequenceGroup]:
20 - return sorted(
21 - seq_groups,
22 - key=lambda seq_group: self.get_priority(now, seq_group),
23 - reverse=True,
24 - )
19 + seq_groups: Deque[SequenceGroup],
20 + ) -> Deque[SequenceGroup]:
21 + return deque(
22 + sorted(
23 + seq_groups,
24 + key=lambda seq_group: self.get_priority(now, seq_group),
25 + reverse=True,
26 + ))
27
28
29 class FCFS(Policy):
vllm/core/scheduler.py
+14 -14
@@ -1,6 +1,7 @@
1 +from collections import deque
2 import enum
3 import time
3 -from typing import Dict, Iterable, List, Optional, Tuple, Union
4 +from typing import Deque, Dict, Iterable, List, Optional, Tuple, Union
5
6 from vllm.config import CacheConfig, SchedulerConfig
7 from vllm.core.block_manager import AllocStatus, BlockSpaceManager
@@ -29,7 +30,7 @@ class SchedulerOutputs:
30
31 def __init__(
32 self,
32 - scheduled_seq_groups: List[SequenceGroup],
33 + scheduled_seq_groups: Iterable[SequenceGroup],
34 prompt_run: bool,
35 num_batched_tokens: int,
36 blocks_to_swap_in: Dict[int, int],
@@ -75,13 +76,12 @@ class Scheduler:
76 num_cpu_blocks=self.cache_config.num_cpu_blocks,
77 sliding_window=self.cache_config.sliding_window)
78
78 - # TODO(zhuohan): Use deque instead of list for better performance.
79 # Sequence groups in the WAITING state.
80 - self.waiting: List[SequenceGroup] = []
80 + self.waiting: Deque[SequenceGroup] = deque()
81 # Sequence groups in the RUNNING state.
82 - self.running: List[SequenceGroup] = []
82 + self.running: Deque[SequenceGroup] = deque()
83 # Sequence groups in the SWAPPED state.
84 - self.swapped: List[SequenceGroup] = []
84 + self.swapped: Deque[SequenceGroup] = deque()
85
86 def add_seq_group(self, seq_group: SequenceGroup) -> None:
87 # Add sequence groups to the waiting queue.
@@ -152,7 +152,7 @@ class Scheduler:
152 for seq in waiting_seqs:
153 seq.status = SequenceStatus.FINISHED_IGNORED
154 ignored_seq_groups.append(seq_group)
155 - self.waiting.pop(0)
155 + self.waiting.popleft()
156 continue
157
158 # If the sequence group cannot be allocated, stop.
@@ -166,7 +166,7 @@ class Scheduler:
166 for seq in waiting_seqs:
167 seq.status = SequenceStatus.FINISHED_IGNORED
168 ignored_seq_groups.append(seq_group)
169 - self.waiting.pop(0)
169 + self.waiting.popleft()
170 continue
171
172 # If the number of batched tokens exceeds the limit, stop.
@@ -188,7 +188,7 @@ class Scheduler:
188 break
189 seq_lens = new_seq_lens
190
191 - seq_group = self.waiting.pop(0)
191 + seq_group = self.waiting.popleft()
192 self._allocate(seq_group)
193 self.running.append(seq_group)
194 num_curr_seqs += num_new_seqs
@@ -214,14 +214,14 @@ class Scheduler:
214 self.running = self.policy.sort_by_priority(now, self.running)
215
216 # Reserve new token slots for the running sequence groups.
217 - running: List[SequenceGroup] = []
217 + running: Deque[SequenceGroup] = deque()
218 preempted: List[SequenceGroup] = []
219 while self.running:
220 - seq_group = self.running.pop(0)
220 + seq_group = self.running.popleft()
221 while not self.block_manager.can_append_slot(seq_group):
222 if self.running:
223 # Preempt the lowest-priority sequence groups.
224 - victim_seq_group = self.running.pop(-1)
224 + victim_seq_group = self.running.pop()
225 self._preempt(victim_seq_group, blocks_to_swap_out)
226 preempted.append(victim_seq_group)
227 else:
@@ -255,7 +255,7 @@ class Scheduler:
255 self.scheduler_config.max_num_seqs):
256 break
257
258 - seq_group = self.swapped.pop(0)
258 + seq_group = self.swapped.popleft()
259 self._swap_in(seq_group, blocks_to_swap_in)
260 self._append_slot(seq_group, blocks_to_copy)
261 num_curr_seqs += num_new_seqs
@@ -376,7 +376,7 @@ class Scheduler:
376 self.block_manager.free(seq)
377 # NOTE: For FCFS, we insert the preempted sequence group to the front
378 # of the waiting queue.
379 - self.waiting.insert(0, seq_group)
379 + self.waiting.appendleft(seq_group)
380
381 def _preempt_by_swap(
382 self,
vllm/engine/llm_engine.py
+4 -2
@@ -601,8 +601,10 @@ class LLMEngine:
601
602 # Create the outputs.
603 request_outputs: List[RequestOutput] = []
604 - for seq_group in (scheduled_seq_groups +
605 - scheduler_outputs.ignored_seq_groups):
604 + for seq_group in scheduled_seq_groups:
605 + request_output = RequestOutput.from_seq_group(seq_group)
606 + request_outputs.append(request_output)
607 + for seq_group in scheduler_outputs.ignored_seq_groups:
608 request_output = RequestOutput.from_seq_group(seq_group)
609 request_outputs.append(request_output)
610