A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Changed scheduler to use deques instead of lists (#2290)
Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Nadav Shmayovits committed
Jan 7, 2024 at 19:48 UTC
05921a9a7ab2bd5694d28dd5a986b8aa00ddbbd1
3 files changed
+28
-24
vllm/core/policy.py
+10
-8
@@ -1,4 +1,5 @@
1
-from typing import List
1
+from collections import deque
2
+from typing import Deque
3
4
from vllm.sequence import SequenceGroup
5
@@ -15,13 +16,14 @@ class Policy:
16
def sort_by_priority(
17
self,
18
now: float,
18
- seq_groups: List[SequenceGroup],
19
- ) -> List[SequenceGroup]:
20
- return sorted(
21
- seq_groups,
22
- key=lambda seq_group: self.get_priority(now, seq_group),
23
- reverse=True,
24
- )
19
+ seq_groups: Deque[SequenceGroup],
20
+ ) -> Deque[SequenceGroup]:
21
+ return deque(
22
+ sorted(
23
+ seq_groups,
24
+ key=lambda seq_group: self.get_priority(now, seq_group),
25
+ reverse=True,
26
+ ))
27
28
29
class FCFS(Policy):
vllm/core/scheduler.py
+14
-14
@@ -1,6 +1,7 @@
1
+from collections import deque
2
import enum
3
import time
3
-from typing import Dict, Iterable, List, Optional, Tuple, Union
4
+from typing import Deque, Dict, Iterable, List, Optional, Tuple, Union
5
6
from vllm.config import CacheConfig, SchedulerConfig
7
from vllm.core.block_manager import AllocStatus, BlockSpaceManager
@@ -29,7 +30,7 @@ class SchedulerOutputs:
30
31
def __init__(
32
self,
32
- scheduled_seq_groups: List[SequenceGroup],
33
+ scheduled_seq_groups: Iterable[SequenceGroup],
34
prompt_run: bool,
35
num_batched_tokens: int,
36
blocks_to_swap_in: Dict[int, int],
@@ -75,13 +76,12 @@ class Scheduler:
76
num_cpu_blocks=self.cache_config.num_cpu_blocks,
77
sliding_window=self.cache_config.sliding_window)
78
78
- # TODO(zhuohan): Use deque instead of list for better performance.
79
# Sequence groups in the WAITING state.
80
- self.waiting: List[SequenceGroup] = []
80
+ self.waiting: Deque[SequenceGroup] = deque()
81
# Sequence groups in the RUNNING state.
82
- self.running: List[SequenceGroup] = []
82
+ self.running: Deque[SequenceGroup] = deque()
83
# Sequence groups in the SWAPPED state.
84
- self.swapped: List[SequenceGroup] = []
84
+ self.swapped: Deque[SequenceGroup] = deque()
85
86
def add_seq_group(self, seq_group: SequenceGroup) -> None:
87
# Add sequence groups to the waiting queue.
@@ -152,7 +152,7 @@ class Scheduler:
152
for seq in waiting_seqs:
153
seq.status = SequenceStatus.FINISHED_IGNORED
154
ignored_seq_groups.append(seq_group)
155
- self.waiting.pop(0)
155
+ self.waiting.popleft()
156
continue
157
158
# If the sequence group cannot be allocated, stop.
@@ -166,7 +166,7 @@ class Scheduler:
166
for seq in waiting_seqs:
167
seq.status = SequenceStatus.FINISHED_IGNORED
168
ignored_seq_groups.append(seq_group)
169
- self.waiting.pop(0)
169
+ self.waiting.popleft()
170
continue
171
172
# If the number of batched tokens exceeds the limit, stop.
@@ -188,7 +188,7 @@ class Scheduler:
188
break
189
seq_lens = new_seq_lens
190
191
- seq_group = self.waiting.pop(0)
191
+ seq_group = self.waiting.popleft()
192
self._allocate(seq_group)
193
self.running.append(seq_group)
194
num_curr_seqs += num_new_seqs
@@ -214,14 +214,14 @@ class Scheduler:
214
self.running = self.policy.sort_by_priority(now, self.running)
215
216
# Reserve new token slots for the running sequence groups.
217
- running: List[SequenceGroup] = []
217
+ running: Deque[SequenceGroup] = deque()
218
preempted: List[SequenceGroup] = []
219
while self.running:
220
- seq_group = self.running.pop(0)
220
+ seq_group = self.running.popleft()
221
while not self.block_manager.can_append_slot(seq_group):
222
if self.running:
223
# Preempt the lowest-priority sequence groups.
224
- victim_seq_group = self.running.pop(-1)
224
+ victim_seq_group = self.running.pop()
225
self._preempt(victim_seq_group, blocks_to_swap_out)
226
preempted.append(victim_seq_group)
227
else:
@@ -255,7 +255,7 @@ class Scheduler:
255
self.scheduler_config.max_num_seqs):
256
break
257
258
- seq_group = self.swapped.pop(0)
258
+ seq_group = self.swapped.popleft()
259
self._swap_in(seq_group, blocks_to_swap_in)
260
self._append_slot(seq_group, blocks_to_copy)
261
num_curr_seqs += num_new_seqs
@@ -376,7 +376,7 @@ class Scheduler:
376
self.block_manager.free(seq)
377
# NOTE: For FCFS, we insert the preempted sequence group to the front
378
# of the waiting queue.
379
- self.waiting.insert(0, seq_group)
379
+ self.waiting.appendleft(seq_group)
380
381
def _preempt_by_swap(
382
self,
vllm/engine/llm_engine.py
+4
-2
@@ -601,8 +601,10 @@ class LLMEngine:
601
602
# Create the outputs.
603
request_outputs: List[RequestOutput] = []
604
- for seq_group in (scheduled_seq_groups +
605
- scheduler_outputs.ignored_seq_groups):
604
+ for seq_group in scheduled_seq_groups:
605
+ request_output = RequestOutput.from_seq_group(seq_group)
606
+ request_outputs.append(request_output)
607
+ for seq_group in scheduler_outputs.ignored_seq_groups:
608
request_output = RequestOutput.from_seq_group(seq_group)
609
request_outputs.append(request_output)
610