A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Avoid sorting waiting queue & Minor code cleaning (#93)
Woosuk Kwon committed
May 10, 2023 at 01:57 UTC
ae356774ab60af7ffa665a7b80e28bd73a9a7f48
4 files changed
+10
-16
cacheflow/core/scheduler.py
+10
-13
@@ -1,21 +1,16 @@
1
import enum
2
-import os
3
-import pickle
2
import time
5
-from typing import Any, Dict, List, Optional, Tuple
3
+from typing import Dict, List, Optional, Tuple
4
5
from cacheflow.core.block_manager import BlockSpaceManager
6
from cacheflow.logger import init_logger
7
from cacheflow.core.policy import PolicyFactory
8
from cacheflow.sampling_params import SamplingParams
11
-from cacheflow.sequence import Sequence
12
-from cacheflow.sequence import SequenceGroup
13
-from cacheflow.sequence import SequenceGroupMetadata
14
-from cacheflow.sequence import SequenceOutputs
15
-from cacheflow.sequence import SequenceStatus
16
-
9
+from cacheflow.sequence import (Sequence, SequenceGroup, SequenceGroupMetadata,
10
+ SequenceOutputs, SequenceStatus)
11
12
logger = init_logger(__name__)
13
+
14
_LOGGING_INTERVAL_SEC = 10
15
16
@@ -129,7 +124,6 @@ class Scheduler:
124
125
# Swap in the sequence groups in the SWAPPED state if possible.
126
self.swapped = self.policy.sort_by_priority(now, self.swapped)
132
- # FCFS
127
while self.swapped and not blocks_to_swap_out:
128
seq_group = self.swapped[0]
129
# If the sequence group has been preempted in this step, stop.
@@ -162,7 +156,9 @@ class Scheduler:
156
# This is because we want to bound the amount of CPU memory taken by
157
# the swapped sequence groups.
158
if not self.swapped:
165
- self.waiting = self.policy.sort_by_priority(now, self.waiting)
159
+ # Optimization: We do not sort the waiting queue since the preempted
160
+ # sequence groups are added to the front and the new sequence groups
161
+ # are added to the back.
162
while self.waiting:
163
seq_group = self.waiting[0]
164
# If the sequence group has been preempted in this step, stop.
@@ -347,7 +343,6 @@ class Scheduler:
343
self.block_manager.allocate(seq_group)
344
for seq in seq_group.seqs:
345
seq.status = SequenceStatus.RUNNING
350
- # FIXME(woosuk): Support interactive generation.
346
if seq_group.group_id not in self.num_steps:
347
self.num_steps[seq_group.group_id] = 0
348
@@ -404,7 +399,9 @@ class Scheduler:
399
for seq in seqs:
400
seq.status = SequenceStatus.WAITING
401
self.block_manager.free(seq)
407
- self.waiting.append(seq_group)
402
+ # NOTE: For FCFS, we insert the preempted sequence group to the front
403
+ # of the waiting queue.
404
+ self.waiting.insert(0, seq_group)
405
406
def _preempt_by_swap(
407
self,
cacheflow/core/server.py
-1
@@ -17,7 +17,6 @@ from cacheflow.sequence import SequenceGroup
17
from cacheflow.utils import get_gpu_memory, get_cpu_memory
18
from cacheflow.worker.controller import Controller, DeviceID
19
20
-
20
logger = init_logger(__name__)
21
22
cacheflow/frontend/simple_frontend.py
-1
@@ -7,7 +7,6 @@ from cacheflow.sampling_params import SamplingParams
7
from cacheflow.sequence import Sequence, SequenceGroup
8
from cacheflow.utils import Counter
9
10
-
10
logger = init_logger(__name__)
11
12
cacheflow/model_executor/memory_analyzer.py
-1
@@ -4,7 +4,6 @@ from transformers import AutoConfig
4
from cacheflow.logger import init_logger
5
from cacheflow.model_executor.utils import get_dtype_size
6
7
-
7
logger = init_logger(__name__)
8
9
_GiB = 1 << 30