@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ae356774ab
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Avoid sorting waiting queue & Minor code cleaning (#93)

Woosuk Kwon committed May 10, 2023 at 01:57 UTC ae356774ab60af7ffa665a7b80e28bd73a9a7f48
4 files changed +10 -16
cacheflow/core/scheduler.py
+10 -13
@@ -1,21 +1,16 @@
1 import enum
2 -import os
3 -import pickle
2 import time
5 -from typing import Any, Dict, List, Optional, Tuple
3 +from typing import Dict, List, Optional, Tuple
4
5 from cacheflow.core.block_manager import BlockSpaceManager
6 from cacheflow.logger import init_logger
7 from cacheflow.core.policy import PolicyFactory
8 from cacheflow.sampling_params import SamplingParams
11 -from cacheflow.sequence import Sequence
12 -from cacheflow.sequence import SequenceGroup
13 -from cacheflow.sequence import SequenceGroupMetadata
14 -from cacheflow.sequence import SequenceOutputs
15 -from cacheflow.sequence import SequenceStatus
16 -
9 +from cacheflow.sequence import (Sequence, SequenceGroup, SequenceGroupMetadata,
10 + SequenceOutputs, SequenceStatus)
11
12 logger = init_logger(__name__)
13 +
14 _LOGGING_INTERVAL_SEC = 10
15
16
@@ -129,7 +124,6 @@ class Scheduler:
124
125 # Swap in the sequence groups in the SWAPPED state if possible.
126 self.swapped = self.policy.sort_by_priority(now, self.swapped)
132 - # FCFS
127 while self.swapped and not blocks_to_swap_out:
128 seq_group = self.swapped[0]
129 # If the sequence group has been preempted in this step, stop.
@@ -162,7 +156,9 @@ class Scheduler:
156 # This is because we want to bound the amount of CPU memory taken by
157 # the swapped sequence groups.
158 if not self.swapped:
165 - self.waiting = self.policy.sort_by_priority(now, self.waiting)
159 + # Optimization: We do not sort the waiting queue since the preempted
160 + # sequence groups are added to the front and the new sequence groups
161 + # are added to the back.
162 while self.waiting:
163 seq_group = self.waiting[0]
164 # If the sequence group has been preempted in this step, stop.
@@ -347,7 +343,6 @@ class Scheduler:
343 self.block_manager.allocate(seq_group)
344 for seq in seq_group.seqs:
345 seq.status = SequenceStatus.RUNNING
350 - # FIXME(woosuk): Support interactive generation.
346 if seq_group.group_id not in self.num_steps:
347 self.num_steps[seq_group.group_id] = 0
348
@@ -404,7 +399,9 @@ class Scheduler:
399 for seq in seqs:
400 seq.status = SequenceStatus.WAITING
401 self.block_manager.free(seq)
407 - self.waiting.append(seq_group)
402 + # NOTE: For FCFS, we insert the preempted sequence group to the front
403 + # of the waiting queue.
404 + self.waiting.insert(0, seq_group)
405
406 def _preempt_by_swap(
407 self,
cacheflow/core/server.py
-1
@@ -17,7 +17,6 @@ from cacheflow.sequence import SequenceGroup
17 from cacheflow.utils import get_gpu_memory, get_cpu_memory
18 from cacheflow.worker.controller import Controller, DeviceID
19
20 -
20 logger = init_logger(__name__)
21
22
cacheflow/frontend/simple_frontend.py
-1
@@ -7,7 +7,6 @@ from cacheflow.sampling_params import SamplingParams
7 from cacheflow.sequence import Sequence, SequenceGroup
8 from cacheflow.utils import Counter
9
10 -
10 logger = init_logger(__name__)
11
12
cacheflow/model_executor/memory_analyzer.py
-1
@@ -4,7 +4,6 @@ from transformers import AutoConfig
4 from cacheflow.logger import init_logger
5 from cacheflow.model_executor.utils import get_dtype_size
6
7 -
7 logger = init_logger(__name__)
8
9 _GiB = 1 << 30