A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
SERVING -> RUNNING
Woosuk Kwon committed
Feb 12, 2023 at 08:25 UTC
a2a9869cb7e11a46c215e0cd55401509395c035d
2 files changed
+2
-2
cacheflow/master/block_manager.py
+1
-1
@@ -95,7 +95,7 @@ class BlockSpaceManager:
95
# Simple heuristic: If there is at least one free block
96
# for each sequence, we can append.
97
num_free_gpu_blocks = self.gpu_allocator.get_num_free_blocks()
98
- num_seqs = seq_group.num_seqs(status=SequenceStatus.SERVING)
98
+ num_seqs = seq_group.num_seqs(status=SequenceStatus.RUNNING)
99
return num_seqs <= num_free_gpu_blocks
100
101
def append(self, seq: Sequence) -> Optional[Tuple[int, int]]:
cacheflow/sequence.py
+1
-1
@@ -7,7 +7,7 @@ from cacheflow.decoding import DecodingParams
7
8
class SequenceStatus(enum.Enum):
9
PENDING = enum.auto()
10
- SERVING = enum.auto()
10
+ RUNNING = enum.auto()
11
SWAPPED = enum.auto()
12
FINISHED = enum.auto()
13