@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a2a9869cb7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

SERVING -> RUNNING

Woosuk Kwon committed Feb 12, 2023 at 08:25 UTC a2a9869cb7e11a46c215e0cd55401509395c035d
2 files changed +2 -2
cacheflow/master/block_manager.py
+1 -1
@@ -95,7 +95,7 @@ class BlockSpaceManager:
95 # Simple heuristic: If there is at least one free block
96 # for each sequence, we can append.
97 num_free_gpu_blocks = self.gpu_allocator.get_num_free_blocks()
98 - num_seqs = seq_group.num_seqs(status=SequenceStatus.SERVING)
98 + num_seqs = seq_group.num_seqs(status=SequenceStatus.RUNNING)
99 return num_seqs <= num_free_gpu_blocks
100
101 def append(self, seq: Sequence) -> Optional[Tuple[int, int]]:
cacheflow/sequence.py
+1 -1
@@ -7,7 +7,7 @@ from cacheflow.decoding import DecodingParams
7
8 class SequenceStatus(enum.Enum):
9 PENDING = enum.auto()
10 - SERVING = enum.auto()
10 + RUNNING = enum.auto()
11 SWAPPED = enum.auto()
12 FINISHED = enum.auto()
13