A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
[BugFix] Fix recovery logic for sequence group (#2186)
Woosuk Kwon committed
Dec 20, 2023 at 21:52 UTC
a1b9cb2a3469e4e682e741af9a0f91e16923205d
2 files changed
+10
-8
vllm/core/block_manager.py
+3
-3
@@ -103,7 +103,7 @@ class BlockSpaceManager:
103
def can_allocate(self, seq_group: SequenceGroup) -> AllocStatus:
104
# FIXME(woosuk): Here we assume that all sequences in the group share
105
# the same prompt. This may not be true for preempted sequences.
106
- seq = seq_group.get_seqs()[0]
106
+ seq = seq_group.get_seqs(status=SequenceStatus.WAITING)[0]
107
num_required_blocks = len(seq.logical_token_blocks)
108
if self.block_sliding_window is not None:
109
num_required_blocks = min(num_required_blocks,
@@ -122,7 +122,7 @@ class BlockSpaceManager:
122
def allocate(self, seq_group: SequenceGroup) -> None:
123
# NOTE: Here we assume that all sequences in the group have the same
124
# prompt.
125
- seq = seq_group.get_seqs()[0]
125
+ seq = seq_group.get_seqs(status=SequenceStatus.WAITING)[0]
126
127
# Allocate new physical token blocks that will store the prompt tokens.
128
block_table: BlockTable = []
@@ -137,7 +137,7 @@ class BlockSpaceManager:
137
block_table.append(block)
138
139
# Assign the block table for each sequence.
140
- for seq in seq_group.get_seqs():
140
+ for seq in seq_group.get_seqs(status=SequenceStatus.WAITING):
141
self.block_tables[seq.seq_id] = block_table.copy()
142
143
def can_append_slot(self, seq_group: SequenceGroup) -> bool:
vllm/core/scheduler.py
+7
-5
@@ -139,15 +139,17 @@ class Scheduler:
139
while self.waiting:
140
seq_group = self.waiting[0]
141
142
- assert seq_group.num_seqs() == 1, (
142
+ waiting_seqs = seq_group.get_seqs(
143
+ status=SequenceStatus.WAITING)
144
+ assert len(waiting_seqs) == 1, (
145
"Waiting sequence group should have only one prompt "
146
"sequence.")
145
- num_prompt_tokens = seq_group.get_seqs()[0].get_len()
147
+ num_prompt_tokens = waiting_seqs[0].get_len()
148
if num_prompt_tokens > self.prompt_limit:
149
logger.warning(
150
f"Input prompt ({num_prompt_tokens} tokens) is too long"
151
f" and exceeds limit of {self.prompt_limit}")
150
- for seq in seq_group.get_seqs():
152
+ for seq in waiting_seqs:
153
seq.status = SequenceStatus.FINISHED_IGNORED
154
ignored_seq_groups.append(seq_group)
155
self.waiting.pop(0)
@@ -161,7 +163,7 @@ class Scheduler:
163
logger.warning(
164
f"Input prompt ({num_prompt_tokens} tokens) is too long"
165
f" and exceeds the capacity of block_manager")
164
- for seq in seq_group.get_seqs():
166
+ for seq in waiting_seqs:
167
seq.status = SequenceStatus.FINISHED_IGNORED
168
ignored_seq_groups.append(seq_group)
169
self.waiting.pop(0)
@@ -317,7 +319,7 @@ class Scheduler:
319
320
def _allocate(self, seq_group: SequenceGroup) -> None:
321
self.block_manager.allocate(seq_group)
320
- for seq in seq_group.get_seqs():
322
+ for seq in seq_group.get_seqs(status=SequenceStatus.WAITING):
323
seq.status = SequenceStatus.RUNNING
324
325
def _append_slot(