@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a1b9cb2a34
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

[BugFix] Fix recovery logic for sequence group (#2186)

Woosuk Kwon committed Dec 20, 2023 at 21:52 UTC a1b9cb2a3469e4e682e741af9a0f91e16923205d
2 files changed +10 -8
vllm/core/block_manager.py
+3 -3
@@ -103,7 +103,7 @@ class BlockSpaceManager:
103 def can_allocate(self, seq_group: SequenceGroup) -> AllocStatus:
104 # FIXME(woosuk): Here we assume that all sequences in the group share
105 # the same prompt. This may not be true for preempted sequences.
106 - seq = seq_group.get_seqs()[0]
106 + seq = seq_group.get_seqs(status=SequenceStatus.WAITING)[0]
107 num_required_blocks = len(seq.logical_token_blocks)
108 if self.block_sliding_window is not None:
109 num_required_blocks = min(num_required_blocks,
@@ -122,7 +122,7 @@ class BlockSpaceManager:
122 def allocate(self, seq_group: SequenceGroup) -> None:
123 # NOTE: Here we assume that all sequences in the group have the same
124 # prompt.
125 - seq = seq_group.get_seqs()[0]
125 + seq = seq_group.get_seqs(status=SequenceStatus.WAITING)[0]
126
127 # Allocate new physical token blocks that will store the prompt tokens.
128 block_table: BlockTable = []
@@ -137,7 +137,7 @@ class BlockSpaceManager:
137 block_table.append(block)
138
139 # Assign the block table for each sequence.
140 - for seq in seq_group.get_seqs():
140 + for seq in seq_group.get_seqs(status=SequenceStatus.WAITING):
141 self.block_tables[seq.seq_id] = block_table.copy()
142
143 def can_append_slot(self, seq_group: SequenceGroup) -> bool:
vllm/core/scheduler.py
+7 -5
@@ -139,15 +139,17 @@ class Scheduler:
139 while self.waiting:
140 seq_group = self.waiting[0]
141
142 - assert seq_group.num_seqs() == 1, (
142 + waiting_seqs = seq_group.get_seqs(
143 + status=SequenceStatus.WAITING)
144 + assert len(waiting_seqs) == 1, (
145 "Waiting sequence group should have only one prompt "
146 "sequence.")
145 - num_prompt_tokens = seq_group.get_seqs()[0].get_len()
147 + num_prompt_tokens = waiting_seqs[0].get_len()
148 if num_prompt_tokens > self.prompt_limit:
149 logger.warning(
150 f"Input prompt ({num_prompt_tokens} tokens) is too long"
151 f" and exceeds limit of {self.prompt_limit}")
150 - for seq in seq_group.get_seqs():
152 + for seq in waiting_seqs:
153 seq.status = SequenceStatus.FINISHED_IGNORED
154 ignored_seq_groups.append(seq_group)
155 self.waiting.pop(0)
@@ -161,7 +163,7 @@ class Scheduler:
163 logger.warning(
164 f"Input prompt ({num_prompt_tokens} tokens) is too long"
165 f" and exceeds the capacity of block_manager")
164 - for seq in seq_group.get_seqs():
166 + for seq in waiting_seqs:
167 seq.status = SequenceStatus.FINISHED_IGNORED
168 ignored_seq_groups.append(seq_group)
169 self.waiting.pop(0)
@@ -317,7 +319,7 @@ class Scheduler:
319
320 def _allocate(self, seq_group: SequenceGroup) -> None:
321 self.block_manager.allocate(seq_group)
320 - for seq in seq_group.get_seqs():
322 + for seq in seq_group.get_seqs(status=SequenceStatus.WAITING):
323 seq.status = SequenceStatus.RUNNING
324
325 def _append_slot(