A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[BugFix][Core] Multistep Fix Crash on Request Cancellation (#8059)
Robert Shaw committed
Aug 31, 2024 at 15:44 UTC
8423aef4c867818524e90b2e2e58730b6ee5592c
1 file changed
+7
-1
vllm/engine/output_processor/multi_step.py
+7
-1
@@ -88,9 +88,15 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
88
# TODO: Add support for async if necessary
89
assert not is_async
90
91
+ # Sequences can be in RUNNING or FINISHED_ABORTED state
92
+ # once scheduled, as a sequence is moved to FINSIHED_ABORTED
93
+ # if a client disconnects from the api server.
94
seqs = sequence_group.get_seqs(status=SequenceStatus.RUNNING)
95
+ if seqs is None:
96
+ seqs = sequence_group.get_seqs(
97
+ status=SequenceStatus.FINISHED_ABORTED)
98
93
- assert seqs, "expected running sequences"
99
+ assert seqs, "Expected RUNNING or FINISHED_ABORTED sequences"
100
assert len(seqs) == 1, (
101
"Beam search not supported in multi-step decoding.")
102
seq = seqs[0]