@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8423aef4c8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[BugFix][Core] Multistep Fix Crash on Request Cancellation (#8059)

Robert Shaw committed Aug 31, 2024 at 15:44 UTC 8423aef4c867818524e90b2e2e58730b6ee5592c
1 file changed +7 -1
vllm/engine/output_processor/multi_step.py
+7 -1
@@ -88,9 +88,15 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
88 # TODO: Add support for async if necessary
89 assert not is_async
90
91 + # Sequences can be in RUNNING or FINISHED_ABORTED state
92 + # once scheduled, as a sequence is moved to FINSIHED_ABORTED
93 + # if a client disconnects from the api server.
94 seqs = sequence_group.get_seqs(status=SequenceStatus.RUNNING)
95 + if seqs is None:
96 + seqs = sequence_group.get_seqs(
97 + status=SequenceStatus.FINISHED_ABORTED)
98
93 - assert seqs, "expected running sequences"
99 + assert seqs, "Expected RUNNING or FINISHED_ABORTED sequences"
100 assert len(seqs) == 1, (
101 "Beam search not supported in multi-step decoding.")
102 seq = seqs[0]