@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3f8d42c81f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

Pipeline Parallel: Guard for KeyErrors at request abort (#6587)

Signed-off-by: Travis Johnson <tsjohnso@us.ibm.com>

Travis Johnson committed Jul 19, 2024 at 20:18 UTC 3f8d42c81fe8d3842a9e05c9f5d98290b7f79736
2 files changed +9 -2
vllm/engine/async_llm_engine.py
+4 -1
@@ -131,7 +131,10 @@ class RequestTracker:
131 """Process a request output from the engine."""
132 request_id = request_output.request_id
133
134 - self._request_streams[request_id].put(request_output)
134 + # Guard against a KeyError which can occur if the request was aborted
135 + # while the output was generated
136 + if (stream := self._request_streams.get(request_id)) is not None:
137 + stream.put(request_output)
138 if request_output.finished:
139 if verbose:
140 logger.info("Finished request %s.", request_id)
vllm/engine/output_processor/single_step.py
+5 -1
@@ -90,7 +90,11 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
90 for parent_seq in parent_seqs
91 }
92 for sample in samples:
93 - parent_child_dict[sample.parent_seq_id].append(sample)
93 + # Guard against a KeyError which can occur if the request was
94 + # aborted while the output was generated
95 + if (child_list :=
96 + parent_child_dict.get(sample.parent_seq_id)) is not None:
97 + child_list.append(sample)
98 # List of (child, parent)
99 child_seqs: List[Tuple[Sequence, Sequence]] = []
100