A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
Pipeline Parallel: Guard for KeyErrors at request abort (#6587)
Signed-off-by: Travis Johnson <tsjohnso@us.ibm.com>
Travis Johnson committed
Jul 19, 2024 at 20:18 UTC
3f8d42c81fe8d3842a9e05c9f5d98290b7f79736
2 files changed
+9
-2
vllm/engine/async_llm_engine.py
+4
-1
@@ -131,7 +131,10 @@ class RequestTracker:
131
"""Process a request output from the engine."""
132
request_id = request_output.request_id
133
134
- self._request_streams[request_id].put(request_output)
134
+ # Guard against a KeyError which can occur if the request was aborted
135
+ # while the output was generated
136
+ if (stream := self._request_streams.get(request_id)) is not None:
137
+ stream.put(request_output)
138
if request_output.finished:
139
if verbose:
140
logger.info("Finished request %s.", request_id)
vllm/engine/output_processor/single_step.py
+5
-1
@@ -90,7 +90,11 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
90
for parent_seq in parent_seqs
91
}
92
for sample in samples:
93
- parent_child_dict[sample.parent_seq_id].append(sample)
93
+ # Guard against a KeyError which can occur if the request was
94
+ # aborted while the output was generated
95
+ if (child_list :=
96
+ parent_child_dict.get(sample.parent_seq_id)) is not None:
97
+ child_list.append(sample)
98
# List of (child, parent)
99
child_seqs: List[Tuple[Sequence, Sequence]] = []
100