A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 13:30 UTC
[Bugfix][Core] add seq_id_to_seq_group clearing to avoid memory leak when s… (#16472)
Signed-off-by: 开哲 <kaizhe.zy@alibaba-inc.com> Co-authored-by: 开哲 <kaizhe.zy@alibaba-inc.com>
张宇 committed
Apr 24, 2025 at 11:25 UTC
6167c0e5d2a18fb4c4d14be3b4e9f5e35b712b76
1 file changed
+7
vllm/outputs.py
+7
@@ -179,6 +179,13 @@ class RequestOutput:
179
group.finish_seq(seq_group)
180
if assembled_seq_group is None:
181
return None
182
+
183
+ # clear finished seq in seq_id_to_seq_group
184
+ if len(group.to_be_finished) == 0:
185
+ for sub_request_id in list(group.seq_id_to_index.keys()):
186
+ if sub_request_id in seq_id_to_seq_group:
187
+ del seq_id_to_seq_group[sub_request_id]
188
+
189
return cls.from_seq_group(assembled_seq_group, use_cache,
190
seq_id_to_seq_group)
191