A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[Bugfix] Free cross attention block table for preempted-for-recompute sequence group. (#10013)
Signed-off-by: Kathy Yu <feiyangyu@google.com>
Kathy Yu committed
Jan 2, 2025 at 10:28 UTC
2f385183f35497e030ef22c9820d83b83bc4f6db
1 file changed
+1
vllm/core/scheduler.py
+1
@@ -1579,6 +1579,7 @@ class Scheduler:
1579
seq.status = SequenceStatus.WAITING
1580
self.free_seq(seq)
1581
seq.reset_state_for_recompute()
1582
+ self._free_seq_group_cross_attn_blocks(seq_group)
1583
1584
def _preempt_by_swap(
1585
self,