A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
Addition of lacked ignored_seq_groups in _schedule_chunked_prefill (#5296)
limingshu committed
Jun 7, 2024 at 21:35 UTC
dc49fb892ca32cb364dfc39d711ab84d3b35a28f
1 file changed
+2
-1
vllm/core/scheduler.py
+2
-1
@@ -905,7 +905,8 @@ class Scheduler:
905
blocks_to_swap_out=running_scheduled.blocks_to_swap_out,
906
blocks_to_copy=running_scheduled.blocks_to_copy +
907
swapped_in.blocks_to_copy,
908
- ignored_seq_groups=prefills.ignored_seq_groups,
908
+ ignored_seq_groups=prefills.ignored_seq_groups +
909
+ swapped_in.infeasible_seq_groups,
910
num_lookahead_slots=running_scheduled.num_lookahead_slots,
911
running_queue_size=len(self.running),
912
preempted=(len(running_scheduled.preempted) +