A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
[Fix] Fix bugs in scheduler (#1727)
Zhuofan committed
Nov 21, 2023 at 08:10 UTC
19849db573466eff7a83a4c203948c761a6cda85
1 file changed
+2
-1
vllm/core/scheduler.py
+2
-1
@@ -196,7 +196,8 @@ class Scheduler:
196
scheduler_outputs = SchedulerOutputs(
197
scheduled_seq_groups=scheduled,
198
prompt_run=True,
199
- num_batched_tokens=len(seq_lens) * max(seq_lens),
199
+ num_batched_tokens=len(seq_lens) *
200
+ max(seq_lens) if seq_lens else 0,
201
blocks_to_swap_in=blocks_to_swap_in,
202
blocks_to_swap_out=blocks_to_swap_out,
203
blocks_to_copy=blocks_to_copy,