@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 19849db573
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

[Fix] Fix bugs in scheduler (#1727)

Zhuofan committed Nov 21, 2023 at 08:10 UTC 19849db573466eff7a83a4c203948c761a6cda85
1 file changed +2 -1
vllm/core/scheduler.py
+2 -1
@@ -196,7 +196,8 @@ class Scheduler:
196 scheduler_outputs = SchedulerOutputs(
197 scheduled_seq_groups=scheduled,
198 prompt_run=True,
199 - num_batched_tokens=len(seq_lens) * max(seq_lens),
199 + num_batched_tokens=len(seq_lens) *
200 + max(seq_lens) if seq_lens else 0,
201 blocks_to_swap_in=blocks_to_swap_in,
202 blocks_to_swap_out=blocks_to_swap_out,
203 blocks_to_copy=blocks_to_copy,