@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 65ae8c2c8f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Core] Fix scheduler considering "no LoRA" as "LoRA" (#4897)

Antoni Baum committed May 20, 2024 at 17:48 UTC 65ae8c2c8f52e0c98e4e26ad1255772d888592a6
1 file changed +2 -2
vllm/core/scheduler.py
+2 -2
@@ -744,8 +744,8 @@ class Scheduler:
744 budget.add_num_seqs(seq_group.request_id,
745 seq_group.get_max_num_running_seqs())
746 curr_loras = set(
747 - seq_group.lora_int_id
748 - for seq_group in self.running) if self.lora_enabled else None
747 + seq_group.lora_int_id for seq_group in self.running
748 + if seq_group.lora_int_id > 0) if self.lora_enabled else None
749
750 remaining_waiting, prefills = (self.waiting,
751 SchedulerPrefillOutputs.create_empty())