A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Core] Fix scheduler considering "no LoRA" as "LoRA" (#4897)
Antoni Baum committed
May 20, 2024 at 17:48 UTC
65ae8c2c8f52e0c98e4e26ad1255772d888592a6
1 file changed
+2
-2
vllm/core/scheduler.py
+2
-2
@@ -744,8 +744,8 @@ class Scheduler:
744
budget.add_num_seqs(seq_group.request_id,
745
seq_group.get_max_num_running_seqs())
746
curr_loras = set(
747
- seq_group.lora_int_id
748
- for seq_group in self.running) if self.lora_enabled else None
747
+ seq_group.lora_int_id for seq_group in self.running
748
+ if seq_group.lora_int_id > 0) if self.lora_enabled else None
749
750
remaining_waiting, prefills = (self.waiting,
751
SchedulerPrefillOutputs.create_empty())