A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Core] Scheduler perf fix (#4270)
SangBin Cho committed
Apr 23, 2024 at 06:11 UTC
ad8d696a99ca1eee19f1404e16e8e82df592ff85
2 files changed
+11
-14
tests/core/test_scheduler.py
+9
-9
@@ -540,7 +540,7 @@ def test_decode_schedule_preempted():
540
curr_loras = None
541
for i in range(3):
542
_, seq_group = create_dummy_prompt(str(i), prompt_length=60)
543
- scheduler._allocate_and_set_running(seq_group, 60)
543
+ scheduler._allocate_and_set_running(seq_group)
544
append_new_token_seq_group(60, seq_group, 1)
545
running.append(seq_group)
546
scheduler.block_manager.can_append_slots = MagicMock()
@@ -581,7 +581,7 @@ def test_decode_swap_beam_search():
581
budget = create_token_budget()
582
for i in range(3):
583
_, seq_group = create_dummy_prompt(str(i), prompt_length=60, best_of=2)
584
- scheduler._allocate_and_set_running(seq_group, 60)
584
+ scheduler._allocate_and_set_running(seq_group)
585
running.append(seq_group)
586
append_new_token_seq_group(60, seq_group, 1)
587
budget.add_num_seqs(seq_group.request_id,
@@ -629,7 +629,7 @@ def test_schedule_decode_blocks_to_copy_update():
629
running = deque()
630
policy = PolicyFactory.get_policy(policy_name="fcfs")
631
curr_loras = None
632
- scheduler._allocate_and_set_running(seq_group, 60)
632
+ scheduler._allocate_and_set_running(seq_group)
633
append_new_token_seq_group(60, seq_group, 1)
634
running.append(seq_group)
635
@@ -659,7 +659,7 @@ def test_schedule_swapped_simple():
659
curr_loras = None
660
blocks_to_swap_out = {}
661
_, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
662
- scheduler._allocate_and_set_running(seq_group, 60)
662
+ scheduler._allocate_and_set_running(seq_group)
663
append_new_token_seq_group(60, seq_group, 1)
664
scheduler._swap_out(seq_group, blocks_to_swap_out)
665
swapped.append(seq_group)
@@ -687,7 +687,7 @@ def test_schedule_swapped_max_token_budget():
687
blocks_to_swap_out = {}
688
for _ in range(2):
689
_, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
690
- scheduler._allocate_and_set_running(seq_group, 60)
690
+ scheduler._allocate_and_set_running(seq_group)
691
append_new_token_seq_group(60, seq_group, 1)
692
scheduler._swap_out(seq_group, blocks_to_swap_out)
693
swapped.append(seq_group)
@@ -721,7 +721,7 @@ def test_schedule_swapped_max_seqs():
721
blocks_to_swap_out = {}
722
for i in range(4):
723
_, seq_group = create_dummy_prompt(str(i), prompt_length=60)
724
- scheduler._allocate_and_set_running(seq_group, 60)
724
+ scheduler._allocate_and_set_running(seq_group)
725
append_new_token_seq_group(60, seq_group, 1)
726
scheduler._swap_out(seq_group, blocks_to_swap_out)
727
swapped.append(seq_group)
@@ -759,7 +759,7 @@ def test_schedule_swapped_max_loras():
759
lora_name=str(i),
760
lora_int_id=i + 1,
761
lora_local_path="abc"))
762
- scheduler._allocate_and_set_running(seq_group, 60)
762
+ scheduler._allocate_and_set_running(seq_group)
763
append_new_token_seq_group(60, seq_group, 1)
764
scheduler._swap_out(seq_group, blocks_to_swap_out)
765
swapped.append(seq_group)
@@ -783,7 +783,7 @@ def test_schedule_swapped_cannot_swap_in():
783
blocks_to_swap_out = {}
784
for _ in range(2):
785
_, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
786
- scheduler._allocate_and_set_running(seq_group, 60)
786
+ scheduler._allocate_and_set_running(seq_group)
787
append_new_token_seq_group(60, seq_group, 1)
788
scheduler._swap_out(seq_group, blocks_to_swap_out)
789
swapped.append(seq_group)
@@ -808,7 +808,7 @@ def test_schedule_swapped_blocks_to_copy():
808
policy = PolicyFactory.get_policy(policy_name="fcfs")
809
curr_loras = None
810
_, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
811
- scheduler._allocate_and_set_running(seq_group, 60)
811
+ scheduler._allocate_and_set_running(seq_group)
812
append_new_token_seq_group(60, seq_group, 1)
813
blocks_to_swap_out = {}
814
scheduler._swap_out(seq_group, blocks_to_swap_out)
vllm/core/scheduler.py
+2
-5
@@ -297,7 +297,6 @@ class Scheduler:
297
298
def add_seq_group(self, seq_group: SequenceGroup) -> None:
299
# Add sequence groups to the waiting queue.
300
- logger.debug(f"add_seq_group {seq_group.request_id}")
300
self.waiting.append(seq_group)
301
302
def abort_seq_group(self, request_id: Union[str, Iterable[str]]) -> None:
@@ -427,7 +426,6 @@ class Scheduler:
426
swapped_out.append(seq_group)
427
break
428
else:
430
- logger.debug(f"append slot for {seq_group}")
429
self._append_slots(seq_group, blocks_to_copy)
430
is_prefill = seq_group.is_prefill()
431
if is_prefill:
@@ -659,7 +657,7 @@ class Scheduler:
657
if curr_loras is not None and lora_int_id > 0:
658
curr_loras.add(lora_int_id)
659
waiting_queue.popleft()
662
- self._allocate_and_set_running(seq_group, num_new_tokens)
660
+ self._allocate_and_set_running(seq_group)
661
seq_groups.append(
662
ScheduledSequenceGroup(seq_group=seq_group,
663
token_chunk_size=num_new_tokens))
@@ -952,8 +950,7 @@ class Scheduler:
950
self.running = deque(seq_group for seq_group in self.running
951
if not seq_group.is_finished())
952
955
- def _allocate_and_set_running(self, seq_group: SequenceGroup,
956
- num_new_tokens: int) -> None:
953
+ def _allocate_and_set_running(self, seq_group: SequenceGroup) -> None:
954
self.block_manager.allocate(seq_group)
955
for seq in seq_group.get_seqs(status=SequenceStatus.WAITING):
956
seq.status = SequenceStatus.RUNNING