@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ad8d696a99
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Core] Scheduler perf fix (#4270)

SangBin Cho committed Apr 23, 2024 at 06:11 UTC ad8d696a99ca1eee19f1404e16e8e82df592ff85
2 files changed +11 -14
tests/core/test_scheduler.py
+9 -9
@@ -540,7 +540,7 @@ def test_decode_schedule_preempted():
540 curr_loras = None
541 for i in range(3):
542 _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
543 - scheduler._allocate_and_set_running(seq_group, 60)
543 + scheduler._allocate_and_set_running(seq_group)
544 append_new_token_seq_group(60, seq_group, 1)
545 running.append(seq_group)
546 scheduler.block_manager.can_append_slots = MagicMock()
@@ -581,7 +581,7 @@ def test_decode_swap_beam_search():
581 budget = create_token_budget()
582 for i in range(3):
583 _, seq_group = create_dummy_prompt(str(i), prompt_length=60, best_of=2)
584 - scheduler._allocate_and_set_running(seq_group, 60)
584 + scheduler._allocate_and_set_running(seq_group)
585 running.append(seq_group)
586 append_new_token_seq_group(60, seq_group, 1)
587 budget.add_num_seqs(seq_group.request_id,
@@ -629,7 +629,7 @@ def test_schedule_decode_blocks_to_copy_update():
629 running = deque()
630 policy = PolicyFactory.get_policy(policy_name="fcfs")
631 curr_loras = None
632 - scheduler._allocate_and_set_running(seq_group, 60)
632 + scheduler._allocate_and_set_running(seq_group)
633 append_new_token_seq_group(60, seq_group, 1)
634 running.append(seq_group)
635
@@ -659,7 +659,7 @@ def test_schedule_swapped_simple():
659 curr_loras = None
660 blocks_to_swap_out = {}
661 _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
662 - scheduler._allocate_and_set_running(seq_group, 60)
662 + scheduler._allocate_and_set_running(seq_group)
663 append_new_token_seq_group(60, seq_group, 1)
664 scheduler._swap_out(seq_group, blocks_to_swap_out)
665 swapped.append(seq_group)
@@ -687,7 +687,7 @@ def test_schedule_swapped_max_token_budget():
687 blocks_to_swap_out = {}
688 for _ in range(2):
689 _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
690 - scheduler._allocate_and_set_running(seq_group, 60)
690 + scheduler._allocate_and_set_running(seq_group)
691 append_new_token_seq_group(60, seq_group, 1)
692 scheduler._swap_out(seq_group, blocks_to_swap_out)
693 swapped.append(seq_group)
@@ -721,7 +721,7 @@ def test_schedule_swapped_max_seqs():
721 blocks_to_swap_out = {}
722 for i in range(4):
723 _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
724 - scheduler._allocate_and_set_running(seq_group, 60)
724 + scheduler._allocate_and_set_running(seq_group)
725 append_new_token_seq_group(60, seq_group, 1)
726 scheduler._swap_out(seq_group, blocks_to_swap_out)
727 swapped.append(seq_group)
@@ -759,7 +759,7 @@ def test_schedule_swapped_max_loras():
759 lora_name=str(i),
760 lora_int_id=i + 1,
761 lora_local_path="abc"))
762 - scheduler._allocate_and_set_running(seq_group, 60)
762 + scheduler._allocate_and_set_running(seq_group)
763 append_new_token_seq_group(60, seq_group, 1)
764 scheduler._swap_out(seq_group, blocks_to_swap_out)
765 swapped.append(seq_group)
@@ -783,7 +783,7 @@ def test_schedule_swapped_cannot_swap_in():
783 blocks_to_swap_out = {}
784 for _ in range(2):
785 _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
786 - scheduler._allocate_and_set_running(seq_group, 60)
786 + scheduler._allocate_and_set_running(seq_group)
787 append_new_token_seq_group(60, seq_group, 1)
788 scheduler._swap_out(seq_group, blocks_to_swap_out)
789 swapped.append(seq_group)
@@ -808,7 +808,7 @@ def test_schedule_swapped_blocks_to_copy():
808 policy = PolicyFactory.get_policy(policy_name="fcfs")
809 curr_loras = None
810 _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
811 - scheduler._allocate_and_set_running(seq_group, 60)
811 + scheduler._allocate_and_set_running(seq_group)
812 append_new_token_seq_group(60, seq_group, 1)
813 blocks_to_swap_out = {}
814 scheduler._swap_out(seq_group, blocks_to_swap_out)
vllm/core/scheduler.py
+2 -5
@@ -297,7 +297,6 @@ class Scheduler:
297
298 def add_seq_group(self, seq_group: SequenceGroup) -> None:
299 # Add sequence groups to the waiting queue.
300 - logger.debug(f"add_seq_group {seq_group.request_id}")
300 self.waiting.append(seq_group)
301
302 def abort_seq_group(self, request_id: Union[str, Iterable[str]]) -> None:
@@ -427,7 +426,6 @@ class Scheduler:
426 swapped_out.append(seq_group)
427 break
428 else:
430 - logger.debug(f"append slot for {seq_group}")
429 self._append_slots(seq_group, blocks_to_copy)
430 is_prefill = seq_group.is_prefill()
431 if is_prefill:
@@ -659,7 +657,7 @@ class Scheduler:
657 if curr_loras is not None and lora_int_id > 0:
658 curr_loras.add(lora_int_id)
659 waiting_queue.popleft()
662 - self._allocate_and_set_running(seq_group, num_new_tokens)
660 + self._allocate_and_set_running(seq_group)
661 seq_groups.append(
662 ScheduledSequenceGroup(seq_group=seq_group,
663 token_chunk_size=num_new_tokens))
@@ -952,8 +950,7 @@ class Scheduler:
950 self.running = deque(seq_group for seq_group in self.running
951 if not seq_group.is_finished())
952
955 - def _allocate_and_set_running(self, seq_group: SequenceGroup,
956 - num_new_tokens: int) -> None:
953 + def _allocate_and_set_running(self, seq_group: SequenceGroup) -> None:
954 self.block_manager.allocate(seq_group)
955 for seq in seq_group.get_seqs(status=SequenceStatus.WAITING):
956 seq.status = SequenceStatus.RUNNING