@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c8a7e93273
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[core][scheduler] simplify and improve scheduler (#6867)

youkaichao committed Jul 31, 2024 at 23:51 UTC c8a7e93273ff4338d6f89f8a63ff16426ac240b8
4 files changed +112 -214
tests/core/block/e2e/test_correctness.py
+1 -1
@@ -183,7 +183,7 @@ def test_v1_v2_greedy_equality_with_cow(baseline_llm_generator,
183
184 # Allow only 2 sequences of ~128 tokens in worst case.
185 # Note 16 = 128/block_size
186 - "num_gpu_blocks_override": 2 * (16 + 1),
186 + "num_gpu_blocks_override": 2 * (16 + 2),
187 }
188 ])
189 @pytest.mark.parametrize("baseline_llm_kwargs", [{
tests/core/test_scheduler.py
+68 -95
@@ -1,13 +1,12 @@
1 import time
2 from collections import deque
3 -from typing import Deque, List, Set, Tuple
3 +from typing import List, Set, Tuple
4 from unittest.mock import MagicMock
5
6 import pytest # noqa
7
8 from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig
9 from vllm.core.interfaces import AllocStatus
10 -from vllm.core.policy import PolicyFactory
10 from vllm.core.scheduler import Scheduler, SchedulingBudget
11 from vllm.lora.request import LoRARequest
12 from vllm.sequence import Logprob, SequenceGroup, SequenceStatus
@@ -348,10 +347,10 @@ def test_prefill_schedule_max_prompt_len():
347 """
348 scheduler = initialize_scheduler(max_model_len=30)
349 _, seq_group = create_dummy_prompt("0", prompt_length=60)
351 - waiting = deque([seq_group])
350 + scheduler.add_seq_group(seq_group)
351 budget = create_token_budget()
353 - remaining_waiting, output = scheduler._schedule_prefills(
354 - waiting, budget, None)
352 + output = scheduler._schedule_prefills(budget, None)
353 + remaining_waiting = scheduler.waiting
354 assert len(output.ignored_seq_groups) == 1
355 assert len(output.seq_groups) == 0
356 assert budget.num_batched_tokens == 0
@@ -364,15 +363,14 @@ def test_prefill_schedule_token_budget():
363 Test token budget respected.
364 """
365 scheduler = initialize_scheduler()
367 - waiting: Deque[SequenceGroup] = deque()
366 budget = create_token_budget(token_budget=0)
367 for i in range(2):
368 _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
371 - waiting.append(seq_group)
369 + scheduler.add_seq_group(seq_group)
370
371 # 0 token budget == nothing is scheduled.
374 - remaining_waiting, output = scheduler._schedule_prefills(
375 - waiting, budget, None)
372 + output = scheduler._schedule_prefills(budget, None)
373 + remaining_waiting = scheduler.waiting
374 assert len(output.ignored_seq_groups) == 0
375 assert len(output.seq_groups) == 0
376 assert budget.num_batched_tokens == 0
@@ -381,8 +379,8 @@ def test_prefill_schedule_token_budget():
379
380 # 60 token budget == 1 request scheduled.
381 budget = create_token_budget(token_budget=60)
384 - remaining_waiting, output = scheduler._schedule_prefills(
385 - waiting, budget, None)
382 + output = scheduler._schedule_prefills(budget, None)
383 + remaining_waiting = scheduler.waiting
384 assert len(output.ignored_seq_groups) == 0
385 assert len(output.seq_groups) == 1
386 assert budget.num_batched_tokens == 60
@@ -391,14 +389,13 @@ def test_prefill_schedule_token_budget():
389
390 # Test when current_batched_tokens respected.
391 scheduler = initialize_scheduler()
394 - waiting = deque()
392 budget = create_token_budget(token_budget=60)
393 add_token_budget(budget, 30, 0)
394 _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
395 # Cannot schedule a prompt that doesn't fit the budget.
399 - waiting.append(seq_group)
400 - remaining_waiting, output = scheduler._schedule_prefills(
401 - waiting, budget, None)
396 + scheduler.add_seq_group(seq_group)
397 + output = scheduler._schedule_prefills(budget, None)
398 + remaining_waiting = scheduler.waiting
399 assert len(output.ignored_seq_groups) == 0
400 assert len(output.seq_groups) == 0
401 assert budget.num_batched_tokens == 30
@@ -406,8 +403,8 @@ def test_prefill_schedule_token_budget():
403 assert len(remaining_waiting) == 1
404 budget = create_token_budget(token_budget=90)
405 add_token_budget(budget, 30, 0)
409 - remaining_waiting, output = scheduler._schedule_prefills(
410 - waiting, budget, None)
406 + output = scheduler._schedule_prefills(budget, None)
407 + remaining_waiting = scheduler.waiting
408 assert len(output.seq_groups) == 1
409 assert budget.num_batched_tokens == 90
410 assert budget.num_curr_seqs == 1
@@ -419,13 +416,12 @@ def test_prefill_schedule_max_seqs():
416 Test max seq respected.
417 """
418 scheduler = initialize_scheduler()
422 - waiting: Deque[SequenceGroup] = deque()
419 budget = create_token_budget(max_num_seqs=2)
420 for i in range(3):
421 _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
426 - waiting.append(seq_group)
427 - remaining_waiting, output = scheduler._schedule_prefills(
428 - waiting, budget, None)
422 + scheduler.add_seq_group(seq_group)
423 + output = scheduler._schedule_prefills(budget, None)
424 + remaining_waiting = scheduler.waiting
425 assert len(output.ignored_seq_groups) == 0
426 assert len(output.seq_groups) == 2
427 assert budget.num_batched_tokens == 120
@@ -433,13 +429,13 @@ def test_prefill_schedule_max_seqs():
429 assert len(remaining_waiting) == 1
430
431 # Verify curr_num_seqs respected.
436 - waiting = deque()
432 + scheduler.waiting = deque()
433 budget = create_token_budget(max_num_seqs=2)
434 add_token_budget(budget, 0, 2)
435 _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
440 - waiting.append(seq_group)
441 - remaining_waiting, output = scheduler._schedule_prefills(
442 - waiting, budget, None)
436 + scheduler.add_seq_group(seq_group)
437 + output = scheduler._schedule_prefills(budget, None)
438 + remaining_waiting = scheduler.waiting
439 assert len(output.ignored_seq_groups) == 0
440 assert len(output.seq_groups) == 0
441 assert budget.num_batched_tokens == 0
@@ -453,7 +449,6 @@ def test_prefill_schedule_max_lora():
449 """
450 lora_config = LoRAConfig(max_lora_rank=8, max_loras=1)
451 scheduler = initialize_scheduler(lora_config=lora_config)
456 - waiting: Deque[SequenceGroup] = deque()
452 budget = create_token_budget(token_budget=120)
453 curr_loras: Set[int] = set()
454 for i in range(2):
@@ -463,7 +458,7 @@ def test_prefill_schedule_max_lora():
458 lora_name=str(i),
459 lora_int_id=i + 1,
460 lora_path="abc"))
466 - waiting.append(seq_group)
461 + scheduler.add_seq_group(seq_group)
462 # Add two more requests to verify lora is prioritized.
463 # 0: Lora, 1: Lora, 2: regular, 3: regular
464 # In the first iteration, index 0, 2 is scheduled.
@@ -471,10 +466,10 @@ def test_prefill_schedule_max_lora():
466 # prioritized. Verify that.
467 for i in range(2, 4):
468 _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
474 - waiting.append(seq_group)
469 + scheduler.add_seq_group(seq_group)
470 # Schedule 2 requests (0 and 2)
476 - remaining_waiting, output = scheduler._schedule_prefills(
477 - waiting, budget, curr_loras)
471 + output = scheduler._schedule_prefills(budget, curr_loras)
472 + remaining_waiting = scheduler.waiting
473 assert len(output.ignored_seq_groups) == 0
474 assert len(output.seq_groups) == 2
475 assert budget.num_batched_tokens == 120
@@ -485,8 +480,8 @@ def test_prefill_schedule_max_lora():
480 # Reset curr_loras so that it can be scheduled.
481 curr_loras = set()
482 budget = create_token_budget(token_budget=60)
488 - remaining_waiting, output = scheduler._schedule_prefills(
489 - remaining_waiting, budget, curr_loras)
483 + output = scheduler._schedule_prefills(budget, curr_loras)
484 + remaining_waiting = scheduler.waiting
485 assert len(output.seq_groups) == 1
486 assert output.seq_groups[0].seq_group.request_id == "1"
487 assert len(remaining_waiting) == 1
@@ -499,31 +494,29 @@ def test_prefill_schedule_no_block_manager_capacity():
494 Test sequence cannot be scheduled due to block manager has no capacity.
495 """
496 scheduler = initialize_scheduler()
502 - waiting: Deque[SequenceGroup] = deque()
497 budget = create_token_budget()
498 for i in range(3):
499 _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
506 - waiting.append(seq_group)
500 + scheduler.add_seq_group(seq_group)
501 scheduler.block_manager.can_allocate = MagicMock()
502 scheduler.block_manager.can_allocate.return_value = AllocStatus.LATER
509 - remainig_waiting, output = scheduler._schedule_prefills(
510 - waiting, budget, None)
503 + output = scheduler._schedule_prefills(budget, None)
504 + remaining_waiting = scheduler.waiting
505 assert len(output.ignored_seq_groups) == 0
506 assert len(output.seq_groups) == 0
507 assert budget.num_batched_tokens == 0
508 assert budget.num_curr_seqs == 0
515 - assert len(remainig_waiting) == 3
509 + assert len(remaining_waiting) == 3
510
511 scheduler = initialize_scheduler()
518 - waiting = deque()
512 budget = create_token_budget()
513 for i in range(3):
514 _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
522 - waiting.append(seq_group)
515 + scheduler.add_seq_group(seq_group)
516 scheduler.block_manager.can_allocate = MagicMock()
517 scheduler.block_manager.can_allocate.return_value = AllocStatus.NEVER
525 - remaining_waiting, output = scheduler._schedule_prefills(
526 - waiting, budget, None)
518 + output = scheduler._schedule_prefills(budget, None)
519 + remaining_waiting = scheduler.waiting
520 assert len(output.ignored_seq_groups) == 3
521 assert len(output.seq_groups) == 0
522 assert budget.num_batched_tokens == 0
@@ -536,14 +529,12 @@ def test_decode_schedule_preempted():
529 Test decodes cannot be scheduled and preempted.
530 """
531 scheduler = initialize_scheduler()
539 - running: Deque[SequenceGroup] = deque()
540 - policy = PolicyFactory.get_policy(policy_name="fcfs")
532 curr_loras = None
533 for i in range(3):
534 _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
535 scheduler._allocate_and_set_running(seq_group)
536 append_new_token_seq_group(60, seq_group, 1)
546 - running.append(seq_group)
537 + scheduler._add_seq_group_to_running(seq_group)
538 scheduler.block_manager.can_append_slots = MagicMock()
539
540 def cannot_append_second_group(seq_group, num_lookahead_slots):
@@ -555,8 +546,8 @@ def test_decode_schedule_preempted():
546 # 1 cannot be scheduled, and the lowest priority (request 2)
547 # should be preempted. 1 will also be preempted.
548 budget = create_token_budget()
558 - remainig_running, output = scheduler._schedule_running(
559 - running, budget, curr_loras, policy)
549 + output = scheduler._schedule_running(budget, curr_loras)
550 + remainig_running = scheduler.running
551 assert len(remainig_running) == 0
552 assert len(output.decode_seq_groups) == 1
553 assert len(output.prefill_seq_groups) == 0
@@ -577,14 +568,12 @@ def test_decode_swap_beam_search():
568 Test best_of > 1 swap out blocks
569 """
570 scheduler = initialize_scheduler()
580 - running: Deque[SequenceGroup] = deque()
581 - policy = PolicyFactory.get_policy(policy_name="fcfs")
571 curr_loras = None
572 budget = create_token_budget()
573 for i in range(3):
574 _, seq_group = create_dummy_prompt(str(i), prompt_length=60, best_of=2)
575 scheduler._allocate_and_set_running(seq_group)
587 - running.append(seq_group)
576 + scheduler._add_seq_group_to_running(seq_group)
577 append_new_token_seq_group(60, seq_group, 1)
578 budget.add_num_seqs(seq_group.request_id,
579 seq_group.get_max_num_running_seqs())
@@ -603,8 +592,8 @@ def test_decode_swap_beam_search():
592 expected_swap_mapping = [("5", "7")]
593 scheduler.block_manager.swap_out.return_value = expected_swap_mapping
594
606 - remainig_running, output = scheduler._schedule_running(
607 - running, budget, curr_loras, policy)
595 + output = scheduler._schedule_running(budget, curr_loras)
596 + remainig_running = scheduler.running
597 assert len(remainig_running) == 0
598 assert len(output.decode_seq_groups) == 2
599 assert len(output.prefill_seq_groups) == 0
@@ -628,20 +617,18 @@ def test_schedule_decode_blocks_to_copy_update():
617 """
618 scheduler = initialize_scheduler()
619 _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
631 - running: Deque[SequenceGroup] = deque()
632 - policy = PolicyFactory.get_policy(policy_name="fcfs")
620 curr_loras = None
621 scheduler._allocate_and_set_running(seq_group)
622 append_new_token_seq_group(60, seq_group, 1)
636 - running.append(seq_group)
623 + scheduler._add_seq_group_to_running(seq_group)
624
625 # The last request should be swapped out.
626 scheduler.block_manager.append_slots = MagicMock()
627 scheduler.block_manager.append_slots.return_value = [(2, 3)]
628
629 budget = create_token_budget()
643 - remaining_running, output = scheduler._schedule_running(
644 - running, budget, curr_loras, policy)
630 + output = scheduler._schedule_running(budget, curr_loras)
631 + remaining_running = scheduler.running
632 assert len(remaining_running) == 0
633 assert len(output.decode_seq_groups) == 1
634 assert len(output.prefill_seq_groups) == 0
@@ -656,19 +643,17 @@ def test_schedule_decode_blocks_to_copy_update():
643
644 def test_schedule_swapped_simple():
645 scheduler = initialize_scheduler()
659 - swapped: Deque[SequenceGroup] = deque()
660 - policy = PolicyFactory.get_policy(policy_name="fcfs")
646 curr_loras = None
647 blocks_to_swap_out: List[Tuple[int, int]] = []
648 _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
649 scheduler._allocate_and_set_running(seq_group)
650 append_new_token_seq_group(60, seq_group, 1)
651 scheduler._swap_out(seq_group, blocks_to_swap_out)
667 - swapped.append(seq_group)
652 + scheduler._add_seq_group_to_swapped(seq_group)
653
654 budget = create_token_budget()
670 - remaining_swapped, output = scheduler._schedule_swapped(
671 - swapped, budget, curr_loras, policy)
655 + output = scheduler._schedule_swapped(budget, curr_loras)
656 + remaining_swapped = scheduler.swapped
657 assert len(remaining_swapped) == 0
658 assert budget.num_batched_tokens == 1
659 assert budget.num_curr_seqs == 2
@@ -683,8 +668,6 @@ def test_schedule_swapped_simple():
668
669 def test_schedule_swapped_max_token_budget():
670 scheduler = initialize_scheduler()
686 - swapped: Deque[SequenceGroup] = deque()
687 - policy = PolicyFactory.get_policy(policy_name="fcfs")
671 curr_loras = None
672 blocks_to_swap_out: List[Tuple[int, int]] = []
673 for _ in range(2):
@@ -692,11 +675,11 @@ def test_schedule_swapped_max_token_budget():
675 scheduler._allocate_and_set_running(seq_group)
676 append_new_token_seq_group(60, seq_group, 1)
677 scheduler._swap_out(seq_group, blocks_to_swap_out)
695 - swapped.append(seq_group)
678 + scheduler._add_seq_group_to_swapped(seq_group)
679
680 budget = create_token_budget(token_budget=1)
698 - remaining_swapped, output = scheduler._schedule_swapped(
699 - swapped, budget, curr_loras, policy)
681 + output = scheduler._schedule_swapped(budget, curr_loras)
682 + remaining_swapped = scheduler.swapped
683 assert len(remaining_swapped) == 1
684 assert budget.num_batched_tokens == 1
685 assert budget.num_curr_seqs == 2
@@ -706,8 +689,8 @@ def test_schedule_swapped_max_token_budget():
689 # Verify num_batched_tokens are respected.
690 budget = create_token_budget(token_budget=1)
691 add_token_budget(budget, 1, 0)
709 - remaining_swapped, output = scheduler._schedule_swapped(
710 - remaining_swapped, budget, curr_loras, policy)
692 + output = scheduler._schedule_swapped(budget, curr_loras)
693 + remaining_swapped = scheduler.swapped
694 assert len(remaining_swapped) == 1
695 assert budget.num_batched_tokens == 1
696 assert budget.num_curr_seqs == 0
@@ -717,8 +700,6 @@ def test_schedule_swapped_max_token_budget():
700
701 def test_schedule_swapped_max_seqs():
702 scheduler = initialize_scheduler()
720 - swapped: Deque[SequenceGroup] = deque()
721 - policy = PolicyFactory.get_policy(policy_name="fcfs")
703 curr_loras = None
704 blocks_to_swap_out: List[Tuple[int, int]] = []
705 for i in range(4):
@@ -726,11 +707,11 @@ def test_schedule_swapped_max_seqs():
707 scheduler._allocate_and_set_running(seq_group)
708 append_new_token_seq_group(60, seq_group, 1)
709 scheduler._swap_out(seq_group, blocks_to_swap_out)
729 - swapped.append(seq_group)
710 + scheduler._add_seq_group_to_swapped(seq_group)
711
712 budget = create_token_budget(max_num_seqs=2)
732 - remaining_swapped, output = scheduler._schedule_swapped(
733 - swapped, budget, curr_loras, policy)
713 + output = scheduler._schedule_swapped(budget, curr_loras)
714 + remaining_swapped = scheduler.swapped
715 assert len(remaining_swapped) == 2
716 assert budget.num_batched_tokens == 2
717 assert budget.num_curr_seqs == 2
@@ -738,8 +719,8 @@ def test_schedule_swapped_max_seqs():
719 assert len(output.prefill_seq_groups) == 0
720
721 # Verify num_curr_seqs are respected.
741 - remaining_swapped, output = scheduler._schedule_swapped(
742 - remaining_swapped, budget, curr_loras, policy)
722 + output = scheduler._schedule_swapped(budget, curr_loras)
723 + remaining_swapped = scheduler.swapped
724 assert len(remaining_swapped) == 2
725 assert budget.num_batched_tokens == 2
726 assert budget.num_curr_seqs == 2
@@ -750,8 +731,6 @@ def test_schedule_swapped_max_seqs():
731 def test_schedule_swapped_max_loras():
732 lora_config = LoRAConfig(max_lora_rank=8, max_loras=1)
733 scheduler = initialize_scheduler(lora_config=lora_config)
753 - swapped: Deque[SequenceGroup] = deque()
754 - policy = PolicyFactory.get_policy(policy_name="fcfs")
734 curr_loras: Set[int] = set()
735 blocks_to_swap_out: List[Tuple[int, int]] = []
736 for i in range(2):
@@ -764,11 +743,11 @@ def test_schedule_swapped_max_loras():
743 scheduler._allocate_and_set_running(seq_group)
744 append_new_token_seq_group(60, seq_group, 1)
745 scheduler._swap_out(seq_group, blocks_to_swap_out)
767 - swapped.append(seq_group)
746 + scheduler._add_seq_group_to_swapped(seq_group)
747
748 budget = create_token_budget()
770 - remaining_swapped, output = scheduler._schedule_swapped(
771 - swapped, budget, curr_loras, policy)
749 + output = scheduler._schedule_swapped(budget, curr_loras)
750 + remaining_swapped = scheduler.swapped
751 assert len(remaining_swapped) == 1
752 assert budget.num_batched_tokens == 1
753 assert budget.num_curr_seqs == 1
@@ -779,8 +758,6 @@ def test_schedule_swapped_max_loras():
758
759 def test_schedule_swapped_cannot_swap_in():
760 scheduler = initialize_scheduler()
782 - swapped: Deque[SequenceGroup] = deque()
783 - policy = PolicyFactory.get_policy(policy_name="fcfs")
761 curr_loras = None
762 blocks_to_swap_out: List[Tuple[int, int]] = []
763 for _ in range(2):
@@ -788,15 +765,15 @@ def test_schedule_swapped_cannot_swap_in():
765 scheduler._allocate_and_set_running(seq_group)
766 append_new_token_seq_group(60, seq_group, 1)
767 scheduler._swap_out(seq_group, blocks_to_swap_out)
791 - swapped.append(seq_group)
768 + scheduler._add_seq_group_to_swapped(seq_group)
769
770 # The last request should be swapped out.
771 scheduler.block_manager.can_swap_in = MagicMock()
772 scheduler.block_manager.can_swap_in.return_value = AllocStatus.LATER
773 # Since we cannot swap in, none of the requests are swapped in.
774 budget = create_token_budget()
798 - remaining_swapped, output = scheduler._schedule_swapped(
799 - swapped, budget, curr_loras, policy)
775 + output = scheduler._schedule_swapped(budget, curr_loras)
776 + remaining_swapped = scheduler.swapped
777 assert len(remaining_swapped) == 2
778 assert budget.num_batched_tokens == 0
779 assert budget.num_curr_seqs == 0
@@ -806,8 +783,6 @@ def test_schedule_swapped_cannot_swap_in():
783
784 def test_infeasible_swap():
785 scheduler = initialize_scheduler()
809 - swapped: Deque[SequenceGroup] = deque()
810 - policy = PolicyFactory.get_policy(policy_name="fcfs")
786 curr_loras = None
787 blocks_to_swap_out: List[Tuple[int, int]] = []
788 for _ in range(2):
@@ -815,15 +790,15 @@ def test_infeasible_swap():
790 scheduler._allocate_and_set_running(seq_group)
791 append_new_token_seq_group(60, seq_group, 1)
792 scheduler._swap_out(seq_group, blocks_to_swap_out)
818 - swapped.append(seq_group)
793 + scheduler._add_seq_group_to_swapped(seq_group)
794
795 # The last request should be swapped out.
796 scheduler.block_manager.can_swap_in = MagicMock()
797 scheduler.block_manager.can_swap_in.return_value = AllocStatus.NEVER
798 # Since we cannot swap in, none of the requests are swapped in.
799 budget = create_token_budget()
825 - remaining_swapped, output = scheduler._schedule_swapped(
826 - swapped, budget, curr_loras, policy)
800 + output = scheduler._schedule_swapped(budget, curr_loras)
801 + remaining_swapped = scheduler.swapped
802 assert len(remaining_swapped) == 0
803 assert len(output.infeasible_seq_groups) == 2
804 assert budget.num_batched_tokens == 0
@@ -834,23 +809,21 @@ def test_infeasible_swap():
809
810 def test_schedule_swapped_blocks_to_copy():
811 scheduler = initialize_scheduler()
837 - swapped: Deque[SequenceGroup] = deque()
838 - policy = PolicyFactory.get_policy(policy_name="fcfs")
812 curr_loras = None
813 _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
814 scheduler._allocate_and_set_running(seq_group)
815 append_new_token_seq_group(60, seq_group, 1)
816 blocks_to_swap_out: List[Tuple[int, int]] = []
817 scheduler._swap_out(seq_group, blocks_to_swap_out)
845 - swapped.append(seq_group)
818 + scheduler._add_seq_group_to_swapped(seq_group)
819
820 # The last request should be swapped out.
821 scheduler.block_manager.append_slots = MagicMock()
822 scheduler.block_manager.append_slots.return_value = [(2, 3)]
823
824 budget = create_token_budget()
852 - remaining_swapped, output = scheduler._schedule_swapped(
853 - swapped, budget, curr_loras, policy)
825 + output = scheduler._schedule_swapped(budget, curr_loras)
826 + remaining_swapped = scheduler.swapped
827 assert len(remaining_swapped) == 0
828 assert len(output.decode_seq_groups) == 1
829 assert len(output.prefill_seq_groups) == 0
vllm/core/policy.py deleted
-45
@@ -1,45 +0,0 @@
1 -from collections import deque
2 -from typing import Deque
3 -
4 -from vllm.sequence import SequenceGroup
5 -
6 -
7 -class Policy:
8 -
9 - def get_priority(
10 - self,
11 - now: float,
12 - seq_group: SequenceGroup,
13 - ) -> float:
14 - raise NotImplementedError
15 -
16 - def sort_by_priority(
17 - self,
18 - now: float,
19 - seq_groups: Deque[SequenceGroup],
20 - ) -> Deque[SequenceGroup]:
21 - return deque(
22 - sorted(
23 - seq_groups,
24 - key=lambda seq_group: self.get_priority(now, seq_group),
25 - reverse=True,
26 - ))
27 -
28 -
29 -class FCFS(Policy):
30 -
31 - def get_priority(
32 - self,
33 - now: float,
34 - seq_group: SequenceGroup,
35 - ) -> float:
36 - return now - seq_group.metrics.arrival_time
37 -
38 -
39 -class PolicyFactory:
40 -
41 - _POLICY_REGISTRY = {'fcfs': FCFS}
42 -
43 - @classmethod
44 - def get_policy(cls, policy_name: str, **kwargs) -> Policy:
45 - return cls._POLICY_REGISTRY[policy_name](**kwargs)
vllm/core/scheduler.py
+43 -73
@@ -8,7 +8,6 @@ from typing import Deque, Dict, Iterable, List, Optional, Set, Tuple, Union
8
9 from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig
10 from vllm.core.interfaces import AllocStatus, BlockSpaceManager
11 -from vllm.core.policy import Policy, PolicyFactory
11 from vllm.logger import init_logger
12 from vllm.lora.request import LoRARequest
13 from vllm.prompt_adapter.request import PromptAdapterRequest
@@ -345,6 +344,16 @@ class Scheduler:
344 # Add sequence groups to the waiting queue.
345 self.waiting.append(seq_group)
346
347 + def _add_seq_group_to_running(self, seq_group: SequenceGroup) -> None:
348 + # Add sequence groups to the running queue.
349 + # Only for testing purposes.
350 + self.running.append(seq_group)
351 +
352 + def _add_seq_group_to_swapped(self, seq_group: SequenceGroup) -> None:
353 + # Add sequence groups to the swapped queue.
354 + # Only for testing purposes.
355 + self.swapped.append(seq_group)
356 +
357 def abort_seq_group(self, request_id: Union[str, Iterable[str]]) -> None:
358 """Aborts a sequence group with the given ID.
359
@@ -398,32 +407,26 @@ class Scheduler:
407
408 def _schedule_running(
409 self,
401 - running_queue: deque,
410 budget: SchedulingBudget,
411 curr_loras: Optional[Set[int]],
404 - policy: Policy,
412 enable_chunking: bool = False,
406 - ) -> Tuple[deque, SchedulerRunningOutputs]:
413 + ) -> SchedulerRunningOutputs:
414 """Schedule sequence groups that are running.
415
416 Running queue should include decode and chunked prefill requests.
417
418 Args:
412 - running_queue: The queue that contains running requests (i.e.,
413 - decodes). The given arguments are NOT in-place modified.
419 budget: The scheduling budget. The argument is in-place updated
420 when any decodes are preempted.
421 curr_loras: Currently batched lora request ids. The argument is
422 in-place updated when any decodes are preempted.
418 - policy: The sorting policy to sort running_queue.
423 enable_chunking: If True, seq group can be chunked and only a
424 chunked number of tokens are scheduled if
425 `budget.num_batched_tokens` has not enough capacity to schedule
426 all tokens.
427
428 Returns:
425 - A tuple of remaining running queue (should be always 0) after
426 - scheduling and SchedulerRunningOutputs.
429 + SchedulerRunningOutputs.
430 """
431 # Blocks that need to be swapped or copied before model execution.
432 blocks_to_swap_out: List[Tuple[int, int]] = []
@@ -436,10 +439,9 @@ class Scheduler:
439
440 # NOTE(woosuk): Preemption happens only when there is no available slot
441 # to keep all the sequence groups in the RUNNING state.
439 - # In this case, the policy is responsible for deciding which sequence
440 - # groups to preempt.
441 - now = time.time()
442 - running_queue = policy.sort_by_priority(now, running_queue)
442 +
443 + running_queue = self.running
444 +
445 while running_queue:
446 seq_group = running_queue[0]
447 num_running_tokens = self._get_num_new_tokens(
@@ -503,7 +505,7 @@ class Scheduler:
505 if curr_loras is not None and seq_group.lora_int_id > 0:
506 curr_loras.add(seq_group.lora_int_id)
507
506 - return running_queue, SchedulerRunningOutputs(
508 + return SchedulerRunningOutputs(
509 decode_seq_groups=decode_seq_groups,
510 prefill_seq_groups=prefill_seq_groups,
511 preempted=preempted,
@@ -515,12 +517,10 @@ class Scheduler:
517
518 def _schedule_swapped(
519 self,
518 - swapped_queue: deque,
520 budget: SchedulingBudget,
521 curr_loras: Optional[Set[int]],
521 - policy: Policy,
522 enable_chunking: bool = False,
523 - ) -> Tuple[deque, SchedulerSwappedInOutputs]:
523 + ) -> SchedulerSwappedInOutputs:
524 """Schedule sequence groups that are swapped out.
525
526 It schedules swapped requests as long as it fits `budget` and
@@ -528,20 +528,16 @@ class Scheduler:
528 `budget` and `curr_loras` are updated based on scheduled seq_groups.
529
530 Args:
531 - swapped_queue: The queue that contains swapped out requests.
532 - The given arguments are NOT in-place modified.
531 budget: The scheduling budget. The argument is in-place updated
532 when any requests are swapped in.
533 curr_loras: Currently batched lora request ids. The argument is
534 in-place updated when any requests are swapped in.
537 - policy: The sorting policy to sort swapped_queue.
535 enable_chunking: If True, seq group can be chunked and only a
536 chunked number of tokens are scheduled if
537 `budget.num_batched_tokens` has not enough capacity to schedule
538 all tokens.
539
540 Returns:
544 - A tuple of remaining swapped_queue after scheduling and
541 SchedulerSwappedInOutputs.
542 """
543 # Blocks that need to be swapped or copied before model execution.
@@ -549,10 +545,10 @@ class Scheduler:
545 blocks_to_copy: List[Tuple[int, int]] = []
546 decode_seq_groups: List[ScheduledSequenceGroup] = []
547 prefill_seq_groups: List[ScheduledSequenceGroup] = []
552 - now = time.time()
553 - swapped_queue = policy.sort_by_priority(now, swapped_queue)
548 infeasible_seq_groups: List[SequenceGroup] = []
549
550 + swapped_queue = self.swapped
551 +
552 leftover_swapped: Deque[SequenceGroup] = deque()
553 while swapped_queue:
554 seq_group = swapped_queue[0]
@@ -617,7 +613,7 @@ class Scheduler:
613
614 swapped_queue.extendleft(leftover_swapped)
615
620 - return swapped_queue, SchedulerSwappedInOutputs(
616 + return SchedulerSwappedInOutputs(
617 decode_seq_groups=decode_seq_groups,
618 prefill_seq_groups=prefill_seq_groups,
619 blocks_to_swap_in=blocks_to_swap_in,
@@ -644,11 +640,10 @@ class Scheduler:
640
641 def _schedule_prefills(
642 self,
647 - waiting_queue: deque,
643 budget: SchedulingBudget,
644 curr_loras: Optional[Set[int]],
645 enable_chunking: bool = False,
651 - ) -> Tuple[deque, SchedulerPrefillOutputs]:
646 + ) -> SchedulerPrefillOutputs:
647 """Schedule sequence groups that are in prefill stage.
648
649 Note that the current scheduler treats PREEMPTED_FOR_RECOMPUTE
@@ -660,8 +655,6 @@ class Scheduler:
655 `budget` and `curr_loras` are updated based on scheduled seq_groups.
656
657 Args:
663 - waiting_queue: The queue that contains prefill requests.
664 - The given arguments are NOT in-place modified.
658 budget: The scheduling budget. The argument is in-place updated
659 when any requests are scheduled.
660 curr_loras: Currently batched lora request ids. The argument is
@@ -672,14 +665,12 @@ class Scheduler:
665 all tokens.
666
667 Returns:
675 - A tuple of remaining waiting_queue after scheduling and
668 SchedulerSwappedInOutputs.
669 """
670 ignored_seq_groups: List[SequenceGroup] = []
671 seq_groups: List[SequenceGroup] = []
680 - # We don't sort waiting queue because we assume it is sorted.
681 - # Copy the queue so that the input queue is not modified.
682 - waiting_queue = deque([s for s in waiting_queue])
672 +
673 + waiting_queue = self.waiting
674
675 leftover_waiting_sequences: Deque[SequenceGroup] = deque()
676 while self._passed_delay(time.time()) and waiting_queue:
@@ -758,7 +749,7 @@ class Scheduler:
749 if len(seq_groups) > 0:
750 self.prev_prompt = True
751
761 - return waiting_queue, SchedulerPrefillOutputs(
752 + return SchedulerPrefillOutputs(
753 seq_groups=seq_groups,
754 ignored_seq_groups=ignored_seq_groups,
755 num_lookahead_slots=self._get_num_lookahead_slots(is_prefill=True))
@@ -785,53 +776,43 @@ class Scheduler:
776 seq_group.lora_int_id for seq_group in self.running
777 if seq_group.lora_int_id > 0) if self.lora_enabled else None
778
788 - remaining_waiting, prefills = (self.waiting,
789 - SchedulerPrefillOutputs.create_empty())
790 - remaining_running, running_scheduled = (
791 - self.running, SchedulerRunningOutputs.create_empty())
792 - remaining_swapped, swapped_in = (
793 - self.swapped, SchedulerSwappedInOutputs.create_empty())
779 + prefills = SchedulerPrefillOutputs.create_empty()
780 + running_scheduled = SchedulerRunningOutputs.create_empty()
781 + swapped_in = SchedulerSwappedInOutputs.create_empty()
782
783 # If any requests are swapped, prioritized swapped requests.
784 if not self.swapped:
797 - remaining_waiting, prefills = self._schedule_prefills(
798 - self.waiting, budget, curr_loras, enable_chunking=False)
785 + prefills = self._schedule_prefills(budget,
786 + curr_loras,
787 + enable_chunking=False)
788
800 - fcfs_policy = PolicyFactory.get_policy(policy_name="fcfs")
789 # Don't schedule decodes if prefills are scheduled.
790 # NOTE: If `_schedule_prefills` doesn't enable chunking, self.running
791 # only contains decode requests, not chunked prefills.
792 if len(prefills.seq_groups) == 0:
805 - remaining_running, running_scheduled = self._schedule_running(
806 - self.running,
807 - budget,
808 - curr_loras,
809 - fcfs_policy,
810 - enable_chunking=False)
793 + running_scheduled = self._schedule_running(budget,
794 + curr_loras,
795 + enable_chunking=False)
796
797 # If any sequence group is preempted, do not swap in any sequence
798 # group. because it means there's no slot for new running requests.
799 if len(running_scheduled.preempted) + len(
800 running_scheduled.swapped_out) == 0:
816 - remaining_swapped, swapped_in = self._schedule_swapped(
817 - self.swapped, budget, curr_loras, fcfs_policy)
801 + swapped_in = self._schedule_swapped(budget, curr_loras)
802
803 assert (budget.num_batched_tokens <=
804 self.scheduler_config.max_num_batched_tokens)
805 assert budget.num_curr_seqs <= self.scheduler_config.max_num_seqs
806
807 # Update waiting requests.
824 - self.waiting = remaining_waiting
808 self.waiting.extendleft(running_scheduled.preempted)
809 # Update new running requests.
827 - self.running = remaining_running
810 self.running.extend([s.seq_group for s in prefills.seq_groups])
811 self.running.extend(
812 [s.seq_group for s in running_scheduled.decode_seq_groups])
813 self.running.extend(
814 [s.seq_group for s in swapped_in.decode_seq_groups])
815 # Update swapped requests.
834 - self.swapped = remaining_swapped
816 self.swapped.extend(running_scheduled.swapped_out)
817 preempted = (len(running_scheduled.preempted) +
818 len(running_scheduled.swapped_out))
@@ -877,42 +858,32 @@ class Scheduler:
858 )
859 curr_loras: Set[int] = set()
860
880 - remaining_waiting, prefills = (self.waiting,
881 - SchedulerPrefillOutputs.create_empty())
882 - remaining_running, running_scheduled = (
883 - self.running, SchedulerRunningOutputs.create_empty())
884 - remaining_swapped, swapped_in = (
885 - self.swapped, SchedulerSwappedInOutputs.create_empty())
861 + prefills = SchedulerPrefillOutputs.create_empty()
862 + swapped_in = SchedulerSwappedInOutputs.create_empty()
863
864 # Decoding should be always scheduled first by fcfs.
888 - fcfs_policy = PolicyFactory.get_policy(policy_name="fcfs")
889 - remaining_running, running_scheduled = self._schedule_running(
890 - self.running,
891 - budget,
892 - curr_loras,
893 - fcfs_policy,
894 - enable_chunking=True)
865 + running_scheduled = self._schedule_running(budget,
866 + curr_loras,
867 + enable_chunking=True)
868
869 # Schedule swapped out requests.
870 # If preemption happens, it means we don't have space for swap-in.
871 if len(running_scheduled.preempted) + len(
872 running_scheduled.swapped_out) == 0:
900 - remaining_swapped, swapped_in = self._schedule_swapped(
901 - self.swapped, budget, curr_loras, fcfs_policy)
873 + swapped_in = self._schedule_swapped(budget, curr_loras)
874
875 # Schedule new prefills.
904 - remaining_waiting, prefills = self._schedule_prefills(
905 - self.waiting, budget, curr_loras, enable_chunking=True)
876 + prefills = self._schedule_prefills(budget,
877 + curr_loras,
878 + enable_chunking=True)
879
880 assert (budget.num_batched_tokens <=
881 self.scheduler_config.max_num_batched_tokens)
882 assert budget.num_curr_seqs <= self.scheduler_config.max_num_seqs
883
884 # Update waiting requests.
912 - self.waiting = remaining_waiting
885 self.waiting.extendleft(running_scheduled.preempted)
886 # Update new running requests.
915 - self.running = remaining_running
887 self.running.extend([s.seq_group for s in prefills.seq_groups])
888 self.running.extend(
889 [s.seq_group for s in running_scheduled.decode_seq_groups])
@@ -923,7 +894,6 @@ class Scheduler:
894 self.running.extend(
895 [s.seq_group for s in swapped_in.prefill_seq_groups])
896 # Update swapped requests.
926 - self.swapped = remaining_swapped
897 self.swapped.extend(running_scheduled.swapped_out)
898 return SchedulerOutputs(
899 scheduled_seq_groups=(prefills.seq_groups +