A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[core][scheduler] simplify and improve scheduler (#6867)
youkaichao committed
Jul 31, 2024 at 23:51 UTC
c8a7e93273ff4338d6f89f8a63ff16426ac240b8
4 files changed
+112
-214
tests/core/block/e2e/test_correctness.py
+1
-1
@@ -183,7 +183,7 @@ def test_v1_v2_greedy_equality_with_cow(baseline_llm_generator,
183
184
# Allow only 2 sequences of ~128 tokens in worst case.
185
# Note 16 = 128/block_size
186
- "num_gpu_blocks_override": 2 * (16 + 1),
186
+ "num_gpu_blocks_override": 2 * (16 + 2),
187
}
188
])
189
@pytest.mark.parametrize("baseline_llm_kwargs", [{
tests/core/test_scheduler.py
+68
-95
@@ -1,13 +1,12 @@
1
import time
2
from collections import deque
3
-from typing import Deque, List, Set, Tuple
3
+from typing import List, Set, Tuple
4
from unittest.mock import MagicMock
5
6
import pytest # noqa
7
8
from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig
9
from vllm.core.interfaces import AllocStatus
10
-from vllm.core.policy import PolicyFactory
10
from vllm.core.scheduler import Scheduler, SchedulingBudget
11
from vllm.lora.request import LoRARequest
12
from vllm.sequence import Logprob, SequenceGroup, SequenceStatus
@@ -348,10 +347,10 @@ def test_prefill_schedule_max_prompt_len():
347
"""
348
scheduler = initialize_scheduler(max_model_len=30)
349
_, seq_group = create_dummy_prompt("0", prompt_length=60)
351
- waiting = deque([seq_group])
350
+ scheduler.add_seq_group(seq_group)
351
budget = create_token_budget()
353
- remaining_waiting, output = scheduler._schedule_prefills(
354
- waiting, budget, None)
352
+ output = scheduler._schedule_prefills(budget, None)
353
+ remaining_waiting = scheduler.waiting
354
assert len(output.ignored_seq_groups) == 1
355
assert len(output.seq_groups) == 0
356
assert budget.num_batched_tokens == 0
@@ -364,15 +363,14 @@ def test_prefill_schedule_token_budget():
363
Test token budget respected.
364
"""
365
scheduler = initialize_scheduler()
367
- waiting: Deque[SequenceGroup] = deque()
366
budget = create_token_budget(token_budget=0)
367
for i in range(2):
368
_, seq_group = create_dummy_prompt(str(i), prompt_length=60)
371
- waiting.append(seq_group)
369
+ scheduler.add_seq_group(seq_group)
370
371
# 0 token budget == nothing is scheduled.
374
- remaining_waiting, output = scheduler._schedule_prefills(
375
- waiting, budget, None)
372
+ output = scheduler._schedule_prefills(budget, None)
373
+ remaining_waiting = scheduler.waiting
374
assert len(output.ignored_seq_groups) == 0
375
assert len(output.seq_groups) == 0
376
assert budget.num_batched_tokens == 0
@@ -381,8 +379,8 @@ def test_prefill_schedule_token_budget():
379
380
# 60 token budget == 1 request scheduled.
381
budget = create_token_budget(token_budget=60)
384
- remaining_waiting, output = scheduler._schedule_prefills(
385
- waiting, budget, None)
382
+ output = scheduler._schedule_prefills(budget, None)
383
+ remaining_waiting = scheduler.waiting
384
assert len(output.ignored_seq_groups) == 0
385
assert len(output.seq_groups) == 1
386
assert budget.num_batched_tokens == 60
@@ -391,14 +389,13 @@ def test_prefill_schedule_token_budget():
389
390
# Test when current_batched_tokens respected.
391
scheduler = initialize_scheduler()
394
- waiting = deque()
392
budget = create_token_budget(token_budget=60)
393
add_token_budget(budget, 30, 0)
394
_, seq_group = create_dummy_prompt(str(i), prompt_length=60)
395
# Cannot schedule a prompt that doesn't fit the budget.
399
- waiting.append(seq_group)
400
- remaining_waiting, output = scheduler._schedule_prefills(
401
- waiting, budget, None)
396
+ scheduler.add_seq_group(seq_group)
397
+ output = scheduler._schedule_prefills(budget, None)
398
+ remaining_waiting = scheduler.waiting
399
assert len(output.ignored_seq_groups) == 0
400
assert len(output.seq_groups) == 0
401
assert budget.num_batched_tokens == 30
@@ -406,8 +403,8 @@ def test_prefill_schedule_token_budget():
403
assert len(remaining_waiting) == 1
404
budget = create_token_budget(token_budget=90)
405
add_token_budget(budget, 30, 0)
409
- remaining_waiting, output = scheduler._schedule_prefills(
410
- waiting, budget, None)
406
+ output = scheduler._schedule_prefills(budget, None)
407
+ remaining_waiting = scheduler.waiting
408
assert len(output.seq_groups) == 1
409
assert budget.num_batched_tokens == 90
410
assert budget.num_curr_seqs == 1
@@ -419,13 +416,12 @@ def test_prefill_schedule_max_seqs():
416
Test max seq respected.
417
"""
418
scheduler = initialize_scheduler()
422
- waiting: Deque[SequenceGroup] = deque()
419
budget = create_token_budget(max_num_seqs=2)
420
for i in range(3):
421
_, seq_group = create_dummy_prompt(str(i), prompt_length=60)
426
- waiting.append(seq_group)
427
- remaining_waiting, output = scheduler._schedule_prefills(
428
- waiting, budget, None)
422
+ scheduler.add_seq_group(seq_group)
423
+ output = scheduler._schedule_prefills(budget, None)
424
+ remaining_waiting = scheduler.waiting
425
assert len(output.ignored_seq_groups) == 0
426
assert len(output.seq_groups) == 2
427
assert budget.num_batched_tokens == 120
@@ -433,13 +429,13 @@ def test_prefill_schedule_max_seqs():
429
assert len(remaining_waiting) == 1
430
431
# Verify curr_num_seqs respected.
436
- waiting = deque()
432
+ scheduler.waiting = deque()
433
budget = create_token_budget(max_num_seqs=2)
434
add_token_budget(budget, 0, 2)
435
_, seq_group = create_dummy_prompt(str(i), prompt_length=60)
440
- waiting.append(seq_group)
441
- remaining_waiting, output = scheduler._schedule_prefills(
442
- waiting, budget, None)
436
+ scheduler.add_seq_group(seq_group)
437
+ output = scheduler._schedule_prefills(budget, None)
438
+ remaining_waiting = scheduler.waiting
439
assert len(output.ignored_seq_groups) == 0
440
assert len(output.seq_groups) == 0
441
assert budget.num_batched_tokens == 0
@@ -453,7 +449,6 @@ def test_prefill_schedule_max_lora():
449
"""
450
lora_config = LoRAConfig(max_lora_rank=8, max_loras=1)
451
scheduler = initialize_scheduler(lora_config=lora_config)
456
- waiting: Deque[SequenceGroup] = deque()
452
budget = create_token_budget(token_budget=120)
453
curr_loras: Set[int] = set()
454
for i in range(2):
@@ -463,7 +458,7 @@ def test_prefill_schedule_max_lora():
458
lora_name=str(i),
459
lora_int_id=i + 1,
460
lora_path="abc"))
466
- waiting.append(seq_group)
461
+ scheduler.add_seq_group(seq_group)
462
# Add two more requests to verify lora is prioritized.
463
# 0: Lora, 1: Lora, 2: regular, 3: regular
464
# In the first iteration, index 0, 2 is scheduled.
@@ -471,10 +466,10 @@ def test_prefill_schedule_max_lora():
466
# prioritized. Verify that.
467
for i in range(2, 4):
468
_, seq_group = create_dummy_prompt(str(i), prompt_length=60)
474
- waiting.append(seq_group)
469
+ scheduler.add_seq_group(seq_group)
470
# Schedule 2 requests (0 and 2)
476
- remaining_waiting, output = scheduler._schedule_prefills(
477
- waiting, budget, curr_loras)
471
+ output = scheduler._schedule_prefills(budget, curr_loras)
472
+ remaining_waiting = scheduler.waiting
473
assert len(output.ignored_seq_groups) == 0
474
assert len(output.seq_groups) == 2
475
assert budget.num_batched_tokens == 120
@@ -485,8 +480,8 @@ def test_prefill_schedule_max_lora():
480
# Reset curr_loras so that it can be scheduled.
481
curr_loras = set()
482
budget = create_token_budget(token_budget=60)
488
- remaining_waiting, output = scheduler._schedule_prefills(
489
- remaining_waiting, budget, curr_loras)
483
+ output = scheduler._schedule_prefills(budget, curr_loras)
484
+ remaining_waiting = scheduler.waiting
485
assert len(output.seq_groups) == 1
486
assert output.seq_groups[0].seq_group.request_id == "1"
487
assert len(remaining_waiting) == 1
@@ -499,31 +494,29 @@ def test_prefill_schedule_no_block_manager_capacity():
494
Test sequence cannot be scheduled due to block manager has no capacity.
495
"""
496
scheduler = initialize_scheduler()
502
- waiting: Deque[SequenceGroup] = deque()
497
budget = create_token_budget()
498
for i in range(3):
499
_, seq_group = create_dummy_prompt(str(i), prompt_length=60)
506
- waiting.append(seq_group)
500
+ scheduler.add_seq_group(seq_group)
501
scheduler.block_manager.can_allocate = MagicMock()
502
scheduler.block_manager.can_allocate.return_value = AllocStatus.LATER
509
- remainig_waiting, output = scheduler._schedule_prefills(
510
- waiting, budget, None)
503
+ output = scheduler._schedule_prefills(budget, None)
504
+ remaining_waiting = scheduler.waiting
505
assert len(output.ignored_seq_groups) == 0
506
assert len(output.seq_groups) == 0
507
assert budget.num_batched_tokens == 0
508
assert budget.num_curr_seqs == 0
515
- assert len(remainig_waiting) == 3
509
+ assert len(remaining_waiting) == 3
510
511
scheduler = initialize_scheduler()
518
- waiting = deque()
512
budget = create_token_budget()
513
for i in range(3):
514
_, seq_group = create_dummy_prompt(str(i), prompt_length=60)
522
- waiting.append(seq_group)
515
+ scheduler.add_seq_group(seq_group)
516
scheduler.block_manager.can_allocate = MagicMock()
517
scheduler.block_manager.can_allocate.return_value = AllocStatus.NEVER
525
- remaining_waiting, output = scheduler._schedule_prefills(
526
- waiting, budget, None)
518
+ output = scheduler._schedule_prefills(budget, None)
519
+ remaining_waiting = scheduler.waiting
520
assert len(output.ignored_seq_groups) == 3
521
assert len(output.seq_groups) == 0
522
assert budget.num_batched_tokens == 0
@@ -536,14 +529,12 @@ def test_decode_schedule_preempted():
529
Test decodes cannot be scheduled and preempted.
530
"""
531
scheduler = initialize_scheduler()
539
- running: Deque[SequenceGroup] = deque()
540
- policy = PolicyFactory.get_policy(policy_name="fcfs")
532
curr_loras = None
533
for i in range(3):
534
_, seq_group = create_dummy_prompt(str(i), prompt_length=60)
535
scheduler._allocate_and_set_running(seq_group)
536
append_new_token_seq_group(60, seq_group, 1)
546
- running.append(seq_group)
537
+ scheduler._add_seq_group_to_running(seq_group)
538
scheduler.block_manager.can_append_slots = MagicMock()
539
540
def cannot_append_second_group(seq_group, num_lookahead_slots):
@@ -555,8 +546,8 @@ def test_decode_schedule_preempted():
546
# 1 cannot be scheduled, and the lowest priority (request 2)
547
# should be preempted. 1 will also be preempted.
548
budget = create_token_budget()
558
- remainig_running, output = scheduler._schedule_running(
559
- running, budget, curr_loras, policy)
549
+ output = scheduler._schedule_running(budget, curr_loras)
550
+ remainig_running = scheduler.running
551
assert len(remainig_running) == 0
552
assert len(output.decode_seq_groups) == 1
553
assert len(output.prefill_seq_groups) == 0
@@ -577,14 +568,12 @@ def test_decode_swap_beam_search():
568
Test best_of > 1 swap out blocks
569
"""
570
scheduler = initialize_scheduler()
580
- running: Deque[SequenceGroup] = deque()
581
- policy = PolicyFactory.get_policy(policy_name="fcfs")
571
curr_loras = None
572
budget = create_token_budget()
573
for i in range(3):
574
_, seq_group = create_dummy_prompt(str(i), prompt_length=60, best_of=2)
575
scheduler._allocate_and_set_running(seq_group)
587
- running.append(seq_group)
576
+ scheduler._add_seq_group_to_running(seq_group)
577
append_new_token_seq_group(60, seq_group, 1)
578
budget.add_num_seqs(seq_group.request_id,
579
seq_group.get_max_num_running_seqs())
@@ -603,8 +592,8 @@ def test_decode_swap_beam_search():
592
expected_swap_mapping = [("5", "7")]
593
scheduler.block_manager.swap_out.return_value = expected_swap_mapping
594
606
- remainig_running, output = scheduler._schedule_running(
607
- running, budget, curr_loras, policy)
595
+ output = scheduler._schedule_running(budget, curr_loras)
596
+ remainig_running = scheduler.running
597
assert len(remainig_running) == 0
598
assert len(output.decode_seq_groups) == 2
599
assert len(output.prefill_seq_groups) == 0
@@ -628,20 +617,18 @@ def test_schedule_decode_blocks_to_copy_update():
617
"""
618
scheduler = initialize_scheduler()
619
_, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
631
- running: Deque[SequenceGroup] = deque()
632
- policy = PolicyFactory.get_policy(policy_name="fcfs")
620
curr_loras = None
621
scheduler._allocate_and_set_running(seq_group)
622
append_new_token_seq_group(60, seq_group, 1)
636
- running.append(seq_group)
623
+ scheduler._add_seq_group_to_running(seq_group)
624
625
# The last request should be swapped out.
626
scheduler.block_manager.append_slots = MagicMock()
627
scheduler.block_manager.append_slots.return_value = [(2, 3)]
628
629
budget = create_token_budget()
643
- remaining_running, output = scheduler._schedule_running(
644
- running, budget, curr_loras, policy)
630
+ output = scheduler._schedule_running(budget, curr_loras)
631
+ remaining_running = scheduler.running
632
assert len(remaining_running) == 0
633
assert len(output.decode_seq_groups) == 1
634
assert len(output.prefill_seq_groups) == 0
@@ -656,19 +643,17 @@ def test_schedule_decode_blocks_to_copy_update():
643
644
def test_schedule_swapped_simple():
645
scheduler = initialize_scheduler()
659
- swapped: Deque[SequenceGroup] = deque()
660
- policy = PolicyFactory.get_policy(policy_name="fcfs")
646
curr_loras = None
647
blocks_to_swap_out: List[Tuple[int, int]] = []
648
_, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
649
scheduler._allocate_and_set_running(seq_group)
650
append_new_token_seq_group(60, seq_group, 1)
651
scheduler._swap_out(seq_group, blocks_to_swap_out)
667
- swapped.append(seq_group)
652
+ scheduler._add_seq_group_to_swapped(seq_group)
653
654
budget = create_token_budget()
670
- remaining_swapped, output = scheduler._schedule_swapped(
671
- swapped, budget, curr_loras, policy)
655
+ output = scheduler._schedule_swapped(budget, curr_loras)
656
+ remaining_swapped = scheduler.swapped
657
assert len(remaining_swapped) == 0
658
assert budget.num_batched_tokens == 1
659
assert budget.num_curr_seqs == 2
@@ -683,8 +668,6 @@ def test_schedule_swapped_simple():
668
669
def test_schedule_swapped_max_token_budget():
670
scheduler = initialize_scheduler()
686
- swapped: Deque[SequenceGroup] = deque()
687
- policy = PolicyFactory.get_policy(policy_name="fcfs")
671
curr_loras = None
672
blocks_to_swap_out: List[Tuple[int, int]] = []
673
for _ in range(2):
@@ -692,11 +675,11 @@ def test_schedule_swapped_max_token_budget():
675
scheduler._allocate_and_set_running(seq_group)
676
append_new_token_seq_group(60, seq_group, 1)
677
scheduler._swap_out(seq_group, blocks_to_swap_out)
695
- swapped.append(seq_group)
678
+ scheduler._add_seq_group_to_swapped(seq_group)
679
680
budget = create_token_budget(token_budget=1)
698
- remaining_swapped, output = scheduler._schedule_swapped(
699
- swapped, budget, curr_loras, policy)
681
+ output = scheduler._schedule_swapped(budget, curr_loras)
682
+ remaining_swapped = scheduler.swapped
683
assert len(remaining_swapped) == 1
684
assert budget.num_batched_tokens == 1
685
assert budget.num_curr_seqs == 2
@@ -706,8 +689,8 @@ def test_schedule_swapped_max_token_budget():
689
# Verify num_batched_tokens are respected.
690
budget = create_token_budget(token_budget=1)
691
add_token_budget(budget, 1, 0)
709
- remaining_swapped, output = scheduler._schedule_swapped(
710
- remaining_swapped, budget, curr_loras, policy)
692
+ output = scheduler._schedule_swapped(budget, curr_loras)
693
+ remaining_swapped = scheduler.swapped
694
assert len(remaining_swapped) == 1
695
assert budget.num_batched_tokens == 1
696
assert budget.num_curr_seqs == 0
@@ -717,8 +700,6 @@ def test_schedule_swapped_max_token_budget():
700
701
def test_schedule_swapped_max_seqs():
702
scheduler = initialize_scheduler()
720
- swapped: Deque[SequenceGroup] = deque()
721
- policy = PolicyFactory.get_policy(policy_name="fcfs")
703
curr_loras = None
704
blocks_to_swap_out: List[Tuple[int, int]] = []
705
for i in range(4):
@@ -726,11 +707,11 @@ def test_schedule_swapped_max_seqs():
707
scheduler._allocate_and_set_running(seq_group)
708
append_new_token_seq_group(60, seq_group, 1)
709
scheduler._swap_out(seq_group, blocks_to_swap_out)
729
- swapped.append(seq_group)
710
+ scheduler._add_seq_group_to_swapped(seq_group)
711
712
budget = create_token_budget(max_num_seqs=2)
732
- remaining_swapped, output = scheduler._schedule_swapped(
733
- swapped, budget, curr_loras, policy)
713
+ output = scheduler._schedule_swapped(budget, curr_loras)
714
+ remaining_swapped = scheduler.swapped
715
assert len(remaining_swapped) == 2
716
assert budget.num_batched_tokens == 2
717
assert budget.num_curr_seqs == 2
@@ -738,8 +719,8 @@ def test_schedule_swapped_max_seqs():
719
assert len(output.prefill_seq_groups) == 0
720
721
# Verify num_curr_seqs are respected.
741
- remaining_swapped, output = scheduler._schedule_swapped(
742
- remaining_swapped, budget, curr_loras, policy)
722
+ output = scheduler._schedule_swapped(budget, curr_loras)
723
+ remaining_swapped = scheduler.swapped
724
assert len(remaining_swapped) == 2
725
assert budget.num_batched_tokens == 2
726
assert budget.num_curr_seqs == 2
@@ -750,8 +731,6 @@ def test_schedule_swapped_max_seqs():
731
def test_schedule_swapped_max_loras():
732
lora_config = LoRAConfig(max_lora_rank=8, max_loras=1)
733
scheduler = initialize_scheduler(lora_config=lora_config)
753
- swapped: Deque[SequenceGroup] = deque()
754
- policy = PolicyFactory.get_policy(policy_name="fcfs")
734
curr_loras: Set[int] = set()
735
blocks_to_swap_out: List[Tuple[int, int]] = []
736
for i in range(2):
@@ -764,11 +743,11 @@ def test_schedule_swapped_max_loras():
743
scheduler._allocate_and_set_running(seq_group)
744
append_new_token_seq_group(60, seq_group, 1)
745
scheduler._swap_out(seq_group, blocks_to_swap_out)
767
- swapped.append(seq_group)
746
+ scheduler._add_seq_group_to_swapped(seq_group)
747
748
budget = create_token_budget()
770
- remaining_swapped, output = scheduler._schedule_swapped(
771
- swapped, budget, curr_loras, policy)
749
+ output = scheduler._schedule_swapped(budget, curr_loras)
750
+ remaining_swapped = scheduler.swapped
751
assert len(remaining_swapped) == 1
752
assert budget.num_batched_tokens == 1
753
assert budget.num_curr_seqs == 1
@@ -779,8 +758,6 @@ def test_schedule_swapped_max_loras():
758
759
def test_schedule_swapped_cannot_swap_in():
760
scheduler = initialize_scheduler()
782
- swapped: Deque[SequenceGroup] = deque()
783
- policy = PolicyFactory.get_policy(policy_name="fcfs")
761
curr_loras = None
762
blocks_to_swap_out: List[Tuple[int, int]] = []
763
for _ in range(2):
@@ -788,15 +765,15 @@ def test_schedule_swapped_cannot_swap_in():
765
scheduler._allocate_and_set_running(seq_group)
766
append_new_token_seq_group(60, seq_group, 1)
767
scheduler._swap_out(seq_group, blocks_to_swap_out)
791
- swapped.append(seq_group)
768
+ scheduler._add_seq_group_to_swapped(seq_group)
769
770
# The last request should be swapped out.
771
scheduler.block_manager.can_swap_in = MagicMock()
772
scheduler.block_manager.can_swap_in.return_value = AllocStatus.LATER
773
# Since we cannot swap in, none of the requests are swapped in.
774
budget = create_token_budget()
798
- remaining_swapped, output = scheduler._schedule_swapped(
799
- swapped, budget, curr_loras, policy)
775
+ output = scheduler._schedule_swapped(budget, curr_loras)
776
+ remaining_swapped = scheduler.swapped
777
assert len(remaining_swapped) == 2
778
assert budget.num_batched_tokens == 0
779
assert budget.num_curr_seqs == 0
@@ -806,8 +783,6 @@ def test_schedule_swapped_cannot_swap_in():
783
784
def test_infeasible_swap():
785
scheduler = initialize_scheduler()
809
- swapped: Deque[SequenceGroup] = deque()
810
- policy = PolicyFactory.get_policy(policy_name="fcfs")
786
curr_loras = None
787
blocks_to_swap_out: List[Tuple[int, int]] = []
788
for _ in range(2):
@@ -815,15 +790,15 @@ def test_infeasible_swap():
790
scheduler._allocate_and_set_running(seq_group)
791
append_new_token_seq_group(60, seq_group, 1)
792
scheduler._swap_out(seq_group, blocks_to_swap_out)
818
- swapped.append(seq_group)
793
+ scheduler._add_seq_group_to_swapped(seq_group)
794
795
# The last request should be swapped out.
796
scheduler.block_manager.can_swap_in = MagicMock()
797
scheduler.block_manager.can_swap_in.return_value = AllocStatus.NEVER
798
# Since we cannot swap in, none of the requests are swapped in.
799
budget = create_token_budget()
825
- remaining_swapped, output = scheduler._schedule_swapped(
826
- swapped, budget, curr_loras, policy)
800
+ output = scheduler._schedule_swapped(budget, curr_loras)
801
+ remaining_swapped = scheduler.swapped
802
assert len(remaining_swapped) == 0
803
assert len(output.infeasible_seq_groups) == 2
804
assert budget.num_batched_tokens == 0
@@ -834,23 +809,21 @@ def test_infeasible_swap():
809
810
def test_schedule_swapped_blocks_to_copy():
811
scheduler = initialize_scheduler()
837
- swapped: Deque[SequenceGroup] = deque()
838
- policy = PolicyFactory.get_policy(policy_name="fcfs")
812
curr_loras = None
813
_, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
814
scheduler._allocate_and_set_running(seq_group)
815
append_new_token_seq_group(60, seq_group, 1)
816
blocks_to_swap_out: List[Tuple[int, int]] = []
817
scheduler._swap_out(seq_group, blocks_to_swap_out)
845
- swapped.append(seq_group)
818
+ scheduler._add_seq_group_to_swapped(seq_group)
819
820
# The last request should be swapped out.
821
scheduler.block_manager.append_slots = MagicMock()
822
scheduler.block_manager.append_slots.return_value = [(2, 3)]
823
824
budget = create_token_budget()
852
- remaining_swapped, output = scheduler._schedule_swapped(
853
- swapped, budget, curr_loras, policy)
825
+ output = scheduler._schedule_swapped(budget, curr_loras)
826
+ remaining_swapped = scheduler.swapped
827
assert len(remaining_swapped) == 0
828
assert len(output.decode_seq_groups) == 1
829
assert len(output.prefill_seq_groups) == 0
vllm/core/policy.py
deleted
-45
@@ -1,45 +0,0 @@
1
-from collections import deque
2
-from typing import Deque
3
-
4
-from vllm.sequence import SequenceGroup
5
-
6
-
7
-class Policy:
8
-
9
- def get_priority(
10
- self,
11
- now: float,
12
- seq_group: SequenceGroup,
13
- ) -> float:
14
- raise NotImplementedError
15
-
16
- def sort_by_priority(
17
- self,
18
- now: float,
19
- seq_groups: Deque[SequenceGroup],
20
- ) -> Deque[SequenceGroup]:
21
- return deque(
22
- sorted(
23
- seq_groups,
24
- key=lambda seq_group: self.get_priority(now, seq_group),
25
- reverse=True,
26
- ))
27
-
28
-
29
-class FCFS(Policy):
30
-
31
- def get_priority(
32
- self,
33
- now: float,
34
- seq_group: SequenceGroup,
35
- ) -> float:
36
- return now - seq_group.metrics.arrival_time
37
-
38
-
39
-class PolicyFactory:
40
-
41
- _POLICY_REGISTRY = {'fcfs': FCFS}
42
-
43
- @classmethod
44
- def get_policy(cls, policy_name: str, **kwargs) -> Policy:
45
- return cls._POLICY_REGISTRY[policy_name](**kwargs)
vllm/core/scheduler.py
+43
-73
@@ -8,7 +8,6 @@ from typing import Deque, Dict, Iterable, List, Optional, Set, Tuple, Union
8
9
from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig
10
from vllm.core.interfaces import AllocStatus, BlockSpaceManager
11
-from vllm.core.policy import Policy, PolicyFactory
11
from vllm.logger import init_logger
12
from vllm.lora.request import LoRARequest
13
from vllm.prompt_adapter.request import PromptAdapterRequest
@@ -345,6 +344,16 @@ class Scheduler:
344
# Add sequence groups to the waiting queue.
345
self.waiting.append(seq_group)
346
347
+ def _add_seq_group_to_running(self, seq_group: SequenceGroup) -> None:
348
+ # Add sequence groups to the running queue.
349
+ # Only for testing purposes.
350
+ self.running.append(seq_group)
351
+
352
+ def _add_seq_group_to_swapped(self, seq_group: SequenceGroup) -> None:
353
+ # Add sequence groups to the swapped queue.
354
+ # Only for testing purposes.
355
+ self.swapped.append(seq_group)
356
+
357
def abort_seq_group(self, request_id: Union[str, Iterable[str]]) -> None:
358
"""Aborts a sequence group with the given ID.
359
@@ -398,32 +407,26 @@ class Scheduler:
407
408
def _schedule_running(
409
self,
401
- running_queue: deque,
410
budget: SchedulingBudget,
411
curr_loras: Optional[Set[int]],
404
- policy: Policy,
412
enable_chunking: bool = False,
406
- ) -> Tuple[deque, SchedulerRunningOutputs]:
413
+ ) -> SchedulerRunningOutputs:
414
"""Schedule sequence groups that are running.
415
416
Running queue should include decode and chunked prefill requests.
417
418
Args:
412
- running_queue: The queue that contains running requests (i.e.,
413
- decodes). The given arguments are NOT in-place modified.
419
budget: The scheduling budget. The argument is in-place updated
420
when any decodes are preempted.
421
curr_loras: Currently batched lora request ids. The argument is
422
in-place updated when any decodes are preempted.
418
- policy: The sorting policy to sort running_queue.
423
enable_chunking: If True, seq group can be chunked and only a
424
chunked number of tokens are scheduled if
425
`budget.num_batched_tokens` has not enough capacity to schedule
426
all tokens.
427
428
Returns:
425
- A tuple of remaining running queue (should be always 0) after
426
- scheduling and SchedulerRunningOutputs.
429
+ SchedulerRunningOutputs.
430
"""
431
# Blocks that need to be swapped or copied before model execution.
432
blocks_to_swap_out: List[Tuple[int, int]] = []
@@ -436,10 +439,9 @@ class Scheduler:
439
440
# NOTE(woosuk): Preemption happens only when there is no available slot
441
# to keep all the sequence groups in the RUNNING state.
439
- # In this case, the policy is responsible for deciding which sequence
440
- # groups to preempt.
441
- now = time.time()
442
- running_queue = policy.sort_by_priority(now, running_queue)
442
+
443
+ running_queue = self.running
444
+
445
while running_queue:
446
seq_group = running_queue[0]
447
num_running_tokens = self._get_num_new_tokens(
@@ -503,7 +505,7 @@ class Scheduler:
505
if curr_loras is not None and seq_group.lora_int_id > 0:
506
curr_loras.add(seq_group.lora_int_id)
507
506
- return running_queue, SchedulerRunningOutputs(
508
+ return SchedulerRunningOutputs(
509
decode_seq_groups=decode_seq_groups,
510
prefill_seq_groups=prefill_seq_groups,
511
preempted=preempted,
@@ -515,12 +517,10 @@ class Scheduler:
517
518
def _schedule_swapped(
519
self,
518
- swapped_queue: deque,
520
budget: SchedulingBudget,
521
curr_loras: Optional[Set[int]],
521
- policy: Policy,
522
enable_chunking: bool = False,
523
- ) -> Tuple[deque, SchedulerSwappedInOutputs]:
523
+ ) -> SchedulerSwappedInOutputs:
524
"""Schedule sequence groups that are swapped out.
525
526
It schedules swapped requests as long as it fits `budget` and
@@ -528,20 +528,16 @@ class Scheduler:
528
`budget` and `curr_loras` are updated based on scheduled seq_groups.
529
530
Args:
531
- swapped_queue: The queue that contains swapped out requests.
532
- The given arguments are NOT in-place modified.
531
budget: The scheduling budget. The argument is in-place updated
532
when any requests are swapped in.
533
curr_loras: Currently batched lora request ids. The argument is
534
in-place updated when any requests are swapped in.
537
- policy: The sorting policy to sort swapped_queue.
535
enable_chunking: If True, seq group can be chunked and only a
536
chunked number of tokens are scheduled if
537
`budget.num_batched_tokens` has not enough capacity to schedule
538
all tokens.
539
540
Returns:
544
- A tuple of remaining swapped_queue after scheduling and
541
SchedulerSwappedInOutputs.
542
"""
543
# Blocks that need to be swapped or copied before model execution.
@@ -549,10 +545,10 @@ class Scheduler:
545
blocks_to_copy: List[Tuple[int, int]] = []
546
decode_seq_groups: List[ScheduledSequenceGroup] = []
547
prefill_seq_groups: List[ScheduledSequenceGroup] = []
552
- now = time.time()
553
- swapped_queue = policy.sort_by_priority(now, swapped_queue)
548
infeasible_seq_groups: List[SequenceGroup] = []
549
550
+ swapped_queue = self.swapped
551
+
552
leftover_swapped: Deque[SequenceGroup] = deque()
553
while swapped_queue:
554
seq_group = swapped_queue[0]
@@ -617,7 +613,7 @@ class Scheduler:
613
614
swapped_queue.extendleft(leftover_swapped)
615
620
- return swapped_queue, SchedulerSwappedInOutputs(
616
+ return SchedulerSwappedInOutputs(
617
decode_seq_groups=decode_seq_groups,
618
prefill_seq_groups=prefill_seq_groups,
619
blocks_to_swap_in=blocks_to_swap_in,
@@ -644,11 +640,10 @@ class Scheduler:
640
641
def _schedule_prefills(
642
self,
647
- waiting_queue: deque,
643
budget: SchedulingBudget,
644
curr_loras: Optional[Set[int]],
645
enable_chunking: bool = False,
651
- ) -> Tuple[deque, SchedulerPrefillOutputs]:
646
+ ) -> SchedulerPrefillOutputs:
647
"""Schedule sequence groups that are in prefill stage.
648
649
Note that the current scheduler treats PREEMPTED_FOR_RECOMPUTE
@@ -660,8 +655,6 @@ class Scheduler:
655
`budget` and `curr_loras` are updated based on scheduled seq_groups.
656
657
Args:
663
- waiting_queue: The queue that contains prefill requests.
664
- The given arguments are NOT in-place modified.
658
budget: The scheduling budget. The argument is in-place updated
659
when any requests are scheduled.
660
curr_loras: Currently batched lora request ids. The argument is
@@ -672,14 +665,12 @@ class Scheduler:
665
all tokens.
666
667
Returns:
675
- A tuple of remaining waiting_queue after scheduling and
668
SchedulerSwappedInOutputs.
669
"""
670
ignored_seq_groups: List[SequenceGroup] = []
671
seq_groups: List[SequenceGroup] = []
680
- # We don't sort waiting queue because we assume it is sorted.
681
- # Copy the queue so that the input queue is not modified.
682
- waiting_queue = deque([s for s in waiting_queue])
672
+
673
+ waiting_queue = self.waiting
674
675
leftover_waiting_sequences: Deque[SequenceGroup] = deque()
676
while self._passed_delay(time.time()) and waiting_queue:
@@ -758,7 +749,7 @@ class Scheduler:
749
if len(seq_groups) > 0:
750
self.prev_prompt = True
751
761
- return waiting_queue, SchedulerPrefillOutputs(
752
+ return SchedulerPrefillOutputs(
753
seq_groups=seq_groups,
754
ignored_seq_groups=ignored_seq_groups,
755
num_lookahead_slots=self._get_num_lookahead_slots(is_prefill=True))
@@ -785,53 +776,43 @@ class Scheduler:
776
seq_group.lora_int_id for seq_group in self.running
777
if seq_group.lora_int_id > 0) if self.lora_enabled else None
778
788
- remaining_waiting, prefills = (self.waiting,
789
- SchedulerPrefillOutputs.create_empty())
790
- remaining_running, running_scheduled = (
791
- self.running, SchedulerRunningOutputs.create_empty())
792
- remaining_swapped, swapped_in = (
793
- self.swapped, SchedulerSwappedInOutputs.create_empty())
779
+ prefills = SchedulerPrefillOutputs.create_empty()
780
+ running_scheduled = SchedulerRunningOutputs.create_empty()
781
+ swapped_in = SchedulerSwappedInOutputs.create_empty()
782
783
# If any requests are swapped, prioritized swapped requests.
784
if not self.swapped:
797
- remaining_waiting, prefills = self._schedule_prefills(
798
- self.waiting, budget, curr_loras, enable_chunking=False)
785
+ prefills = self._schedule_prefills(budget,
786
+ curr_loras,
787
+ enable_chunking=False)
788
800
- fcfs_policy = PolicyFactory.get_policy(policy_name="fcfs")
789
# Don't schedule decodes if prefills are scheduled.
790
# NOTE: If `_schedule_prefills` doesn't enable chunking, self.running
791
# only contains decode requests, not chunked prefills.
792
if len(prefills.seq_groups) == 0:
805
- remaining_running, running_scheduled = self._schedule_running(
806
- self.running,
807
- budget,
808
- curr_loras,
809
- fcfs_policy,
810
- enable_chunking=False)
793
+ running_scheduled = self._schedule_running(budget,
794
+ curr_loras,
795
+ enable_chunking=False)
796
797
# If any sequence group is preempted, do not swap in any sequence
798
# group. because it means there's no slot for new running requests.
799
if len(running_scheduled.preempted) + len(
800
running_scheduled.swapped_out) == 0:
816
- remaining_swapped, swapped_in = self._schedule_swapped(
817
- self.swapped, budget, curr_loras, fcfs_policy)
801
+ swapped_in = self._schedule_swapped(budget, curr_loras)
802
803
assert (budget.num_batched_tokens <=
804
self.scheduler_config.max_num_batched_tokens)
805
assert budget.num_curr_seqs <= self.scheduler_config.max_num_seqs
806
807
# Update waiting requests.
824
- self.waiting = remaining_waiting
808
self.waiting.extendleft(running_scheduled.preempted)
809
# Update new running requests.
827
- self.running = remaining_running
810
self.running.extend([s.seq_group for s in prefills.seq_groups])
811
self.running.extend(
812
[s.seq_group for s in running_scheduled.decode_seq_groups])
813
self.running.extend(
814
[s.seq_group for s in swapped_in.decode_seq_groups])
815
# Update swapped requests.
834
- self.swapped = remaining_swapped
816
self.swapped.extend(running_scheduled.swapped_out)
817
preempted = (len(running_scheduled.preempted) +
818
len(running_scheduled.swapped_out))
@@ -877,42 +858,32 @@ class Scheduler:
858
)
859
curr_loras: Set[int] = set()
860
880
- remaining_waiting, prefills = (self.waiting,
881
- SchedulerPrefillOutputs.create_empty())
882
- remaining_running, running_scheduled = (
883
- self.running, SchedulerRunningOutputs.create_empty())
884
- remaining_swapped, swapped_in = (
885
- self.swapped, SchedulerSwappedInOutputs.create_empty())
861
+ prefills = SchedulerPrefillOutputs.create_empty()
862
+ swapped_in = SchedulerSwappedInOutputs.create_empty()
863
864
# Decoding should be always scheduled first by fcfs.
888
- fcfs_policy = PolicyFactory.get_policy(policy_name="fcfs")
889
- remaining_running, running_scheduled = self._schedule_running(
890
- self.running,
891
- budget,
892
- curr_loras,
893
- fcfs_policy,
894
- enable_chunking=True)
865
+ running_scheduled = self._schedule_running(budget,
866
+ curr_loras,
867
+ enable_chunking=True)
868
869
# Schedule swapped out requests.
870
# If preemption happens, it means we don't have space for swap-in.
871
if len(running_scheduled.preempted) + len(
872
running_scheduled.swapped_out) == 0:
900
- remaining_swapped, swapped_in = self._schedule_swapped(
901
- self.swapped, budget, curr_loras, fcfs_policy)
873
+ swapped_in = self._schedule_swapped(budget, curr_loras)
874
875
# Schedule new prefills.
904
- remaining_waiting, prefills = self._schedule_prefills(
905
- self.waiting, budget, curr_loras, enable_chunking=True)
876
+ prefills = self._schedule_prefills(budget,
877
+ curr_loras,
878
+ enable_chunking=True)
879
880
assert (budget.num_batched_tokens <=
881
self.scheduler_config.max_num_batched_tokens)
882
assert budget.num_curr_seqs <= self.scheduler_config.max_num_seqs
883
884
# Update waiting requests.
912
- self.waiting = remaining_waiting
885
self.waiting.extendleft(running_scheduled.preempted)
886
# Update new running requests.
915
- self.running = remaining_running
887
self.running.extend([s.seq_group for s in prefills.seq_groups])
888
self.running.extend(
889
[s.seq_group for s in running_scheduled.decode_seq_groups])
@@ -923,7 +894,6 @@ class Scheduler:
894
self.running.extend(
895
[s.seq_group for s in swapped_in.prefill_seq_groups])
896
# Update swapped requests.
926
- self.swapped = remaining_swapped
897
self.swapped.extend(running_scheduled.swapped_out)
898
return SchedulerOutputs(
899
scheduled_seq_groups=(prefills.seq_groups +