A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[3/N] Refactor scheduler for chunked prefill scheduling (#3550)
SangBin Cho committed
Apr 4, 2024 at 06:13 UTC
3dcb3e8b9838cbbef83ce326b1a35b31a3cf14f2
5 files changed
+1020
-255
tests/core/test_scheduler.py
+532
-6
@@ -1,10 +1,15 @@
1
import time
2
+from collections import deque
3
from typing import List
4
+from unittest.mock import MagicMock
5
6
import pytest # noqa
7
6
-from vllm.config import CacheConfig, SchedulerConfig
7
-from vllm.core.scheduler import Scheduler
8
+from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig
9
+from vllm.core.interfaces import AllocStatus
10
+from vllm.core.policy import PolicyFactory
11
+from vllm.core.scheduler import Scheduler, SchedulingBudget
12
+from vllm.lora.request import LoRARequest
13
from vllm.sequence import Logprob, SequenceGroup
14
15
from .utils import create_dummy_prompt
@@ -177,7 +182,6 @@ def test_scheduler_max_seqs():
182
183
184
def test_scheduler_delay_factor():
180
-
185
block_size = 4
186
scheduler_config = SchedulerConfig(100, 64, 16, delay_factor=0.5)
187
cache_config = CacheConfig(block_size, 1.0, 1, "auto")
@@ -189,7 +193,7 @@ def test_scheduler_delay_factor():
193
_, seq_group = create_dummy_prompt("0", prompt_length=block_size)
194
scheduler.add_seq_group(seq_group)
195
seq_group_meta, out = scheduler.schedule()
192
- assert out.prompt_run
196
+ assert out.num_prefill_groups > 0
197
assert seq_group_meta[0].request_id == '0'
198
199
# wait for a second before scheduling next prompt
@@ -199,11 +203,533 @@ def test_scheduler_delay_factor():
203
204
# second prompt should *not* be scheduled
205
seq_group_meta, out = scheduler.schedule()
202
- assert not out.prompt_run
206
+ assert out.num_prefill_groups == 0
207
assert seq_group_meta[0].request_id == '0'
208
209
# wait for more than 0.5 second and try again
210
time.sleep(0.6)
211
seq_group_meta, out = scheduler.schedule()
208
- assert out.prompt_run
212
+ assert out.num_prefill_groups > 0
213
assert seq_group_meta[0].request_id == '1'
214
+
215
+
216
+def test_swapped_out_prioritized():
217
+ scheduler = initialize_scheduler(max_num_seqs=6)
218
+ # best_of=2 * 3 == 6 sequences.
219
+ for i in range(3):
220
+ _, seq_group = create_dummy_prompt(str(i), prompt_length=60, best_of=2)
221
+ scheduler.add_seq_group(seq_group)
222
+ _, out = scheduler.schedule()
223
+ # prefill scheduled now.
224
+ assert len(out.scheduled_seq_groups) == 3
225
+
226
+ # The last request should be swapped out.
227
+ scheduler.block_manager.can_append_slots = MagicMock()
228
+
229
+ def cannot_append_second_group(seq_group, num_lookahead_slots):
230
+ return seq_group.request_id != "2"
231
+
232
+ scheduler.block_manager.can_append_slots.side_effect = (
233
+ cannot_append_second_group)
234
+
235
+ _, out = scheduler.schedule()
236
+ assert len(out.scheduled_seq_groups) == 2
237
+ assert out.num_batched_tokens == 2
238
+ assert out.blocks_to_swap_out != {}
239
+ assert out.blocks_to_swap_in == {}
240
+
241
+ # Add 1 more task. Swap should be prioritized over prefill.
242
+ _, seq_group = create_dummy_prompt(str(i), prompt_length=60, best_of=2)
243
+ scheduler.add_seq_group(seq_group)
244
+ _, out = scheduler.schedule()
245
+ assert len(out.scheduled_seq_groups) == 3
246
+ # 3 decodes. It is swapped in.
247
+ assert out.num_batched_tokens == 3
248
+ assert out.blocks_to_swap_in != {}
249
+ assert out.blocks_to_swap_out == {}
250
+
251
+
252
+def initialize_scheduler(*,
253
+ max_num_seqs=1000,
254
+ max_token_budget=1000,
255
+ max_model_len=1000,
256
+ lora_config=None):
257
+ block_size = 4
258
+ scheduler_config = SchedulerConfig(max_token_budget, max_num_seqs,
259
+ max_model_len)
260
+ cache_config = CacheConfig(block_size, 1.0, 1, "auto")
261
+ cache_config.num_cpu_blocks = 8
262
+ cache_config.num_gpu_blocks = 8
263
+ scheduler = Scheduler(scheduler_config, cache_config, lora_config)
264
+ return scheduler
265
+
266
+
267
+def create_token_budget(num_batched_tokens: int = 0,
268
+ num_curr_seqs: int = 0,
269
+ token_budget: int = 10000,
270
+ max_num_seqs: int = 10000) -> SchedulingBudget:
271
+ return SchedulingBudget(
272
+ num_batched_tokens=num_batched_tokens,
273
+ num_curr_seqs=num_curr_seqs,
274
+ token_budget=token_budget,
275
+ max_num_seqs=max_num_seqs,
276
+ )
277
+
278
+
279
+def test_prefill_schedule_max_prompt_len():
280
+ """
281
+ Test prompt longer than max_prompt_len is aborted.
282
+ """
283
+ scheduler = initialize_scheduler(max_model_len=30)
284
+ _, seq_group = create_dummy_prompt(0, prompt_length=60)
285
+ waiting = deque([seq_group])
286
+ budget = create_token_budget()
287
+ remaining_waiting, output = scheduler._schedule_prefills(
288
+ waiting, budget, None)
289
+ assert len(output.ignored_seq_groups) == 1
290
+ assert len(output.seq_groups) == 0
291
+ assert budget.num_batched_tokens == 0
292
+ assert budget.num_curr_seqs == 0
293
+ assert len(remaining_waiting) == 0
294
+
295
+
296
+def test_prefill_schedule_token_budget():
297
+ """
298
+ Test token budget respected.
299
+ """
300
+ scheduler = initialize_scheduler()
301
+ waiting = deque()
302
+ budget = create_token_budget(token_budget=0)
303
+ for i in range(2):
304
+ _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
305
+ waiting.append(seq_group)
306
+
307
+ # 0 token budget == nothing is scheduled.
308
+ remaining_waiting, output = scheduler._schedule_prefills(
309
+ waiting, budget, None)
310
+ assert len(output.ignored_seq_groups) == 0
311
+ assert len(output.seq_groups) == 0
312
+ assert budget.num_batched_tokens == 0
313
+ assert budget.num_curr_seqs == 0
314
+ assert len(remaining_waiting) == 2
315
+
316
+ # 60 token budget == 1 request scheduled.
317
+ budget = create_token_budget(token_budget=60)
318
+ remaining_waiting, output = scheduler._schedule_prefills(
319
+ waiting, budget, None)
320
+ assert len(output.ignored_seq_groups) == 0
321
+ assert len(output.seq_groups) == 1
322
+ assert budget.num_batched_tokens == 60
323
+ assert budget.num_curr_seqs == 1
324
+ assert len(remaining_waiting) == 1
325
+
326
+ # Test when current_batched_tokens respected.
327
+ scheduler = initialize_scheduler()
328
+ waiting = deque()
329
+ budget = create_token_budget(num_batched_tokens=30, token_budget=60)
330
+ _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
331
+ # Cannot schedule a prompt that doesn't fit the budget.
332
+ waiting.append(seq_group)
333
+ remaining_waiting, output = scheduler._schedule_prefills(
334
+ waiting, budget, None)
335
+ assert len(output.ignored_seq_groups) == 0
336
+ assert len(output.seq_groups) == 0
337
+ assert budget.num_batched_tokens == 30
338
+ assert budget.num_curr_seqs == 0
339
+ assert len(remaining_waiting) == 1
340
+ budget = create_token_budget(num_batched_tokens=30, token_budget=90)
341
+ remaining_waiting, output = scheduler._schedule_prefills(
342
+ waiting, budget, None)
343
+ assert len(output.seq_groups) == 1
344
+ assert budget.num_batched_tokens == 90
345
+ assert budget.num_curr_seqs == 1
346
+ assert len(remaining_waiting) == 0
347
+
348
+
349
+def test_prefill_schedule_max_seqs():
350
+ """
351
+ Test max seq respected.
352
+ """
353
+ scheduler = initialize_scheduler()
354
+ waiting = deque()
355
+ budget = create_token_budget(max_num_seqs=2)
356
+ for i in range(3):
357
+ _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
358
+ waiting.append(seq_group)
359
+ remaining_waiting, output = scheduler._schedule_prefills(
360
+ waiting, budget, None)
361
+ assert len(output.ignored_seq_groups) == 0
362
+ assert len(output.seq_groups) == 2
363
+ assert budget.num_batched_tokens == 120
364
+ assert budget.num_curr_seqs == 2
365
+ assert len(remaining_waiting) == 1
366
+
367
+ # Verify curr_num_seqs respected.
368
+ waiting = deque()
369
+ budget = create_token_budget(num_curr_seqs=2, max_num_seqs=2)
370
+ _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
371
+ waiting.append(seq_group)
372
+ remaining_waiting, output = scheduler._schedule_prefills(
373
+ waiting, budget, None)
374
+ assert len(output.ignored_seq_groups) == 0
375
+ assert len(output.seq_groups) == 0
376
+ assert budget.num_batched_tokens == 0
377
+ assert budget.num_curr_seqs == 2
378
+ assert len(remaining_waiting) == 1
379
+
380
+
381
+def test_prefill_schedule_max_lora():
382
+ """
383
+ Test max lora is respected and prioritized.
384
+ """
385
+ lora_config = LoRAConfig(max_lora_rank=8, max_loras=1)
386
+ scheduler = initialize_scheduler(lora_config=lora_config)
387
+ waiting = deque()
388
+ budget = create_token_budget(token_budget=120)
389
+ curr_loras = set()
390
+ for i in range(2):
391
+ _, seq_group = create_dummy_prompt(str(i),
392
+ prompt_length=60,
393
+ lora_request=LoRARequest(
394
+ lora_name=str(i),
395
+ lora_int_id=i + 1,
396
+ lora_local_path="abc"))
397
+ waiting.append(seq_group)
398
+ # Add two more requests to verify lora is prioritized.
399
+ # 0: Lora, 1: Lora, 2: regular, 3: regular
400
+ # In the first iteration, index 0, 2 is scheduled.
401
+ # If a request is not scheduled because it hits max lora, it is
402
+ # prioritized. Verify that.
403
+ for i in range(2, 4):
404
+ _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
405
+ waiting.append(seq_group)
406
+ # Schedule 2 requests (0 and 2)
407
+ remaining_waiting, output = scheduler._schedule_prefills(
408
+ waiting, budget, curr_loras)
409
+ assert len(output.ignored_seq_groups) == 0
410
+ assert len(output.seq_groups) == 2
411
+ assert budget.num_batched_tokens == 120
412
+ assert budget.num_curr_seqs == 2
413
+ assert len(remaining_waiting) == 2
414
+ assert len(curr_loras) == 1
415
+ # The second lora request is scheduled next as FCFS policy.
416
+ # Reset curr_loras so that it can be scheduled.
417
+ curr_loras = set()
418
+ budget = create_token_budget(token_budget=60)
419
+ remaining_waiting, output = scheduler._schedule_prefills(
420
+ remaining_waiting, budget, curr_loras)
421
+ assert len(output.seq_groups) == 1
422
+ assert output.seq_groups[0].seq_group.request_id == "1"
423
+ assert len(remaining_waiting) == 1
424
+ assert len(curr_loras) == 1
425
+ assert budget.num_batched_tokens == 60
426
+
427
+
428
+def test_prefill_schedule_no_block_manager_capacity():
429
+ """
430
+ Test sequence cannot be scheduled due to block manager has no capacity.
431
+ """
432
+ scheduler = initialize_scheduler()
433
+ waiting = deque()
434
+ budget = create_token_budget()
435
+ for i in range(3):
436
+ _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
437
+ waiting.append(seq_group)
438
+ scheduler.block_manager.can_allocate = MagicMock()
439
+ scheduler.block_manager.can_allocate.return_value = AllocStatus.LATER
440
+ remainig_waiting, output = scheduler._schedule_prefills(
441
+ waiting, budget, None)
442
+ assert len(output.ignored_seq_groups) == 0
443
+ assert len(output.seq_groups) == 0
444
+ assert budget.num_batched_tokens == 0
445
+ assert budget.num_curr_seqs == 0
446
+ assert len(remainig_waiting) == 3
447
+
448
+ scheduler = initialize_scheduler()
449
+ waiting = deque()
450
+ budget = create_token_budget()
451
+ for i in range(3):
452
+ _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
453
+ waiting.append(seq_group)
454
+ scheduler.block_manager.can_allocate = MagicMock()
455
+ scheduler.block_manager.can_allocate.return_value = AllocStatus.NEVER
456
+ remaining_waiting, output = scheduler._schedule_prefills(
457
+ waiting, budget, None)
458
+ assert len(output.ignored_seq_groups) == 3
459
+ assert len(output.seq_groups) == 0
460
+ assert budget.num_batched_tokens == 0
461
+ assert budget.num_curr_seqs == 0
462
+ assert len(remaining_waiting) == 0
463
+
464
+
465
+def test_decode_schedule_preempted():
466
+ """
467
+ Test decodes cannot be scheduled and preempted.
468
+ """
469
+ scheduler = initialize_scheduler()
470
+ running = deque()
471
+ policy = PolicyFactory.get_policy(policy_name="fcfs")
472
+ curr_loras = None
473
+ for i in range(3):
474
+ _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
475
+ scheduler._allocate_and_set_running(seq_group)
476
+ running.append(seq_group)
477
+ scheduler.block_manager.can_append_slots = MagicMock()
478
+
479
+ def cannot_append_second_group(seq_group, num_lookahead_slots):
480
+ return seq_group.request_id != "1"
481
+
482
+ scheduler.block_manager.can_append_slots.side_effect = (
483
+ cannot_append_second_group)
484
+
485
+ # 1 cannot be scheduled, and the lowest priority (request 2)
486
+ # should be preempted. 1 will also be preempted.
487
+ budget = create_token_budget(num_batched_tokens=3, num_curr_seqs=3)
488
+ remainig_running, output = scheduler._schedule_decodes(
489
+ running, budget, curr_loras, policy)
490
+ assert len(remainig_running) == 0
491
+ assert len(output.seq_groups) == 1
492
+ assert output.seq_groups[0].seq_group.request_id == "0"
493
+ assert len(output.preempted) == 2
494
+ # Verify budgets are updated.
495
+ assert budget.num_batched_tokens == 1
496
+ assert budget.num_curr_seqs == 1
497
+ # Both should be preempted, not swapped.
498
+ assert output.blocks_to_swap_out == {}
499
+ # Nothing is copied.
500
+ assert output.blocks_to_copy == {}
501
+
502
+
503
+def test_decode_swap_beam_search():
504
+ """
505
+ Test best_of > 1 swap out blocks
506
+ """
507
+ scheduler = initialize_scheduler()
508
+ running = deque()
509
+ policy = PolicyFactory.get_policy(policy_name="fcfs")
510
+ curr_loras = None
511
+ for i in range(3):
512
+ _, seq_group = create_dummy_prompt(str(i), prompt_length=60, best_of=2)
513
+ scheduler._allocate_and_set_running(seq_group)
514
+ running.append(seq_group)
515
+
516
+ # The last request should be swapped out.
517
+ scheduler.block_manager.can_append_slots = MagicMock()
518
+
519
+ def cannot_append_second_group(seq_group, num_lookahead_slots):
520
+ return seq_group.request_id != "2"
521
+
522
+ scheduler.block_manager.can_append_slots.side_effect = (
523
+ cannot_append_second_group)
524
+ scheduler.block_manager.swap_out = MagicMock()
525
+ expected_swap_mapping = {"5": "7"}
526
+ scheduler.block_manager.swap_out.return_value = expected_swap_mapping
527
+
528
+ budget = create_token_budget(num_batched_tokens=3, num_curr_seqs=3)
529
+ remainig_running, output = scheduler._schedule_decodes(
530
+ running, budget, curr_loras, policy)
531
+ assert len(remainig_running) == 0
532
+ assert len(output.seq_groups) == 2
533
+ assert output.seq_groups[0].seq_group.request_id == "0"
534
+ assert output.seq_groups[1].seq_group.request_id == "1"
535
+ assert len(output.preempted) == 0
536
+ assert len(output.swapped_out) == 1
537
+ # Budget should refledct preempted requests.
538
+ assert budget.num_batched_tokens == 2
539
+ # since there are 2 sequences, 2 should be subtracted.
540
+ assert budget.num_curr_seqs == 1
541
+ # Both should be preempted, not swapped.
542
+ assert output.blocks_to_swap_out == expected_swap_mapping
543
+ # Nothing is copied.
544
+ assert output.blocks_to_copy == {}
545
+
546
+
547
+def test_schedule_decode_blocks_to_copy_update():
548
+ """
549
+ Verify blocks_to_copy is updated.
550
+ """
551
+ scheduler = initialize_scheduler()
552
+ _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
553
+ running = deque()
554
+ policy = PolicyFactory.get_policy(policy_name="fcfs")
555
+ curr_loras = None
556
+ scheduler._allocate_and_set_running(seq_group)
557
+ running.append(seq_group)
558
+
559
+ # The last request should be swapped out.
560
+ scheduler.block_manager.append_slots = MagicMock()
561
+ scheduler.block_manager.append_slots.return_value = {2: [3]}
562
+
563
+ budget = create_token_budget()
564
+ remaining_running, output = scheduler._schedule_decodes(
565
+ running, budget, curr_loras, policy)
566
+ assert len(remaining_running) == 0
567
+ assert len(output.seq_groups) == 1
568
+ assert len(output.preempted) == 0
569
+ assert len(output.swapped_out) == 0
570
+ # Nothing is preempted.
571
+ assert output.blocks_to_swap_out == {}
572
+ # Since append_slot returns the source -> dist mapping, it should
573
+ # applied.
574
+ assert output.blocks_to_copy == {2: [3]}
575
+
576
+
577
+def test_schedule_swapped_simple():
578
+ scheduler = initialize_scheduler()
579
+ swapped = deque()
580
+ policy = PolicyFactory.get_policy(policy_name="fcfs")
581
+ curr_loras = None
582
+ blocks_to_swap_out = {}
583
+ _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
584
+ scheduler._allocate_and_set_running(seq_group)
585
+ scheduler._swap_out(seq_group, blocks_to_swap_out)
586
+ swapped.append(seq_group)
587
+
588
+ budget = create_token_budget()
589
+ remaining_swapped, output = scheduler._schedule_swapped(
590
+ swapped, budget, curr_loras, policy)
591
+ assert len(remaining_swapped) == 0
592
+ assert budget.num_batched_tokens == 1
593
+ assert budget.num_curr_seqs == 2
594
+ assert len(output.seq_groups) == 1
595
+ # swap in is the reverse of swap out
596
+ blocks_to_swap_in_reverse = {}
597
+ for swapin, swapout in output.blocks_to_swap_in.items():
598
+ blocks_to_swap_in_reverse[swapout] = swapin
599
+ assert blocks_to_swap_out == blocks_to_swap_in_reverse
600
+
601
+
602
+def test_schedule_swapped_max_token_budget():
603
+ scheduler = initialize_scheduler()
604
+ swapped = deque()
605
+ policy = PolicyFactory.get_policy(policy_name="fcfs")
606
+ curr_loras = None
607
+ blocks_to_swap_out = {}
608
+ for _ in range(2):
609
+ _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
610
+ scheduler._allocate_and_set_running(seq_group)
611
+ scheduler._swap_out(seq_group, blocks_to_swap_out)
612
+ swapped.append(seq_group)
613
+
614
+ budget = create_token_budget(token_budget=1)
615
+ remaining_swapped, output = scheduler._schedule_swapped(
616
+ swapped, budget, curr_loras, policy)
617
+ assert len(remaining_swapped) == 1
618
+ assert budget.num_batched_tokens == 1
619
+ assert budget.num_curr_seqs == 2
620
+ assert len(output.seq_groups) == 1
621
+
622
+ # Verify num_batched_tokens are respected.
623
+ budget = create_token_budget(num_batched_tokens=1, token_budget=1)
624
+ remaining_swapped, output = scheduler._schedule_swapped(
625
+ remaining_swapped, budget, curr_loras, policy)
626
+ assert len(remaining_swapped) == 1
627
+ assert budget.num_batched_tokens == 1
628
+ assert budget.num_curr_seqs == 0
629
+ assert len(output.seq_groups) == 0
630
+
631
+
632
+def test_schedule_swapped_max_seqs():
633
+ scheduler = initialize_scheduler()
634
+ swapped = deque()
635
+ policy = PolicyFactory.get_policy(policy_name="fcfs")
636
+ curr_loras = None
637
+ blocks_to_swap_out = {}
638
+ for _ in range(2):
639
+ _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
640
+ scheduler._allocate_and_set_running(seq_group)
641
+ scheduler._swap_out(seq_group, blocks_to_swap_out)
642
+ swapped.append(seq_group)
643
+
644
+ budget = create_token_budget(max_num_seqs=2)
645
+ remaining_swapped, output = scheduler._schedule_swapped(
646
+ swapped, budget, curr_loras, policy)
647
+ assert len(remaining_swapped) == 1
648
+ assert budget.num_batched_tokens == 1
649
+ assert budget.num_curr_seqs == 2
650
+ assert len(output.seq_groups) == 1
651
+
652
+ # Verify num_curr_seqs are respected.
653
+ budget = create_token_budget(num_curr_seqs=2, max_num_seqs=2)
654
+ remaining_swapped, output = scheduler._schedule_swapped(
655
+ remaining_swapped, budget, curr_loras, policy)
656
+ assert len(remaining_swapped) == 1
657
+ assert budget.num_batched_tokens == 0
658
+ assert budget.num_curr_seqs == 2
659
+ assert len(output.seq_groups) == 0
660
+
661
+
662
+def test_schedule_swapped_max_loras():
663
+ lora_config = LoRAConfig(max_lora_rank=8, max_loras=1)
664
+ scheduler = initialize_scheduler(lora_config=lora_config)
665
+ swapped = deque()
666
+ policy = PolicyFactory.get_policy(policy_name="fcfs")
667
+ curr_loras = set()
668
+ blocks_to_swap_out = {}
669
+ for i in range(2):
670
+ _, seq_group = create_dummy_prompt(str(i),
671
+ prompt_length=60,
672
+ lora_request=LoRARequest(
673
+ lora_name=str(i),
674
+ lora_int_id=i + 1,
675
+ lora_local_path="abc"))
676
+ scheduler._allocate_and_set_running(seq_group)
677
+ scheduler._swap_out(seq_group, blocks_to_swap_out)
678
+ swapped.append(seq_group)
679
+
680
+ budget = create_token_budget()
681
+ remaining_swapped, output = scheduler._schedule_swapped(
682
+ swapped, budget, curr_loras, policy)
683
+ assert len(remaining_swapped) == 1
684
+ assert budget.num_batched_tokens == 1
685
+ assert budget.num_curr_seqs == 1
686
+ assert len(output.seq_groups) == 1
687
+ assert len(curr_loras) == 1
688
+
689
+
690
+def test_schedule_swapped_cannot_swap_in():
691
+ scheduler = initialize_scheduler()
692
+ swapped = deque()
693
+ policy = PolicyFactory.get_policy(policy_name="fcfs")
694
+ curr_loras = None
695
+ blocks_to_swap_out = {}
696
+ for _ in range(2):
697
+ _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
698
+ scheduler._allocate_and_set_running(seq_group)
699
+ scheduler._swap_out(seq_group, blocks_to_swap_out)
700
+ swapped.append(seq_group)
701
+
702
+ # The last request should be swapped out.
703
+ scheduler.block_manager.can_swap_in = MagicMock()
704
+ scheduler.block_manager.can_swap_in.return_value = False
705
+ # Since we cannot swap in, none of the requests are swapped in.
706
+ budget = create_token_budget()
707
+ remaining_swapped, output = scheduler._schedule_swapped(
708
+ swapped, budget, curr_loras, policy)
709
+ assert len(remaining_swapped) == 2
710
+ assert budget.num_batched_tokens == 0
711
+ assert budget.num_curr_seqs == 0
712
+ assert len(output.seq_groups) == 0
713
+
714
+
715
+def test_schedule_swapped_blocks_to_copy():
716
+ scheduler = initialize_scheduler()
717
+ swapped = deque()
718
+ policy = PolicyFactory.get_policy(policy_name="fcfs")
719
+ curr_loras = None
720
+ _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
721
+ scheduler._allocate_and_set_running(seq_group)
722
+ blocks_to_swap_out = {}
723
+ scheduler._swap_out(seq_group, blocks_to_swap_out)
724
+ swapped.append(seq_group)
725
+
726
+ # The last request should be swapped out.
727
+ scheduler.block_manager.append_slots = MagicMock()
728
+ scheduler.block_manager.append_slots.return_value = {2: [3]}
729
+
730
+ budget = create_token_budget()
731
+ remaining_swapped, output = scheduler._schedule_swapped(
732
+ swapped, budget, curr_loras, policy)
733
+ assert len(remaining_swapped) == 0
734
+ assert len(output.seq_groups) == 1
735
+ assert output.blocks_to_copy == {2: [3]}
tests/core/utils.py
+13
-6
@@ -1,14 +1,19 @@
1
import time
2
-from typing import Tuple
2
+from typing import Optional, Tuple
3
4
from vllm import SamplingParams
5
+from vllm.lora.request import LoRARequest
6
from vllm.sequence import Logprob, Sequence, SequenceGroup
7
8
9
def create_dummy_prompt(
9
- request_id: str,
10
- prompt_length: int,
11
- block_size: int = None) -> Tuple[Sequence, SequenceGroup]:
10
+ request_id: str,
11
+ prompt_length: int,
12
+ block_size: Optional[int] = None,
13
+ lora_request: Optional[LoRARequest] = None,
14
+ use_beam_search: bool = False,
15
+ best_of: int = 1,
16
+) -> Tuple[Sequence, SequenceGroup]:
17
if not block_size:
18
block_size = prompt_length
19
@@ -17,8 +22,10 @@ def create_dummy_prompt(
22
prompt_tokens = list(range(prompt_length))
23
prompt_str = " ".join([str(t) for t in prompt_tokens])
24
prompt = Sequence(int(request_id), prompt_str, prompt_tokens, block_size)
20
- seq_group = SequenceGroup(request_id, [prompt], SamplingParams(),
21
- time.time(), None)
25
+ seq_group = SequenceGroup(
26
+ request_id, [prompt],
27
+ SamplingParams(use_beam_search=use_beam_search, best_of=best_of),
28
+ time.time(), lora_request)
29
30
return prompt, seq_group
31
vllm/core/scheduler.py
+456
-241
@@ -6,11 +6,12 @@ from typing import Deque, Dict, Iterable, List, Optional, Set, Tuple, Union
6
7
from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig
8
from vllm.core.interfaces import AllocStatus, BlockSpaceManager
9
-from vllm.core.policy import PolicyFactory
9
+from vllm.core.policy import Policy, PolicyFactory
10
from vllm.logger import init_logger
11
from vllm.lora.request import LoRARequest
12
from vllm.sequence import (Sequence, SequenceData, SequenceGroup,
13
SequenceGroupMetadata, SequenceStatus)
14
+from vllm.utils import merge_dicts
15
16
logger = init_logger(__name__)
17
@@ -28,9 +29,19 @@ class PreemptionMode(enum.Enum):
29
RECOMPUTE = enum.auto()
30
31
31
-# seq_group: SequenceGroup to schedule.
32
-# token_chunk_size: The number of prefill tokens to be processed in the next
33
-# step.
32
+@dataclass
33
+class SchedulingBudget:
34
+ """The available slots for scheduling."""
35
+ num_batched_tokens: int
36
+ num_curr_seqs: int
37
+ token_budget: int
38
+ max_num_seqs: int
39
+
40
+ def can_schedule(self, *, num_new_tokens: int, num_new_seqs: int):
41
+ return (self.num_batched_tokens + num_new_tokens <= self.token_budget
42
+ and self.num_curr_seqs + num_new_seqs <= self.max_num_seqs)
43
+
44
+
45
@dataclass
46
class ScheduledSequenceGroup:
47
# A sequence group that's scheduled.
@@ -41,53 +52,28 @@ class ScheduledSequenceGroup:
52
token_chunk_size: int
53
54
55
+@dataclass
56
class SchedulerOutputs:
45
-
46
- def __init__(
47
- self,
48
- scheduled_seq_groups: Iterable[ScheduledSequenceGroup],
49
- prompt_run: bool,
50
- num_batched_tokens: int,
51
- blocks_to_swap_in: Dict[int, int],
52
- blocks_to_swap_out: Dict[int, int],
53
- blocks_to_copy: Dict[int, List[int]],
54
- ignored_seq_groups: List[SequenceGroup],
55
- num_lookahead_slots: int,
56
- ) -> None:
57
- """A list of sequence groups to be scheduled as a single batch.
58
-
59
- Args:
60
- scheduled_seq_groups: A tuple of scheduled sequence group and its
61
- token chunk size.
62
- prompt_run: True if all sequence groups are in prefill phase.
63
- If False, all sequence groups are in decoding phase.
64
- num_batched_tokens: Total number of batched tokens.
65
- blocks_to_swap_in: Blocks to swap in. Dict of CPU -> GPU block
66
- number.
67
- blocks_to_swap_out: Blocks to swap out. Dict of GPU -> CPU block
68
- number.
69
- blocks_to_copy: Blocks to copy. Source to a list of dest blocks.
70
- ignored_seq_groups: Sequence groups that are going to be ignored.
71
- """
72
- # A tuple of scheduled sequence group and its chunk size.
73
- self.scheduled_seq_groups: ScheduledSequenceGroup = scheduled_seq_groups
74
- # True if all sequence groups are in prefill phase. If False, all
75
- # sequence groups are in decoding phase.
76
- self.prompt_run: bool = prompt_run
77
- # Total number of batched tokens.
78
- self.num_batched_tokens: int = num_batched_tokens
79
- # Blocks to swap in. Dict of CPU -> GPU block number.
80
- self.blocks_to_swap_in: Dict[int, int] = blocks_to_swap_in
81
- # Blocks to swap out. Dict of GPU -> CPU block number.
82
- self.blocks_to_swap_out: Dict[int, int] = blocks_to_swap_out
83
- # Blocks to copy. Source to a list of dest blocks.
84
- self.blocks_to_copy: Dict[int, List[int]] = blocks_to_copy
85
- # Sequence groups that are going to be ignored.
86
- self.ignored_seq_groups: List[SequenceGroup] = ignored_seq_groups
87
-
57
+ # Scheduled sequence groups.
58
+ scheduled_seq_groups: Iterable[ScheduledSequenceGroup]
59
+ # Number of prefill groups scheduled.
60
+ num_prefill_groups: int
61
+ # Total number of batched tokens.
62
+ num_batched_tokens: int
63
+ # Blocks to swap in. Dict of CPU -> GPU block number.
64
+ blocks_to_swap_in: Dict[int, int]
65
+ # Blocks to swap out. Dict of GPU -> CPU block number.
66
+ blocks_to_swap_out: Dict[int, int]
67
+ # Blocks to copy. Source to a list of dest blocks.
68
+ blocks_to_copy: Dict[int, List[int]]
69
+ # Sequence groups that are going to be ignored.
70
+ ignored_seq_groups: List[SequenceGroup]
71
+ # The number of slots for lookahead decoding.
72
+ num_lookahead_slots: int
73
+
74
+ def __post_init__(self):
75
# Swap in and swap out should never happen at the same time.
89
- assert not (blocks_to_swap_in and blocks_to_swap_out)
90
- self.num_lookahead_slots = num_lookahead_slots
76
+ assert not (self.blocks_to_swap_in and self.blocks_to_swap_out)
77
78
self.num_loras: int = len(self.lora_requests)
79
if self.num_loras > 0:
@@ -108,6 +94,73 @@ class SchedulerOutputs:
94
return {g.seq_group.lora_request for g in self.scheduled_seq_groups}
95
96
97
+@dataclass
98
+class SchedulerDecodeOutputs:
99
+ """Outputs of the decoding phase of the scheduler."""
100
+ # Selected sequence groups for decoding.
101
+ seq_groups: List[SequenceGroup]
102
+ # The preempted sequences.
103
+ preempted: List[SequenceGroup]
104
+ # Sequences that are swapped out.
105
+ swapped_out: List[SequenceGroup]
106
+ # The blocks to swap out.
107
+ blocks_to_swap_out: Dict[int, int]
108
+ # The blocks to copy.
109
+ blocks_to_copy: Dict[int, List[int]]
110
+ num_lookahead_slots: int
111
+
112
+ @classmethod
113
+ def create_empty(cls) -> "SchedulerDecodeOutputs":
114
+ return SchedulerDecodeOutputs(
115
+ seq_groups=[],
116
+ preempted=[],
117
+ swapped_out=[],
118
+ blocks_to_swap_out={},
119
+ blocks_to_copy={},
120
+ num_lookahead_slots=0,
121
+ )
122
+
123
+
124
+@dataclass
125
+class SchedulerSwappedInOutputs:
126
+ """Outputs of the decoding phase of the scheduler."""
127
+ # Selected sequence groups for decoding.
128
+ seq_groups: List[SequenceGroup]
129
+ # The blocks to swap in.
130
+ blocks_to_swap_in: Dict[int, int]
131
+ # The blocks to copy.
132
+ blocks_to_copy: Dict[int, List[int]]
133
+ # # The number of batched tokens.
134
+ num_lookahead_slots: int
135
+
136
+ @classmethod
137
+ def create_empty(cls) -> "SchedulerSwappedInOutputs":
138
+ return SchedulerSwappedInOutputs(
139
+ seq_groups=[],
140
+ blocks_to_swap_in={},
141
+ blocks_to_copy={},
142
+ num_lookahead_slots=0,
143
+ )
144
+
145
+
146
+@dataclass
147
+class SchedulerPrefillOutputs:
148
+ """Outputs of the prefill phase of the scheduler."""
149
+ # Selected sequence groups for prefill.
150
+ seq_groups: List[SequenceGroup]
151
+ # Ignored sequence groups.
152
+ ignored_seq_groups: List[SequenceGroup]
153
+ num_lookahead_slots: int
154
+
155
+ @classmethod
156
+ def create_empty(cls) -> "SchedulerPrefillOutputs":
157
+ return SchedulerPrefillOutputs(
158
+ seq_groups=[],
159
+ ignored_seq_groups=[],
160
+ num_lookahead_slots=0,
161
+ )
162
+
163
+
164
class Scheduler:
165
166
def __init__(
@@ -123,6 +176,7 @@ class Scheduler:
176
# LoRAs. This should be improved in the future.
177
self.lora_config = lora_config
178
179
+ # TODO(sang): Fix it after chunked prefill is enabled.
180
self.prompt_limit = min(self.scheduler_config.max_model_len,
181
self.scheduler_config.max_num_batched_tokens)
182
@@ -142,10 +196,13 @@ class Scheduler:
196
enable_caching=self.cache_config.enable_prefix_caching)
197
198
# Sequence groups in the WAITING state.
199
+ # Contain new prefill or preempted requests.
200
self.waiting: Deque[SequenceGroup] = deque()
201
# Sequence groups in the RUNNING state.
202
+ # Contain decode requests.
203
self.running: Deque[SequenceGroup] = deque()
204
# Sequence groups in the SWAPPED state.
205
+ # Contain decode requests that are swapped out.
206
self.swapped: Deque[SequenceGroup] = deque()
207
208
# Time at previous scheduling step
@@ -159,8 +216,14 @@ class Scheduler:
216
def lora_enabled(self) -> bool:
217
return bool(self.lora_config)
218
219
+ @property
220
+ def num_decoding_tokens_per_seq(self) -> int:
221
+ """The number of new tokens."""
222
+ return 1
223
+
224
def add_seq_group(self, seq_group: SequenceGroup) -> None:
225
# Add sequence groups to the waiting queue.
226
+ logger.debug(f"add_seq_group {seq_group.request_id}")
227
self.waiting.append(seq_group)
228
229
def abort_seq_group(self, request_id: Union[str, Iterable[str]]) -> None:
@@ -205,214 +268,365 @@ class Scheduler:
268
def get_num_unfinished_seq_groups(self) -> int:
269
return len(self.waiting) + len(self.running) + len(self.swapped)
270
208
- def _schedule(self) -> SchedulerOutputs:
209
- # Blocks that need to be swapped or copied before model execution.
210
- blocks_to_swap_in: Dict[int, int] = {}
211
- blocks_to_swap_out: Dict[int, int] = {}
212
- blocks_to_copy: Dict[int, List[int]] = {}
213
-
214
- # Fix the current time.
215
- now = time.time()
216
-
217
- # Join waiting sequences if possible.
218
- if not self.swapped:
219
- ignored_seq_groups: List[SequenceGroup] = []
220
- scheduled: List[SequenceGroup] = []
221
- # The total number of sequences on the fly, including the
222
- # requests in the generation phase.
223
- num_curr_seqs = sum(seq_group.get_max_num_running_seqs()
224
- for seq_group in self.running)
225
- curr_loras = set(
226
- seq_group.lora_int_id
227
- for seq_group in self.running) if self.lora_enabled else None
228
-
229
- # Optimization: We do not sort the waiting queue since the preempted
230
- # sequence groups are added to the front and the new sequence groups
231
- # are added to the back.
232
- leftover_waiting_sequences = deque()
233
- num_batched_tokens = 0
234
- while self._passed_delay(now) and self.waiting:
235
- seq_group = self.waiting[0]
236
- waiting_seqs = seq_group.get_seqs(
237
- status=SequenceStatus.WAITING)
238
- assert len(waiting_seqs) == 1, (
239
- "Waiting sequence group should have only one prompt "
240
- "sequence.")
241
- # get_len includes output tokens if the request has been
242
- # preempted.
243
- num_prefill_tokens = waiting_seqs[0].get_len()
244
- if num_prefill_tokens > self.prompt_limit:
245
- logger.warning(
246
- f"Input prompt ({num_prefill_tokens} tokens) is too "
247
- f"long and exceeds limit of {self.prompt_limit}")
248
- for seq in waiting_seqs:
249
- seq.status = SequenceStatus.FINISHED_IGNORED
250
- ignored_seq_groups.append(seq_group)
251
- self.waiting.popleft()
252
- continue
253
-
254
- # If the sequence group cannot be allocated, stop.
255
- can_allocate = self.block_manager.can_allocate(seq_group)
256
- if can_allocate == AllocStatus.LATER:
257
- break
258
- elif can_allocate == AllocStatus.NEVER:
259
- logger.warning(
260
- f"Input prompt ({num_prefill_tokens} tokens) is too "
261
- f"long and exceeds the capacity of block_manager")
262
- for seq in waiting_seqs:
263
- seq.status = SequenceStatus.FINISHED_IGNORED
264
- ignored_seq_groups.append(seq_group)
265
- self.waiting.popleft()
266
- continue
271
+ def _schedule_decodes(
272
+ self,
273
+ running_queue: deque,
274
+ budget: SchedulingBudget,
275
+ curr_loras: Optional[Set[int]],
276
+ policy: Policy,
277
+ ) -> Tuple[deque, SchedulerDecodeOutputs]:
278
+ """Schedule sequence groups in a decoding stage.
279
268
- lora_int_id = 0
269
- if self.lora_enabled:
270
- lora_int_id = seq_group.lora_int_id
271
- if (lora_int_id > 0 and lora_int_id not in curr_loras
272
- and len(curr_loras) >= self.lora_config.max_loras):
273
- # We don't have a space for another LoRA, so
274
- # we ignore this request for now.
275
- leftover_waiting_sequences.appendleft(seq_group)
276
- self.waiting.popleft()
277
- continue
280
+ NOTE(sang): All the RUNNING num_batched_tokens, num_curr_seqs,
281
+ and curr_loras should be already included in `budget` and `curr_loras`.
282
+ The API doesn't ADD UP these values.
283
279
- # If the number of batched tokens exceeds the limit, stop.
280
- num_batched_tokens += num_prefill_tokens
281
- if (num_batched_tokens >
282
- self.scheduler_config.max_num_batched_tokens):
283
- break
284
+ Note that `budget` and `curr_loras` are still subtracted/popped when
285
+ any running requests are preempted from this API.
286
285
- # The total number of sequences in the RUNNING state should not
286
- # exceed the maximum number of sequences.
287
- num_new_seqs = seq_group.get_max_num_running_seqs()
288
- if (num_curr_seqs + num_new_seqs >
289
- self.scheduler_config.max_num_seqs):
290
- break
287
+ Args:
288
+ running_queue: The queue that contains running requests (i.e.,
289
+ decodes). The given arguments are NOT in-place modified.
290
+ budget: The scheduling budget. The argument is in-place updated
291
+ when any decodes are preempted.
292
+ curr_loras: Currently batched lora request ids. The argument is
293
+ in-place updated when any decodes are preempted.
294
+ policy: The sorting policy to sort running_queue.
295
+
296
+ Returns:
297
+ A tuple of remaining running queue (should be always 0) after
298
+ scheduling and SchedulerDecodeOutputs.
299
+ """
300
+ # Blocks that need to be swapped or copied before model execution.
301
+ blocks_to_swap_out: Dict[int, int] = {}
302
+ blocks_to_copy: Dict[int, List[int]] = {}
303
292
- if lora_int_id > 0:
293
- curr_loras.add(lora_int_id)
294
- self.waiting.popleft()
295
- self._allocate(seq_group)
296
- self.running.append(seq_group)
297
- num_curr_seqs += num_new_seqs
298
- scheduled.append(
299
- ScheduledSequenceGroup(
300
- seq_group=seq_group,
301
- token_chunk_size=num_prefill_tokens))
302
- self.waiting.extendleft(leftover_waiting_sequences)
303
-
304
- if scheduled or ignored_seq_groups:
305
- self.prev_prompt = True
306
- scheduler_outputs = SchedulerOutputs(
307
- scheduled_seq_groups=scheduled,
308
- prompt_run=True,
309
- num_batched_tokens=num_batched_tokens,
310
- blocks_to_swap_in=blocks_to_swap_in,
311
- blocks_to_swap_out=blocks_to_swap_out,
312
- blocks_to_copy=blocks_to_copy,
313
- ignored_seq_groups=ignored_seq_groups,
314
- num_lookahead_slots=self._get_num_lookahead_slots(
315
- is_prefill=True),
316
- )
317
- return scheduler_outputs
304
+ seq_groups: List[ScheduledSequenceGroup] = []
305
+ preempted: List[SequenceGroup] = []
306
+ swapped_out: List[SequenceGroup] = []
307
308
# NOTE(woosuk): Preemption happens only when there is no available slot
309
# to keep all the sequence groups in the RUNNING state.
310
# In this case, the policy is responsible for deciding which sequence
311
# groups to preempt.
323
- self.running = self.policy.sort_by_priority(now, self.running)
312
+ now = time.time()
313
+ running_queue = policy.sort_by_priority(now, running_queue)
314
325
- # Reserve new token slots for the running sequence groups.
326
- running: Deque[SequenceGroup] = deque()
327
- preempted: List[SequenceGroup] = []
328
- while self.running:
329
- seq_group = self.running.popleft()
315
+ while running_queue:
316
+ # NOTE: running
317
+ seq_group = running_queue[0]
318
+ num_running_tokens = (
319
+ seq_group.num_seqs(status=SequenceStatus.RUNNING) *
320
+ self.num_decoding_tokens_per_seq)
321
+ num_running_seqs = seq_group.get_max_num_running_seqs()
322
+
323
+ running_queue.popleft()
324
while not self._can_append_slots(seq_group):
331
- if self.running:
325
+ # Increase the budget as requests are preempted.
326
+ budget.num_batched_tokens -= num_running_tokens
327
+ budget.num_curr_seqs -= num_running_seqs
328
+ if curr_loras is not None and seq_group.lora_int_id > 0:
329
+ curr_loras.pop(seq_group.lora_int_id)
330
+
331
+ if running_queue:
332
# Preempt the lowest-priority sequence groups.
333
- victim_seq_group = self.running.pop()
334
- self._preempt(victim_seq_group, blocks_to_swap_out)
335
- preempted.append(victim_seq_group)
333
+ victim_seq_group = running_queue.pop()
334
+ preempted_mode = self._preempt(victim_seq_group,
335
+ blocks_to_swap_out)
336
+ if preempted_mode == PreemptionMode.RECOMPUTE:
337
+ preempted.append(victim_seq_group)
338
+ else:
339
+ swapped_out.append(victim_seq_group)
340
else:
341
# No other sequence groups can be preempted.
342
# Preempt the current sequence group.
339
- self._preempt(seq_group, blocks_to_swap_out)
340
- preempted.append(seq_group)
343
+ preempted_mode = self._preempt(seq_group,
344
+ blocks_to_swap_out)
345
+ if preempted_mode == PreemptionMode.RECOMPUTE:
346
+ preempted.append(seq_group)
347
+ else:
348
+ swapped_out.append(seq_group)
349
break
350
else:
343
- # Append new slots to the sequence group.
351
+ logger.debug(f"append slot for {seq_group}")
352
self._append_slots(seq_group, blocks_to_copy)
345
- running.append(seq_group)
346
- self.running = running
347
-
348
- # Swap in the sequence groups in the SWAPPED state if possible.
349
- self.swapped = self.policy.sort_by_priority(now, self.swapped)
350
- if not preempted:
351
- num_curr_seqs = sum(seq_group.get_max_num_running_seqs()
352
- for seq_group in self.running)
353
- curr_loras = set(
354
- seq_group.lora_int_id
355
- for seq_group in self.running) if self.lora_enabled else None
356
-
357
- leftover_swapped = deque()
358
-
359
- while self.swapped:
360
- seq_group = self.swapped[0]
361
- lora_int_id = 0
362
- if self.lora_enabled:
363
- lora_int_id = seq_group.lora_int_id
364
- if (lora_int_id > 0 and lora_int_id not in curr_loras
365
- and len(curr_loras) >= self.lora_config.max_loras):
366
- # We don't have a space for another LoRA, so
367
- # we ignore this request for now.
368
- leftover_swapped.appendleft(seq_group)
369
- self.swapped.popleft()
370
- continue
353
+ seq_groups.append(
354
+ ScheduledSequenceGroup(seq_group=seq_group,
355
+ token_chunk_size=1))
356
+ # Make sure all queues are updated.
357
+ assert len(running_queue) == 0
358
+
359
+ return running_queue, SchedulerDecodeOutputs(
360
+ seq_groups=seq_groups,
361
+ preempted=preempted,
362
+ swapped_out=swapped_out,
363
+ blocks_to_swap_out=blocks_to_swap_out,
364
+ blocks_to_copy=blocks_to_copy,
365
+ num_lookahead_slots=self._get_num_lookahead_slots(
366
+ is_prefill=False))
367
372
- # If the sequence group cannot be swapped in, stop.
373
- if not self._can_swap_in(seq_group):
374
- break
368
+ def _schedule_swapped(
369
+ self,
370
+ swapped_queue: deque,
371
+ budget: SchedulingBudget,
372
+ curr_loras: Optional[Set[int]],
373
+ policy: Policy,
374
+ ) -> Tuple[deque, SchedulerSwappedInOutputs]:
375
+ """Schedule sequence groups that are swapped out.
376
376
- # The total number of sequences in the RUNNING state should not
377
- # exceed the maximum number of sequences.
378
- num_new_seqs = seq_group.get_max_num_running_seqs()
379
- if (num_curr_seqs + num_new_seqs >
380
- self.scheduler_config.max_num_seqs):
381
- break
377
+ It schedules swapped requests as long as it fits `budget` and
378
+ curr_loras <= max_lora from the scheduling config. The input arguments
379
+ `budget` and `curr_loras` are updated based on scheduled seq_groups.
380
383
- if lora_int_id > 0:
384
- curr_loras.add(lora_int_id)
385
- self.swapped.popleft()
386
- self._swap_in(seq_group, blocks_to_swap_in)
387
- self._append_slots(seq_group, blocks_to_copy)
388
- num_curr_seqs += num_new_seqs
389
- self.running.append(seq_group)
390
-
391
- self.swapped.extendleft(leftover_swapped)
392
-
393
- # Each sequence in the generation phase only takes one token slot.
394
- # Therefore, the number of batched tokens is equal to the number of
395
- # sequences in the RUNNING state.
396
- num_batched_tokens = sum(
397
- seq_group.num_seqs(status=SequenceStatus.RUNNING)
398
- for seq_group in self.running)
399
-
400
- scheduler_outputs = SchedulerOutputs(
401
- scheduled_seq_groups=[
402
- ScheduledSequenceGroup(seq_group=running_group,
403
- token_chunk_size=1)
404
- for running_group in self.running
405
- ],
406
- prompt_run=False,
407
- num_batched_tokens=num_batched_tokens,
381
+ Args:
382
+ swapped_queue: The queue that contains swapped out requests.
383
+ The given arguments are NOT in-place modified.
384
+ budget: The scheduling budget. The argument is in-place updated
385
+ when any requests are swapped in.
386
+ curr_loras: Currently batched lora request ids. The argument is
387
+ in-place updated when any requests are swapped in.
388
+ policy: The sorting policy to sort swapped_queue.
389
+
390
+ Returns:
391
+ A tuple of remaining swapped_queue after scheduling and
392
+ SchedulerSwappedInOutputs.
393
+ """
394
+ # Blocks that need to be swapped or copied before model execution.
395
+ blocks_to_swap_in: Dict[int, int] = {}
396
+ blocks_to_copy: Dict[int, List[int]] = {}
397
+ seq_groups: List[ScheduledSequenceGroup] = []
398
+ now = time.time()
399
+ swapped_queue = policy.sort_by_priority(now, swapped_queue)
400
+
401
+ leftover_swapped = deque()
402
+ while swapped_queue:
403
+ seq_group = swapped_queue[0]
404
+
405
+ # If the sequence group cannot be swapped in, stop.
406
+ if not self.block_manager.can_swap_in(seq_group):
407
+ break
408
+
409
+ lora_int_id = 0
410
+ if self.lora_enabled:
411
+ lora_int_id = seq_group.lora_int_id
412
+ if (lora_int_id > 0 and lora_int_id not in curr_loras
413
+ and len(curr_loras) >= self.lora_config.max_loras):
414
+ # We don't have a space for another LoRA, so
415
+ # we ignore this request for now.
416
+ leftover_swapped.appendleft(seq_group)
417
+ swapped_queue.popleft()
418
+ continue
419
+
420
+ # The total number of sequences in the RUNNING state should not
421
+ # exceed the maximum number of sequences.
422
+ num_new_seqs = seq_group.get_max_num_running_seqs()
423
+ num_new_tokens = (
424
+ seq_group.num_seqs(status=SequenceStatus.SWAPPED) *
425
+ self.num_decoding_tokens_per_seq)
426
+
427
+ if not budget.can_schedule(num_new_tokens=num_new_tokens,
428
+ num_new_seqs=num_new_seqs):
429
+ break
430
+
431
+ if lora_int_id > 0 and curr_loras is not None:
432
+ curr_loras.add(lora_int_id)
433
+ swapped_queue.popleft()
434
+ self._swap_in(seq_group, blocks_to_swap_in)
435
+ self._append_slots(seq_group, blocks_to_copy)
436
+ seq_groups.append(
437
+ ScheduledSequenceGroup(seq_group, token_chunk_size=1))
438
+ budget.num_batched_tokens += num_new_tokens
439
+ budget.num_curr_seqs += num_new_seqs
440
+
441
+ swapped_queue.extendleft(leftover_swapped)
442
+
443
+ return swapped_queue, SchedulerSwappedInOutputs(
444
+ seq_groups=seq_groups,
445
blocks_to_swap_in=blocks_to_swap_in,
409
- blocks_to_swap_out=blocks_to_swap_out,
446
blocks_to_copy=blocks_to_copy,
411
- ignored_seq_groups=[],
447
num_lookahead_slots=self._get_num_lookahead_slots(
413
- is_prefill=False),
448
+ is_prefill=False))
449
+
450
+ def _schedule_prefills(
451
+ self,
452
+ waiting_queue: deque,
453
+ budget: SchedulingBudget,
454
+ curr_loras: Optional[Set[int]],
455
+ ) -> Tuple[deque, SchedulerPrefillOutputs]:
456
+ """Schedule sequence groups that are in prefill stage.
457
+
458
+ Note that the current scheduler treats PREEMPTED_FOR_RECOMPUTE
459
+ as a new prefill (that starts from beginning -> most recently generated
460
+ tokens).
461
+
462
+ It schedules waiting requests as long as it fits `budget` and
463
+ curr_loras <= max_lora from the scheduling config. The input arguments
464
+ `budget` and `curr_loras` are updated based on scheduled seq_groups.
465
+
466
+ Args:
467
+ waiting_queue: The queue that contains prefill requests.
468
+ The given arguments are NOT in-place modified.
469
+ budget: The scheduling budget. The argument is in-place updated
470
+ when any requests are scheduled.
471
+ curr_loras: Currently batched lora request ids. The argument is
472
+ in-place updated when any requests are scheduled.
473
+
474
+ Returns:
475
+ A tuple of remaining waiting_queue after scheduling and
476
+ SchedulerSwappedInOutputs.
477
+ """
478
+ ignored_seq_groups: List[SequenceGroup] = []
479
+ seq_groups: List[SequenceGroup] = []
480
+ # We don't sort waiting queue because we assume it is sorted.
481
+ # Copy the queue so that the input queue is not modified.
482
+ waiting_queue = deque([s for s in waiting_queue])
483
+
484
+ leftover_waiting_sequences = deque()
485
+ while self._passed_delay(time.time()) and waiting_queue:
486
+ seq_group = waiting_queue[0]
487
+
488
+ waiting_seqs = seq_group.get_seqs(status=SequenceStatus.WAITING)
489
+ assert len(waiting_seqs) == 1, (
490
+ "Waiting sequence group should have only one prompt "
491
+ "sequence.")
492
+
493
+ num_prompt_tokens = waiting_seqs[0].get_len()
494
+ if num_prompt_tokens > self.prompt_limit:
495
+ logger.warning(
496
+ f"Input prompt ({num_prompt_tokens} tokens) is too long"
497
+ f" and exceeds limit of {self.prompt_limit}")
498
+ for seq in waiting_seqs:
499
+ seq.status = SequenceStatus.FINISHED_IGNORED
500
+ ignored_seq_groups.append(seq_group)
501
+ waiting_queue.popleft()
502
+ continue
503
+
504
+ # If the sequence group cannot be allocated, stop.
505
+ can_allocate = self.block_manager.can_allocate(seq_group)
506
+ if can_allocate == AllocStatus.LATER:
507
+ break
508
+ elif can_allocate == AllocStatus.NEVER:
509
+ logger.warning(
510
+ f"Input prompt ({num_prompt_tokens} tokens) is too long"
511
+ f" and exceeds the capacity of block_manager")
512
+ for seq in waiting_seqs:
513
+ seq.status = SequenceStatus.FINISHED_IGNORED
514
+ ignored_seq_groups.append(seq_group)
515
+ waiting_queue.popleft()
516
+ continue
517
+
518
+ lora_int_id = 0
519
+ if self.lora_enabled:
520
+ lora_int_id = seq_group.lora_int_id
521
+ if (self.lora_enabled and lora_int_id > 0
522
+ and lora_int_id not in curr_loras
523
+ and len(curr_loras) >= self.lora_config.max_loras):
524
+ # We don't have a space for another LoRA, so
525
+ # we ignore this request for now.
526
+ leftover_waiting_sequences.appendleft(seq_group)
527
+ waiting_queue.popleft()
528
+ continue
529
+
530
+ num_new_seqs = seq_group.get_max_num_running_seqs()
531
+ if not budget.can_schedule(num_new_tokens=num_prompt_tokens,
532
+ num_new_seqs=num_new_seqs):
533
+ break
534
+
535
+ # Can schedule this request.
536
+ if curr_loras is not None and lora_int_id > 0:
537
+ curr_loras.add(lora_int_id)
538
+ waiting_queue.popleft()
539
+ self._allocate_and_set_running(seq_group)
540
+ seq_groups.append(
541
+ ScheduledSequenceGroup(seq_group=seq_group,
542
+ token_chunk_size=num_prompt_tokens))
543
+ budget.num_batched_tokens += num_prompt_tokens
544
+ budget.num_curr_seqs += num_new_seqs
545
+
546
+ # Queue requests that couldn't be scheduled.
547
+ waiting_queue.extendleft(leftover_waiting_sequences)
548
+ if len(seq_groups) > 0:
549
+ self.prev_prompt = True
550
+
551
+ return waiting_queue, SchedulerPrefillOutputs(
552
+ seq_groups=seq_groups,
553
+ ignored_seq_groups=ignored_seq_groups,
554
+ num_lookahead_slots=self._get_num_lookahead_slots(is_prefill=True))
555
+
556
+ def _schedule(self) -> SchedulerOutputs:
557
+ """Batch requests that are queued..
558
+
559
+ The current policy is designed to opimimize the throughput. First,
560
+ it batches as many prefill requests as possible. And it schedules
561
+ decodes. If there's a pressure on GPU memory, decode requests can
562
+ be swapped or preempted.
563
+ """
564
+ # Include running requests to the budget.
565
+ budget = SchedulingBudget(
566
+ num_batched_tokens=sum(
567
+ seq_group.num_seqs(status=SequenceStatus.RUNNING)
568
+ for seq_group in self.running),
569
+ num_curr_seqs=sum(seq_group.get_max_num_running_seqs()
570
+ for seq_group in self.running),
571
+ token_budget=self.scheduler_config.max_num_batched_tokens,
572
+ max_num_seqs=self.scheduler_config.max_num_seqs,
573
+ )
574
+ curr_loras = set(
575
+ seq_group.lora_int_id
576
+ for seq_group in self.running) if self.lora_enabled else None
577
+
578
+ remaining_waiting, prefills = (self.waiting,
579
+ SchedulerPrefillOutputs.create_empty())
580
+ remaining_running, decodes = (self.running,
581
+ SchedulerDecodeOutputs.create_empty())
582
+ remaining_swapped, swapped_in = (
583
+ self.swapped, SchedulerSwappedInOutputs.create_empty())
584
+
585
+ # If any requests are swapped, prioritized swapped requests.
586
+ if not self.swapped:
587
+ remaining_waiting, prefills = self._schedule_prefills(
588
+ self.waiting, budget, curr_loras)
589
+
590
+ # Don't schedule decodes if prefills are scheduled.
591
+ if len(prefills.seq_groups) == 0:
592
+ remaining_running, decodes = self._schedule_decodes(
593
+ self.running, budget, curr_loras, self.policy)
594
+ # If any sequence group is preempted, do not swap in any sequence
595
+ # group. because it means there's no slot for new running requests.
596
+ if len(decodes.preempted) + len(decodes.swapped_out) == 0:
597
+ remaining_swapped, swapped_in = self._schedule_swapped(
598
+ self.swapped, budget, curr_loras, self.policy)
599
+
600
+ assert (budget.num_batched_tokens <=
601
+ self.scheduler_config.max_num_batched_tokens)
602
+ assert budget.num_curr_seqs <= self.scheduler_config.max_num_seqs
603
+
604
+ # Update waiting requests.
605
+ self.waiting = remaining_waiting
606
+ self.waiting.extendleft(decodes.preempted)
607
+ # Update new running requests.
608
+ self.running = remaining_running
609
+ self.running.extend([s.seq_group for s in prefills.seq_groups])
610
+ self.running.extend([s.seq_group for s in decodes.seq_groups])
611
+ self.running.extend([s.seq_group for s in swapped_in.seq_groups])
612
+ # Update swapped requests.
613
+ self.swapped = remaining_swapped
614
+ self.swapped.extend(decodes.swapped_out)
615
+
616
+ return SchedulerOutputs(
617
+ scheduled_seq_groups=prefills.seq_groups + decodes.seq_groups +
618
+ swapped_in.seq_groups,
619
+ num_prefill_groups=len(prefills.seq_groups),
620
+ num_batched_tokens=budget.num_batched_tokens,
621
+ blocks_to_swap_in=swapped_in.blocks_to_swap_in,
622
+ blocks_to_swap_out=decodes.blocks_to_swap_out,
623
+ blocks_to_copy=merge_dicts(decodes.blocks_to_copy,
624
+ swapped_in.blocks_to_copy),
625
+ ignored_seq_groups=prefills.ignored_seq_groups,
626
+ num_lookahead_slots=(prefills.num_lookahead_slots +
627
+ decodes.num_lookahead_slots +
628
+ swapped_in.num_lookahead_slots),
629
)
415
- return scheduler_outputs
630
631
def _can_append_slots(self, seq_group: SequenceGroup) -> bool:
632
"""Determine whether or not we have enough space in the KV cache to
@@ -444,7 +658,8 @@ class Scheduler:
658
659
# Create input data structures.
660
seq_group_metadata_list: List[SequenceGroupMetadata] = []
447
- for scheduled_seq_group in scheduler_outputs.scheduled_seq_groups:
661
+ for i, scheduled_seq_group in enumerate(
662
+ scheduler_outputs.scheduled_seq_groups):
663
seq_group = scheduled_seq_group.seq_group
664
token_chunk_size = scheduled_seq_group.token_chunk_size
665
seq_group.maybe_set_first_scheduled_time(now)
@@ -464,9 +679,12 @@ class Scheduler:
679
self.block_manager.get_common_computed_block_ids(
680
seq_group.get_seqs(status=SequenceStatus.RUNNING)))
681
682
+ # It assumes the scheduled_seq_groups is ordered by
683
+ # prefill < decoding.
684
+ is_prompt = i < scheduler_outputs.num_prefill_groups
685
seq_group_metadata = SequenceGroupMetadata(
686
request_id=seq_group.request_id,
469
- is_prompt=scheduler_outputs.prompt_run,
687
+ is_prompt=is_prompt,
688
seq_data=seq_data,
689
sampling_params=seq_group.sampling_params,
690
block_tables=block_tables,
@@ -479,7 +697,7 @@ class Scheduler:
697
# the subsequent comms can still use delta, but
698
# `multi_modal_data` will be None.
699
multi_modal_data=seq_group.multi_modal_data
482
- if scheduler_outputs.prompt_run else None,
700
+ if scheduler_outputs.num_prefill_groups > 0 else None,
701
)
702
seq_group_metadata_list.append(seq_group_metadata)
703
@@ -504,7 +722,7 @@ class Scheduler:
722
self.running = deque(seq_group for seq_group in self.running
723
if not seq_group.is_finished())
724
507
- def _allocate(self, seq_group: SequenceGroup) -> None:
725
+ def _allocate_and_set_running(self, seq_group: SequenceGroup) -> None:
726
self.block_manager.allocate(seq_group)
727
for seq in seq_group.get_seqs(status=SequenceStatus.WAITING):
728
seq.status = SequenceStatus.RUNNING
@@ -539,7 +757,7 @@ class Scheduler:
757
seq_group: SequenceGroup,
758
blocks_to_swap_out: Dict[int, int],
759
preemption_mode: Optional[PreemptionMode] = None,
542
- ) -> None:
760
+ ) -> PreemptionMode:
761
# If preemption mode is not specified, we determine the mode as follows:
762
# We use recomputation by default since it incurs lower overhead than
763
# swapping. However, when the sequence group has multiple sequences
@@ -562,6 +780,7 @@ class Scheduler:
780
self._preempt_by_swap(seq_group, blocks_to_swap_out)
781
else:
782
raise AssertionError("Invalid preemption mode.")
783
+ return preemption_mode
784
785
def _preempt_by_recompute(
786
self,
@@ -573,9 +792,6 @@ class Scheduler:
792
seq.status = SequenceStatus.WAITING
793
self.free_seq(seq)
794
seq.reset_state_for_recompute()
576
- # NOTE: For FCFS, we insert the preempted sequence group to the front
577
- # of the waiting queue.
578
- self.waiting.appendleft(seq_group)
795
796
def _preempt_by_swap(
797
self,
@@ -583,7 +799,6 @@ class Scheduler:
799
blocks_to_swap_out: Dict[int, int],
800
) -> None:
801
self._swap_out(seq_group, blocks_to_swap_out)
586
- self.swapped.append(seq_group)
802
803
def _swap_in(
804
self,
vllm/engine/llm_engine.py
+1
-1
@@ -728,7 +728,7 @@ class LLMEngine:
728
time_per_output_tokens = []
729
time_e2e_requests = []
730
if scheduler_outputs is not None:
731
- prompt_run = scheduler_outputs.prompt_run
731
+ prompt_run = scheduler_outputs.num_prefill_groups > 0
732
733
# Number of Tokens.
734
if prompt_run:
vllm/utils.py
+18
-1
@@ -6,7 +6,7 @@ import socket
6
import subprocess
7
import uuid
8
import warnings
9
-from collections import OrderedDict
9
+from collections import OrderedDict, defaultdict
10
from functools import lru_cache, partial
11
from platform import uname
12
from typing import (Any, Awaitable, Callable, Generic, Hashable, List,
@@ -450,3 +450,20 @@ def maybe_expand_dim(tensor: torch.Tensor,
450
if tensor.ndim < target_dims:
451
tensor = tensor.view(-1, *([size] * (target_dims - tensor.ndim)))
452
return tensor
453
+
454
+
455
+def merge_dicts(dict1: dict[Any, list[Any]],
456
+ dict2: dict[Any, list[Any]]) -> dict[Any, list[Any]]:
457
+ """Merge 2 dicts that have key -> List of items.
458
+
459
+ When a key conflicts, the values in dict1 is prioritized.
460
+ """
461
+ merged_dict = defaultdict(list)
462
+
463
+ for key, value in dict1.items():
464
+ merged_dict[key].extend(value)
465
+
466
+ for key, value in dict2.items():
467
+ merged_dict[key].extend(value)
468
+
469
+ return dict(merged_dict)