@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3dcb3e8b98
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[3/N] Refactor scheduler for chunked prefill scheduling (#3550)

SangBin Cho committed Apr 4, 2024 at 06:13 UTC 3dcb3e8b9838cbbef83ce326b1a35b31a3cf14f2
5 files changed +1020 -255
tests/core/test_scheduler.py
+532 -6
@@ -1,10 +1,15 @@
1 import time
2 +from collections import deque
3 from typing import List
4 +from unittest.mock import MagicMock
5
6 import pytest # noqa
7
6 -from vllm.config import CacheConfig, SchedulerConfig
7 -from vllm.core.scheduler import Scheduler
8 +from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig
9 +from vllm.core.interfaces import AllocStatus
10 +from vllm.core.policy import PolicyFactory
11 +from vllm.core.scheduler import Scheduler, SchedulingBudget
12 +from vllm.lora.request import LoRARequest
13 from vllm.sequence import Logprob, SequenceGroup
14
15 from .utils import create_dummy_prompt
@@ -177,7 +182,6 @@ def test_scheduler_max_seqs():
182
183
184 def test_scheduler_delay_factor():
180 -
185 block_size = 4
186 scheduler_config = SchedulerConfig(100, 64, 16, delay_factor=0.5)
187 cache_config = CacheConfig(block_size, 1.0, 1, "auto")
@@ -189,7 +193,7 @@ def test_scheduler_delay_factor():
193 _, seq_group = create_dummy_prompt("0", prompt_length=block_size)
194 scheduler.add_seq_group(seq_group)
195 seq_group_meta, out = scheduler.schedule()
192 - assert out.prompt_run
196 + assert out.num_prefill_groups > 0
197 assert seq_group_meta[0].request_id == '0'
198
199 # wait for a second before scheduling next prompt
@@ -199,11 +203,533 @@ def test_scheduler_delay_factor():
203
204 # second prompt should *not* be scheduled
205 seq_group_meta, out = scheduler.schedule()
202 - assert not out.prompt_run
206 + assert out.num_prefill_groups == 0
207 assert seq_group_meta[0].request_id == '0'
208
209 # wait for more than 0.5 second and try again
210 time.sleep(0.6)
211 seq_group_meta, out = scheduler.schedule()
208 - assert out.prompt_run
212 + assert out.num_prefill_groups > 0
213 assert seq_group_meta[0].request_id == '1'
214 +
215 +
216 +def test_swapped_out_prioritized():
217 + scheduler = initialize_scheduler(max_num_seqs=6)
218 + # best_of=2 * 3 == 6 sequences.
219 + for i in range(3):
220 + _, seq_group = create_dummy_prompt(str(i), prompt_length=60, best_of=2)
221 + scheduler.add_seq_group(seq_group)
222 + _, out = scheduler.schedule()
223 + # prefill scheduled now.
224 + assert len(out.scheduled_seq_groups) == 3
225 +
226 + # The last request should be swapped out.
227 + scheduler.block_manager.can_append_slots = MagicMock()
228 +
229 + def cannot_append_second_group(seq_group, num_lookahead_slots):
230 + return seq_group.request_id != "2"
231 +
232 + scheduler.block_manager.can_append_slots.side_effect = (
233 + cannot_append_second_group)
234 +
235 + _, out = scheduler.schedule()
236 + assert len(out.scheduled_seq_groups) == 2
237 + assert out.num_batched_tokens == 2
238 + assert out.blocks_to_swap_out != {}
239 + assert out.blocks_to_swap_in == {}
240 +
241 + # Add 1 more task. Swap should be prioritized over prefill.
242 + _, seq_group = create_dummy_prompt(str(i), prompt_length=60, best_of=2)
243 + scheduler.add_seq_group(seq_group)
244 + _, out = scheduler.schedule()
245 + assert len(out.scheduled_seq_groups) == 3
246 + # 3 decodes. It is swapped in.
247 + assert out.num_batched_tokens == 3
248 + assert out.blocks_to_swap_in != {}
249 + assert out.blocks_to_swap_out == {}
250 +
251 +
252 +def initialize_scheduler(*,
253 + max_num_seqs=1000,
254 + max_token_budget=1000,
255 + max_model_len=1000,
256 + lora_config=None):
257 + block_size = 4
258 + scheduler_config = SchedulerConfig(max_token_budget, max_num_seqs,
259 + max_model_len)
260 + cache_config = CacheConfig(block_size, 1.0, 1, "auto")
261 + cache_config.num_cpu_blocks = 8
262 + cache_config.num_gpu_blocks = 8
263 + scheduler = Scheduler(scheduler_config, cache_config, lora_config)
264 + return scheduler
265 +
266 +
267 +def create_token_budget(num_batched_tokens: int = 0,
268 + num_curr_seqs: int = 0,
269 + token_budget: int = 10000,
270 + max_num_seqs: int = 10000) -> SchedulingBudget:
271 + return SchedulingBudget(
272 + num_batched_tokens=num_batched_tokens,
273 + num_curr_seqs=num_curr_seqs,
274 + token_budget=token_budget,
275 + max_num_seqs=max_num_seqs,
276 + )
277 +
278 +
279 +def test_prefill_schedule_max_prompt_len():
280 + """
281 + Test prompt longer than max_prompt_len is aborted.
282 + """
283 + scheduler = initialize_scheduler(max_model_len=30)
284 + _, seq_group = create_dummy_prompt(0, prompt_length=60)
285 + waiting = deque([seq_group])
286 + budget = create_token_budget()
287 + remaining_waiting, output = scheduler._schedule_prefills(
288 + waiting, budget, None)
289 + assert len(output.ignored_seq_groups) == 1
290 + assert len(output.seq_groups) == 0
291 + assert budget.num_batched_tokens == 0
292 + assert budget.num_curr_seqs == 0
293 + assert len(remaining_waiting) == 0
294 +
295 +
296 +def test_prefill_schedule_token_budget():
297 + """
298 + Test token budget respected.
299 + """
300 + scheduler = initialize_scheduler()
301 + waiting = deque()
302 + budget = create_token_budget(token_budget=0)
303 + for i in range(2):
304 + _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
305 + waiting.append(seq_group)
306 +
307 + # 0 token budget == nothing is scheduled.
308 + remaining_waiting, output = scheduler._schedule_prefills(
309 + waiting, budget, None)
310 + assert len(output.ignored_seq_groups) == 0
311 + assert len(output.seq_groups) == 0
312 + assert budget.num_batched_tokens == 0
313 + assert budget.num_curr_seqs == 0
314 + assert len(remaining_waiting) == 2
315 +
316 + # 60 token budget == 1 request scheduled.
317 + budget = create_token_budget(token_budget=60)
318 + remaining_waiting, output = scheduler._schedule_prefills(
319 + waiting, budget, None)
320 + assert len(output.ignored_seq_groups) == 0
321 + assert len(output.seq_groups) == 1
322 + assert budget.num_batched_tokens == 60
323 + assert budget.num_curr_seqs == 1
324 + assert len(remaining_waiting) == 1
325 +
326 + # Test when current_batched_tokens respected.
327 + scheduler = initialize_scheduler()
328 + waiting = deque()
329 + budget = create_token_budget(num_batched_tokens=30, token_budget=60)
330 + _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
331 + # Cannot schedule a prompt that doesn't fit the budget.
332 + waiting.append(seq_group)
333 + remaining_waiting, output = scheduler._schedule_prefills(
334 + waiting, budget, None)
335 + assert len(output.ignored_seq_groups) == 0
336 + assert len(output.seq_groups) == 0
337 + assert budget.num_batched_tokens == 30
338 + assert budget.num_curr_seqs == 0
339 + assert len(remaining_waiting) == 1
340 + budget = create_token_budget(num_batched_tokens=30, token_budget=90)
341 + remaining_waiting, output = scheduler._schedule_prefills(
342 + waiting, budget, None)
343 + assert len(output.seq_groups) == 1
344 + assert budget.num_batched_tokens == 90
345 + assert budget.num_curr_seqs == 1
346 + assert len(remaining_waiting) == 0
347 +
348 +
349 +def test_prefill_schedule_max_seqs():
350 + """
351 + Test max seq respected.
352 + """
353 + scheduler = initialize_scheduler()
354 + waiting = deque()
355 + budget = create_token_budget(max_num_seqs=2)
356 + for i in range(3):
357 + _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
358 + waiting.append(seq_group)
359 + remaining_waiting, output = scheduler._schedule_prefills(
360 + waiting, budget, None)
361 + assert len(output.ignored_seq_groups) == 0
362 + assert len(output.seq_groups) == 2
363 + assert budget.num_batched_tokens == 120
364 + assert budget.num_curr_seqs == 2
365 + assert len(remaining_waiting) == 1
366 +
367 + # Verify curr_num_seqs respected.
368 + waiting = deque()
369 + budget = create_token_budget(num_curr_seqs=2, max_num_seqs=2)
370 + _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
371 + waiting.append(seq_group)
372 + remaining_waiting, output = scheduler._schedule_prefills(
373 + waiting, budget, None)
374 + assert len(output.ignored_seq_groups) == 0
375 + assert len(output.seq_groups) == 0
376 + assert budget.num_batched_tokens == 0
377 + assert budget.num_curr_seqs == 2
378 + assert len(remaining_waiting) == 1
379 +
380 +
381 +def test_prefill_schedule_max_lora():
382 + """
383 + Test max lora is respected and prioritized.
384 + """
385 + lora_config = LoRAConfig(max_lora_rank=8, max_loras=1)
386 + scheduler = initialize_scheduler(lora_config=lora_config)
387 + waiting = deque()
388 + budget = create_token_budget(token_budget=120)
389 + curr_loras = set()
390 + for i in range(2):
391 + _, seq_group = create_dummy_prompt(str(i),
392 + prompt_length=60,
393 + lora_request=LoRARequest(
394 + lora_name=str(i),
395 + lora_int_id=i + 1,
396 + lora_local_path="abc"))
397 + waiting.append(seq_group)
398 + # Add two more requests to verify lora is prioritized.
399 + # 0: Lora, 1: Lora, 2: regular, 3: regular
400 + # In the first iteration, index 0, 2 is scheduled.
401 + # If a request is not scheduled because it hits max lora, it is
402 + # prioritized. Verify that.
403 + for i in range(2, 4):
404 + _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
405 + waiting.append(seq_group)
406 + # Schedule 2 requests (0 and 2)
407 + remaining_waiting, output = scheduler._schedule_prefills(
408 + waiting, budget, curr_loras)
409 + assert len(output.ignored_seq_groups) == 0
410 + assert len(output.seq_groups) == 2
411 + assert budget.num_batched_tokens == 120
412 + assert budget.num_curr_seqs == 2
413 + assert len(remaining_waiting) == 2
414 + assert len(curr_loras) == 1
415 + # The second lora request is scheduled next as FCFS policy.
416 + # Reset curr_loras so that it can be scheduled.
417 + curr_loras = set()
418 + budget = create_token_budget(token_budget=60)
419 + remaining_waiting, output = scheduler._schedule_prefills(
420 + remaining_waiting, budget, curr_loras)
421 + assert len(output.seq_groups) == 1
422 + assert output.seq_groups[0].seq_group.request_id == "1"
423 + assert len(remaining_waiting) == 1
424 + assert len(curr_loras) == 1
425 + assert budget.num_batched_tokens == 60
426 +
427 +
428 +def test_prefill_schedule_no_block_manager_capacity():
429 + """
430 + Test sequence cannot be scheduled due to block manager has no capacity.
431 + """
432 + scheduler = initialize_scheduler()
433 + waiting = deque()
434 + budget = create_token_budget()
435 + for i in range(3):
436 + _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
437 + waiting.append(seq_group)
438 + scheduler.block_manager.can_allocate = MagicMock()
439 + scheduler.block_manager.can_allocate.return_value = AllocStatus.LATER
440 + remainig_waiting, output = scheduler._schedule_prefills(
441 + waiting, budget, None)
442 + assert len(output.ignored_seq_groups) == 0
443 + assert len(output.seq_groups) == 0
444 + assert budget.num_batched_tokens == 0
445 + assert budget.num_curr_seqs == 0
446 + assert len(remainig_waiting) == 3
447 +
448 + scheduler = initialize_scheduler()
449 + waiting = deque()
450 + budget = create_token_budget()
451 + for i in range(3):
452 + _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
453 + waiting.append(seq_group)
454 + scheduler.block_manager.can_allocate = MagicMock()
455 + scheduler.block_manager.can_allocate.return_value = AllocStatus.NEVER
456 + remaining_waiting, output = scheduler._schedule_prefills(
457 + waiting, budget, None)
458 + assert len(output.ignored_seq_groups) == 3
459 + assert len(output.seq_groups) == 0
460 + assert budget.num_batched_tokens == 0
461 + assert budget.num_curr_seqs == 0
462 + assert len(remaining_waiting) == 0
463 +
464 +
465 +def test_decode_schedule_preempted():
466 + """
467 + Test decodes cannot be scheduled and preempted.
468 + """
469 + scheduler = initialize_scheduler()
470 + running = deque()
471 + policy = PolicyFactory.get_policy(policy_name="fcfs")
472 + curr_loras = None
473 + for i in range(3):
474 + _, seq_group = create_dummy_prompt(str(i), prompt_length=60)
475 + scheduler._allocate_and_set_running(seq_group)
476 + running.append(seq_group)
477 + scheduler.block_manager.can_append_slots = MagicMock()
478 +
479 + def cannot_append_second_group(seq_group, num_lookahead_slots):
480 + return seq_group.request_id != "1"
481 +
482 + scheduler.block_manager.can_append_slots.side_effect = (
483 + cannot_append_second_group)
484 +
485 + # 1 cannot be scheduled, and the lowest priority (request 2)
486 + # should be preempted. 1 will also be preempted.
487 + budget = create_token_budget(num_batched_tokens=3, num_curr_seqs=3)
488 + remainig_running, output = scheduler._schedule_decodes(
489 + running, budget, curr_loras, policy)
490 + assert len(remainig_running) == 0
491 + assert len(output.seq_groups) == 1
492 + assert output.seq_groups[0].seq_group.request_id == "0"
493 + assert len(output.preempted) == 2
494 + # Verify budgets are updated.
495 + assert budget.num_batched_tokens == 1
496 + assert budget.num_curr_seqs == 1
497 + # Both should be preempted, not swapped.
498 + assert output.blocks_to_swap_out == {}
499 + # Nothing is copied.
500 + assert output.blocks_to_copy == {}
501 +
502 +
503 +def test_decode_swap_beam_search():
504 + """
505 + Test best_of > 1 swap out blocks
506 + """
507 + scheduler = initialize_scheduler()
508 + running = deque()
509 + policy = PolicyFactory.get_policy(policy_name="fcfs")
510 + curr_loras = None
511 + for i in range(3):
512 + _, seq_group = create_dummy_prompt(str(i), prompt_length=60, best_of=2)
513 + scheduler._allocate_and_set_running(seq_group)
514 + running.append(seq_group)
515 +
516 + # The last request should be swapped out.
517 + scheduler.block_manager.can_append_slots = MagicMock()
518 +
519 + def cannot_append_second_group(seq_group, num_lookahead_slots):
520 + return seq_group.request_id != "2"
521 +
522 + scheduler.block_manager.can_append_slots.side_effect = (
523 + cannot_append_second_group)
524 + scheduler.block_manager.swap_out = MagicMock()
525 + expected_swap_mapping = {"5": "7"}
526 + scheduler.block_manager.swap_out.return_value = expected_swap_mapping
527 +
528 + budget = create_token_budget(num_batched_tokens=3, num_curr_seqs=3)
529 + remainig_running, output = scheduler._schedule_decodes(
530 + running, budget, curr_loras, policy)
531 + assert len(remainig_running) == 0
532 + assert len(output.seq_groups) == 2
533 + assert output.seq_groups[0].seq_group.request_id == "0"
534 + assert output.seq_groups[1].seq_group.request_id == "1"
535 + assert len(output.preempted) == 0
536 + assert len(output.swapped_out) == 1
537 + # Budget should refledct preempted requests.
538 + assert budget.num_batched_tokens == 2
539 + # since there are 2 sequences, 2 should be subtracted.
540 + assert budget.num_curr_seqs == 1
541 + # Both should be preempted, not swapped.
542 + assert output.blocks_to_swap_out == expected_swap_mapping
543 + # Nothing is copied.
544 + assert output.blocks_to_copy == {}
545 +
546 +
547 +def test_schedule_decode_blocks_to_copy_update():
548 + """
549 + Verify blocks_to_copy is updated.
550 + """
551 + scheduler = initialize_scheduler()
552 + _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
553 + running = deque()
554 + policy = PolicyFactory.get_policy(policy_name="fcfs")
555 + curr_loras = None
556 + scheduler._allocate_and_set_running(seq_group)
557 + running.append(seq_group)
558 +
559 + # The last request should be swapped out.
560 + scheduler.block_manager.append_slots = MagicMock()
561 + scheduler.block_manager.append_slots.return_value = {2: [3]}
562 +
563 + budget = create_token_budget()
564 + remaining_running, output = scheduler._schedule_decodes(
565 + running, budget, curr_loras, policy)
566 + assert len(remaining_running) == 0
567 + assert len(output.seq_groups) == 1
568 + assert len(output.preempted) == 0
569 + assert len(output.swapped_out) == 0
570 + # Nothing is preempted.
571 + assert output.blocks_to_swap_out == {}
572 + # Since append_slot returns the source -> dist mapping, it should
573 + # applied.
574 + assert output.blocks_to_copy == {2: [3]}
575 +
576 +
577 +def test_schedule_swapped_simple():
578 + scheduler = initialize_scheduler()
579 + swapped = deque()
580 + policy = PolicyFactory.get_policy(policy_name="fcfs")
581 + curr_loras = None
582 + blocks_to_swap_out = {}
583 + _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
584 + scheduler._allocate_and_set_running(seq_group)
585 + scheduler._swap_out(seq_group, blocks_to_swap_out)
586 + swapped.append(seq_group)
587 +
588 + budget = create_token_budget()
589 + remaining_swapped, output = scheduler._schedule_swapped(
590 + swapped, budget, curr_loras, policy)
591 + assert len(remaining_swapped) == 0
592 + assert budget.num_batched_tokens == 1
593 + assert budget.num_curr_seqs == 2
594 + assert len(output.seq_groups) == 1
595 + # swap in is the reverse of swap out
596 + blocks_to_swap_in_reverse = {}
597 + for swapin, swapout in output.blocks_to_swap_in.items():
598 + blocks_to_swap_in_reverse[swapout] = swapin
599 + assert blocks_to_swap_out == blocks_to_swap_in_reverse
600 +
601 +
602 +def test_schedule_swapped_max_token_budget():
603 + scheduler = initialize_scheduler()
604 + swapped = deque()
605 + policy = PolicyFactory.get_policy(policy_name="fcfs")
606 + curr_loras = None
607 + blocks_to_swap_out = {}
608 + for _ in range(2):
609 + _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
610 + scheduler._allocate_and_set_running(seq_group)
611 + scheduler._swap_out(seq_group, blocks_to_swap_out)
612 + swapped.append(seq_group)
613 +
614 + budget = create_token_budget(token_budget=1)
615 + remaining_swapped, output = scheduler._schedule_swapped(
616 + swapped, budget, curr_loras, policy)
617 + assert len(remaining_swapped) == 1
618 + assert budget.num_batched_tokens == 1
619 + assert budget.num_curr_seqs == 2
620 + assert len(output.seq_groups) == 1
621 +
622 + # Verify num_batched_tokens are respected.
623 + budget = create_token_budget(num_batched_tokens=1, token_budget=1)
624 + remaining_swapped, output = scheduler._schedule_swapped(
625 + remaining_swapped, budget, curr_loras, policy)
626 + assert len(remaining_swapped) == 1
627 + assert budget.num_batched_tokens == 1
628 + assert budget.num_curr_seqs == 0
629 + assert len(output.seq_groups) == 0
630 +
631 +
632 +def test_schedule_swapped_max_seqs():
633 + scheduler = initialize_scheduler()
634 + swapped = deque()
635 + policy = PolicyFactory.get_policy(policy_name="fcfs")
636 + curr_loras = None
637 + blocks_to_swap_out = {}
638 + for _ in range(2):
639 + _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
640 + scheduler._allocate_and_set_running(seq_group)
641 + scheduler._swap_out(seq_group, blocks_to_swap_out)
642 + swapped.append(seq_group)
643 +
644 + budget = create_token_budget(max_num_seqs=2)
645 + remaining_swapped, output = scheduler._schedule_swapped(
646 + swapped, budget, curr_loras, policy)
647 + assert len(remaining_swapped) == 1
648 + assert budget.num_batched_tokens == 1
649 + assert budget.num_curr_seqs == 2
650 + assert len(output.seq_groups) == 1
651 +
652 + # Verify num_curr_seqs are respected.
653 + budget = create_token_budget(num_curr_seqs=2, max_num_seqs=2)
654 + remaining_swapped, output = scheduler._schedule_swapped(
655 + remaining_swapped, budget, curr_loras, policy)
656 + assert len(remaining_swapped) == 1
657 + assert budget.num_batched_tokens == 0
658 + assert budget.num_curr_seqs == 2
659 + assert len(output.seq_groups) == 0
660 +
661 +
662 +def test_schedule_swapped_max_loras():
663 + lora_config = LoRAConfig(max_lora_rank=8, max_loras=1)
664 + scheduler = initialize_scheduler(lora_config=lora_config)
665 + swapped = deque()
666 + policy = PolicyFactory.get_policy(policy_name="fcfs")
667 + curr_loras = set()
668 + blocks_to_swap_out = {}
669 + for i in range(2):
670 + _, seq_group = create_dummy_prompt(str(i),
671 + prompt_length=60,
672 + lora_request=LoRARequest(
673 + lora_name=str(i),
674 + lora_int_id=i + 1,
675 + lora_local_path="abc"))
676 + scheduler._allocate_and_set_running(seq_group)
677 + scheduler._swap_out(seq_group, blocks_to_swap_out)
678 + swapped.append(seq_group)
679 +
680 + budget = create_token_budget()
681 + remaining_swapped, output = scheduler._schedule_swapped(
682 + swapped, budget, curr_loras, policy)
683 + assert len(remaining_swapped) == 1
684 + assert budget.num_batched_tokens == 1
685 + assert budget.num_curr_seqs == 1
686 + assert len(output.seq_groups) == 1
687 + assert len(curr_loras) == 1
688 +
689 +
690 +def test_schedule_swapped_cannot_swap_in():
691 + scheduler = initialize_scheduler()
692 + swapped = deque()
693 + policy = PolicyFactory.get_policy(policy_name="fcfs")
694 + curr_loras = None
695 + blocks_to_swap_out = {}
696 + for _ in range(2):
697 + _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
698 + scheduler._allocate_and_set_running(seq_group)
699 + scheduler._swap_out(seq_group, blocks_to_swap_out)
700 + swapped.append(seq_group)
701 +
702 + # The last request should be swapped out.
703 + scheduler.block_manager.can_swap_in = MagicMock()
704 + scheduler.block_manager.can_swap_in.return_value = False
705 + # Since we cannot swap in, none of the requests are swapped in.
706 + budget = create_token_budget()
707 + remaining_swapped, output = scheduler._schedule_swapped(
708 + swapped, budget, curr_loras, policy)
709 + assert len(remaining_swapped) == 2
710 + assert budget.num_batched_tokens == 0
711 + assert budget.num_curr_seqs == 0
712 + assert len(output.seq_groups) == 0
713 +
714 +
715 +def test_schedule_swapped_blocks_to_copy():
716 + scheduler = initialize_scheduler()
717 + swapped = deque()
718 + policy = PolicyFactory.get_policy(policy_name="fcfs")
719 + curr_loras = None
720 + _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
721 + scheduler._allocate_and_set_running(seq_group)
722 + blocks_to_swap_out = {}
723 + scheduler._swap_out(seq_group, blocks_to_swap_out)
724 + swapped.append(seq_group)
725 +
726 + # The last request should be swapped out.
727 + scheduler.block_manager.append_slots = MagicMock()
728 + scheduler.block_manager.append_slots.return_value = {2: [3]}
729 +
730 + budget = create_token_budget()
731 + remaining_swapped, output = scheduler._schedule_swapped(
732 + swapped, budget, curr_loras, policy)
733 + assert len(remaining_swapped) == 0
734 + assert len(output.seq_groups) == 1
735 + assert output.blocks_to_copy == {2: [3]}
tests/core/utils.py
+13 -6
@@ -1,14 +1,19 @@
1 import time
2 -from typing import Tuple
2 +from typing import Optional, Tuple
3
4 from vllm import SamplingParams
5 +from vllm.lora.request import LoRARequest
6 from vllm.sequence import Logprob, Sequence, SequenceGroup
7
8
9 def create_dummy_prompt(
9 - request_id: str,
10 - prompt_length: int,
11 - block_size: int = None) -> Tuple[Sequence, SequenceGroup]:
10 + request_id: str,
11 + prompt_length: int,
12 + block_size: Optional[int] = None,
13 + lora_request: Optional[LoRARequest] = None,
14 + use_beam_search: bool = False,
15 + best_of: int = 1,
16 +) -> Tuple[Sequence, SequenceGroup]:
17 if not block_size:
18 block_size = prompt_length
19
@@ -17,8 +22,10 @@ def create_dummy_prompt(
22 prompt_tokens = list(range(prompt_length))
23 prompt_str = " ".join([str(t) for t in prompt_tokens])
24 prompt = Sequence(int(request_id), prompt_str, prompt_tokens, block_size)
20 - seq_group = SequenceGroup(request_id, [prompt], SamplingParams(),
21 - time.time(), None)
25 + seq_group = SequenceGroup(
26 + request_id, [prompt],
27 + SamplingParams(use_beam_search=use_beam_search, best_of=best_of),
28 + time.time(), lora_request)
29
30 return prompt, seq_group
31
vllm/core/scheduler.py
+456 -241
@@ -6,11 +6,12 @@ from typing import Deque, Dict, Iterable, List, Optional, Set, Tuple, Union
6
7 from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig
8 from vllm.core.interfaces import AllocStatus, BlockSpaceManager
9 -from vllm.core.policy import PolicyFactory
9 +from vllm.core.policy import Policy, PolicyFactory
10 from vllm.logger import init_logger
11 from vllm.lora.request import LoRARequest
12 from vllm.sequence import (Sequence, SequenceData, SequenceGroup,
13 SequenceGroupMetadata, SequenceStatus)
14 +from vllm.utils import merge_dicts
15
16 logger = init_logger(__name__)
17
@@ -28,9 +29,19 @@ class PreemptionMode(enum.Enum):
29 RECOMPUTE = enum.auto()
30
31
31 -# seq_group: SequenceGroup to schedule.
32 -# token_chunk_size: The number of prefill tokens to be processed in the next
33 -# step.
32 +@dataclass
33 +class SchedulingBudget:
34 + """The available slots for scheduling."""
35 + num_batched_tokens: int
36 + num_curr_seqs: int
37 + token_budget: int
38 + max_num_seqs: int
39 +
40 + def can_schedule(self, *, num_new_tokens: int, num_new_seqs: int):
41 + return (self.num_batched_tokens + num_new_tokens <= self.token_budget
42 + and self.num_curr_seqs + num_new_seqs <= self.max_num_seqs)
43 +
44 +
45 @dataclass
46 class ScheduledSequenceGroup:
47 # A sequence group that's scheduled.
@@ -41,53 +52,28 @@ class ScheduledSequenceGroup:
52 token_chunk_size: int
53
54
55 +@dataclass
56 class SchedulerOutputs:
45 -
46 - def __init__(
47 - self,
48 - scheduled_seq_groups: Iterable[ScheduledSequenceGroup],
49 - prompt_run: bool,
50 - num_batched_tokens: int,
51 - blocks_to_swap_in: Dict[int, int],
52 - blocks_to_swap_out: Dict[int, int],
53 - blocks_to_copy: Dict[int, List[int]],
54 - ignored_seq_groups: List[SequenceGroup],
55 - num_lookahead_slots: int,
56 - ) -> None:
57 - """A list of sequence groups to be scheduled as a single batch.
58 -
59 - Args:
60 - scheduled_seq_groups: A tuple of scheduled sequence group and its
61 - token chunk size.
62 - prompt_run: True if all sequence groups are in prefill phase.
63 - If False, all sequence groups are in decoding phase.
64 - num_batched_tokens: Total number of batched tokens.
65 - blocks_to_swap_in: Blocks to swap in. Dict of CPU -> GPU block
66 - number.
67 - blocks_to_swap_out: Blocks to swap out. Dict of GPU -> CPU block
68 - number.
69 - blocks_to_copy: Blocks to copy. Source to a list of dest blocks.
70 - ignored_seq_groups: Sequence groups that are going to be ignored.
71 - """
72 - # A tuple of scheduled sequence group and its chunk size.
73 - self.scheduled_seq_groups: ScheduledSequenceGroup = scheduled_seq_groups
74 - # True if all sequence groups are in prefill phase. If False, all
75 - # sequence groups are in decoding phase.
76 - self.prompt_run: bool = prompt_run
77 - # Total number of batched tokens.
78 - self.num_batched_tokens: int = num_batched_tokens
79 - # Blocks to swap in. Dict of CPU -> GPU block number.
80 - self.blocks_to_swap_in: Dict[int, int] = blocks_to_swap_in
81 - # Blocks to swap out. Dict of GPU -> CPU block number.
82 - self.blocks_to_swap_out: Dict[int, int] = blocks_to_swap_out
83 - # Blocks to copy. Source to a list of dest blocks.
84 - self.blocks_to_copy: Dict[int, List[int]] = blocks_to_copy
85 - # Sequence groups that are going to be ignored.
86 - self.ignored_seq_groups: List[SequenceGroup] = ignored_seq_groups
87 -
57 + # Scheduled sequence groups.
58 + scheduled_seq_groups: Iterable[ScheduledSequenceGroup]
59 + # Number of prefill groups scheduled.
60 + num_prefill_groups: int
61 + # Total number of batched tokens.
62 + num_batched_tokens: int
63 + # Blocks to swap in. Dict of CPU -> GPU block number.
64 + blocks_to_swap_in: Dict[int, int]
65 + # Blocks to swap out. Dict of GPU -> CPU block number.
66 + blocks_to_swap_out: Dict[int, int]
67 + # Blocks to copy. Source to a list of dest blocks.
68 + blocks_to_copy: Dict[int, List[int]]
69 + # Sequence groups that are going to be ignored.
70 + ignored_seq_groups: List[SequenceGroup]
71 + # The number of slots for lookahead decoding.
72 + num_lookahead_slots: int
73 +
74 + def __post_init__(self):
75 # Swap in and swap out should never happen at the same time.
89 - assert not (blocks_to_swap_in and blocks_to_swap_out)
90 - self.num_lookahead_slots = num_lookahead_slots
76 + assert not (self.blocks_to_swap_in and self.blocks_to_swap_out)
77
78 self.num_loras: int = len(self.lora_requests)
79 if self.num_loras > 0:
@@ -108,6 +94,73 @@ class SchedulerOutputs:
94 return {g.seq_group.lora_request for g in self.scheduled_seq_groups}
95
96
97 +@dataclass
98 +class SchedulerDecodeOutputs:
99 + """Outputs of the decoding phase of the scheduler."""
100 + # Selected sequence groups for decoding.
101 + seq_groups: List[SequenceGroup]
102 + # The preempted sequences.
103 + preempted: List[SequenceGroup]
104 + # Sequences that are swapped out.
105 + swapped_out: List[SequenceGroup]
106 + # The blocks to swap out.
107 + blocks_to_swap_out: Dict[int, int]
108 + # The blocks to copy.
109 + blocks_to_copy: Dict[int, List[int]]
110 + num_lookahead_slots: int
111 +
112 + @classmethod
113 + def create_empty(cls) -> "SchedulerDecodeOutputs":
114 + return SchedulerDecodeOutputs(
115 + seq_groups=[],
116 + preempted=[],
117 + swapped_out=[],
118 + blocks_to_swap_out={},
119 + blocks_to_copy={},
120 + num_lookahead_slots=0,
121 + )
122 +
123 +
124 +@dataclass
125 +class SchedulerSwappedInOutputs:
126 + """Outputs of the decoding phase of the scheduler."""
127 + # Selected sequence groups for decoding.
128 + seq_groups: List[SequenceGroup]
129 + # The blocks to swap in.
130 + blocks_to_swap_in: Dict[int, int]
131 + # The blocks to copy.
132 + blocks_to_copy: Dict[int, List[int]]
133 + # # The number of batched tokens.
134 + num_lookahead_slots: int
135 +
136 + @classmethod
137 + def create_empty(cls) -> "SchedulerSwappedInOutputs":
138 + return SchedulerSwappedInOutputs(
139 + seq_groups=[],
140 + blocks_to_swap_in={},
141 + blocks_to_copy={},
142 + num_lookahead_slots=0,
143 + )
144 +
145 +
146 +@dataclass
147 +class SchedulerPrefillOutputs:
148 + """Outputs of the prefill phase of the scheduler."""
149 + # Selected sequence groups for prefill.
150 + seq_groups: List[SequenceGroup]
151 + # Ignored sequence groups.
152 + ignored_seq_groups: List[SequenceGroup]
153 + num_lookahead_slots: int
154 +
155 + @classmethod
156 + def create_empty(cls) -> "SchedulerPrefillOutputs":
157 + return SchedulerPrefillOutputs(
158 + seq_groups=[],
159 + ignored_seq_groups=[],
160 + num_lookahead_slots=0,
161 + )
162 +
163 +
164 class Scheduler:
165
166 def __init__(
@@ -123,6 +176,7 @@ class Scheduler:
176 # LoRAs. This should be improved in the future.
177 self.lora_config = lora_config
178
179 + # TODO(sang): Fix it after chunked prefill is enabled.
180 self.prompt_limit = min(self.scheduler_config.max_model_len,
181 self.scheduler_config.max_num_batched_tokens)
182
@@ -142,10 +196,13 @@ class Scheduler:
196 enable_caching=self.cache_config.enable_prefix_caching)
197
198 # Sequence groups in the WAITING state.
199 + # Contain new prefill or preempted requests.
200 self.waiting: Deque[SequenceGroup] = deque()
201 # Sequence groups in the RUNNING state.
202 + # Contain decode requests.
203 self.running: Deque[SequenceGroup] = deque()
204 # Sequence groups in the SWAPPED state.
205 + # Contain decode requests that are swapped out.
206 self.swapped: Deque[SequenceGroup] = deque()
207
208 # Time at previous scheduling step
@@ -159,8 +216,14 @@ class Scheduler:
216 def lora_enabled(self) -> bool:
217 return bool(self.lora_config)
218
219 + @property
220 + def num_decoding_tokens_per_seq(self) -> int:
221 + """The number of new tokens."""
222 + return 1
223 +
224 def add_seq_group(self, seq_group: SequenceGroup) -> None:
225 # Add sequence groups to the waiting queue.
226 + logger.debug(f"add_seq_group {seq_group.request_id}")
227 self.waiting.append(seq_group)
228
229 def abort_seq_group(self, request_id: Union[str, Iterable[str]]) -> None:
@@ -205,214 +268,365 @@ class Scheduler:
268 def get_num_unfinished_seq_groups(self) -> int:
269 return len(self.waiting) + len(self.running) + len(self.swapped)
270
208 - def _schedule(self) -> SchedulerOutputs:
209 - # Blocks that need to be swapped or copied before model execution.
210 - blocks_to_swap_in: Dict[int, int] = {}
211 - blocks_to_swap_out: Dict[int, int] = {}
212 - blocks_to_copy: Dict[int, List[int]] = {}
213 -
214 - # Fix the current time.
215 - now = time.time()
216 -
217 - # Join waiting sequences if possible.
218 - if not self.swapped:
219 - ignored_seq_groups: List[SequenceGroup] = []
220 - scheduled: List[SequenceGroup] = []
221 - # The total number of sequences on the fly, including the
222 - # requests in the generation phase.
223 - num_curr_seqs = sum(seq_group.get_max_num_running_seqs()
224 - for seq_group in self.running)
225 - curr_loras = set(
226 - seq_group.lora_int_id
227 - for seq_group in self.running) if self.lora_enabled else None
228 -
229 - # Optimization: We do not sort the waiting queue since the preempted
230 - # sequence groups are added to the front and the new sequence groups
231 - # are added to the back.
232 - leftover_waiting_sequences = deque()
233 - num_batched_tokens = 0
234 - while self._passed_delay(now) and self.waiting:
235 - seq_group = self.waiting[0]
236 - waiting_seqs = seq_group.get_seqs(
237 - status=SequenceStatus.WAITING)
238 - assert len(waiting_seqs) == 1, (
239 - "Waiting sequence group should have only one prompt "
240 - "sequence.")
241 - # get_len includes output tokens if the request has been
242 - # preempted.
243 - num_prefill_tokens = waiting_seqs[0].get_len()
244 - if num_prefill_tokens > self.prompt_limit:
245 - logger.warning(
246 - f"Input prompt ({num_prefill_tokens} tokens) is too "
247 - f"long and exceeds limit of {self.prompt_limit}")
248 - for seq in waiting_seqs:
249 - seq.status = SequenceStatus.FINISHED_IGNORED
250 - ignored_seq_groups.append(seq_group)
251 - self.waiting.popleft()
252 - continue
253 -
254 - # If the sequence group cannot be allocated, stop.
255 - can_allocate = self.block_manager.can_allocate(seq_group)
256 - if can_allocate == AllocStatus.LATER:
257 - break
258 - elif can_allocate == AllocStatus.NEVER:
259 - logger.warning(
260 - f"Input prompt ({num_prefill_tokens} tokens) is too "
261 - f"long and exceeds the capacity of block_manager")
262 - for seq in waiting_seqs:
263 - seq.status = SequenceStatus.FINISHED_IGNORED
264 - ignored_seq_groups.append(seq_group)
265 - self.waiting.popleft()
266 - continue
271 + def _schedule_decodes(
272 + self,
273 + running_queue: deque,
274 + budget: SchedulingBudget,
275 + curr_loras: Optional[Set[int]],
276 + policy: Policy,
277 + ) -> Tuple[deque, SchedulerDecodeOutputs]:
278 + """Schedule sequence groups in a decoding stage.
279
268 - lora_int_id = 0
269 - if self.lora_enabled:
270 - lora_int_id = seq_group.lora_int_id
271 - if (lora_int_id > 0 and lora_int_id not in curr_loras
272 - and len(curr_loras) >= self.lora_config.max_loras):
273 - # We don't have a space for another LoRA, so
274 - # we ignore this request for now.
275 - leftover_waiting_sequences.appendleft(seq_group)
276 - self.waiting.popleft()
277 - continue
280 + NOTE(sang): All the RUNNING num_batched_tokens, num_curr_seqs,
281 + and curr_loras should be already included in `budget` and `curr_loras`.
282 + The API doesn't ADD UP these values.
283
279 - # If the number of batched tokens exceeds the limit, stop.
280 - num_batched_tokens += num_prefill_tokens
281 - if (num_batched_tokens >
282 - self.scheduler_config.max_num_batched_tokens):
283 - break
284 + Note that `budget` and `curr_loras` are still subtracted/popped when
285 + any running requests are preempted from this API.
286
285 - # The total number of sequences in the RUNNING state should not
286 - # exceed the maximum number of sequences.
287 - num_new_seqs = seq_group.get_max_num_running_seqs()
288 - if (num_curr_seqs + num_new_seqs >
289 - self.scheduler_config.max_num_seqs):
290 - break
287 + Args:
288 + running_queue: The queue that contains running requests (i.e.,
289 + decodes). The given arguments are NOT in-place modified.
290 + budget: The scheduling budget. The argument is in-place updated
291 + when any decodes are preempted.
292 + curr_loras: Currently batched lora request ids. The argument is
293 + in-place updated when any decodes are preempted.
294 + policy: The sorting policy to sort running_queue.
295 +
296 + Returns:
297 + A tuple of remaining running queue (should be always 0) after
298 + scheduling and SchedulerDecodeOutputs.
299 + """
300 + # Blocks that need to be swapped or copied before model execution.
301 + blocks_to_swap_out: Dict[int, int] = {}
302 + blocks_to_copy: Dict[int, List[int]] = {}
303
292 - if lora_int_id > 0:
293 - curr_loras.add(lora_int_id)
294 - self.waiting.popleft()
295 - self._allocate(seq_group)
296 - self.running.append(seq_group)
297 - num_curr_seqs += num_new_seqs
298 - scheduled.append(
299 - ScheduledSequenceGroup(
300 - seq_group=seq_group,
301 - token_chunk_size=num_prefill_tokens))
302 - self.waiting.extendleft(leftover_waiting_sequences)
303 -
304 - if scheduled or ignored_seq_groups:
305 - self.prev_prompt = True
306 - scheduler_outputs = SchedulerOutputs(
307 - scheduled_seq_groups=scheduled,
308 - prompt_run=True,
309 - num_batched_tokens=num_batched_tokens,
310 - blocks_to_swap_in=blocks_to_swap_in,
311 - blocks_to_swap_out=blocks_to_swap_out,
312 - blocks_to_copy=blocks_to_copy,
313 - ignored_seq_groups=ignored_seq_groups,
314 - num_lookahead_slots=self._get_num_lookahead_slots(
315 - is_prefill=True),
316 - )
317 - return scheduler_outputs
304 + seq_groups: List[ScheduledSequenceGroup] = []
305 + preempted: List[SequenceGroup] = []
306 + swapped_out: List[SequenceGroup] = []
307
308 # NOTE(woosuk): Preemption happens only when there is no available slot
309 # to keep all the sequence groups in the RUNNING state.
310 # In this case, the policy is responsible for deciding which sequence
311 # groups to preempt.
323 - self.running = self.policy.sort_by_priority(now, self.running)
312 + now = time.time()
313 + running_queue = policy.sort_by_priority(now, running_queue)
314
325 - # Reserve new token slots for the running sequence groups.
326 - running: Deque[SequenceGroup] = deque()
327 - preempted: List[SequenceGroup] = []
328 - while self.running:
329 - seq_group = self.running.popleft()
315 + while running_queue:
316 + # NOTE: running
317 + seq_group = running_queue[0]
318 + num_running_tokens = (
319 + seq_group.num_seqs(status=SequenceStatus.RUNNING) *
320 + self.num_decoding_tokens_per_seq)
321 + num_running_seqs = seq_group.get_max_num_running_seqs()
322 +
323 + running_queue.popleft()
324 while not self._can_append_slots(seq_group):
331 - if self.running:
325 + # Increase the budget as requests are preempted.
326 + budget.num_batched_tokens -= num_running_tokens
327 + budget.num_curr_seqs -= num_running_seqs
328 + if curr_loras is not None and seq_group.lora_int_id > 0:
329 + curr_loras.pop(seq_group.lora_int_id)
330 +
331 + if running_queue:
332 # Preempt the lowest-priority sequence groups.
333 - victim_seq_group = self.running.pop()
334 - self._preempt(victim_seq_group, blocks_to_swap_out)
335 - preempted.append(victim_seq_group)
333 + victim_seq_group = running_queue.pop()
334 + preempted_mode = self._preempt(victim_seq_group,
335 + blocks_to_swap_out)
336 + if preempted_mode == PreemptionMode.RECOMPUTE:
337 + preempted.append(victim_seq_group)
338 + else:
339 + swapped_out.append(victim_seq_group)
340 else:
341 # No other sequence groups can be preempted.
342 # Preempt the current sequence group.
339 - self._preempt(seq_group, blocks_to_swap_out)
340 - preempted.append(seq_group)
343 + preempted_mode = self._preempt(seq_group,
344 + blocks_to_swap_out)
345 + if preempted_mode == PreemptionMode.RECOMPUTE:
346 + preempted.append(seq_group)
347 + else:
348 + swapped_out.append(seq_group)
349 break
350 else:
343 - # Append new slots to the sequence group.
351 + logger.debug(f"append slot for {seq_group}")
352 self._append_slots(seq_group, blocks_to_copy)
345 - running.append(seq_group)
346 - self.running = running
347 -
348 - # Swap in the sequence groups in the SWAPPED state if possible.
349 - self.swapped = self.policy.sort_by_priority(now, self.swapped)
350 - if not preempted:
351 - num_curr_seqs = sum(seq_group.get_max_num_running_seqs()
352 - for seq_group in self.running)
353 - curr_loras = set(
354 - seq_group.lora_int_id
355 - for seq_group in self.running) if self.lora_enabled else None
356 -
357 - leftover_swapped = deque()
358 -
359 - while self.swapped:
360 - seq_group = self.swapped[0]
361 - lora_int_id = 0
362 - if self.lora_enabled:
363 - lora_int_id = seq_group.lora_int_id
364 - if (lora_int_id > 0 and lora_int_id not in curr_loras
365 - and len(curr_loras) >= self.lora_config.max_loras):
366 - # We don't have a space for another LoRA, so
367 - # we ignore this request for now.
368 - leftover_swapped.appendleft(seq_group)
369 - self.swapped.popleft()
370 - continue
353 + seq_groups.append(
354 + ScheduledSequenceGroup(seq_group=seq_group,
355 + token_chunk_size=1))
356 + # Make sure all queues are updated.
357 + assert len(running_queue) == 0
358 +
359 + return running_queue, SchedulerDecodeOutputs(
360 + seq_groups=seq_groups,
361 + preempted=preempted,
362 + swapped_out=swapped_out,
363 + blocks_to_swap_out=blocks_to_swap_out,
364 + blocks_to_copy=blocks_to_copy,
365 + num_lookahead_slots=self._get_num_lookahead_slots(
366 + is_prefill=False))
367
372 - # If the sequence group cannot be swapped in, stop.
373 - if not self._can_swap_in(seq_group):
374 - break
368 + def _schedule_swapped(
369 + self,
370 + swapped_queue: deque,
371 + budget: SchedulingBudget,
372 + curr_loras: Optional[Set[int]],
373 + policy: Policy,
374 + ) -> Tuple[deque, SchedulerSwappedInOutputs]:
375 + """Schedule sequence groups that are swapped out.
376
376 - # The total number of sequences in the RUNNING state should not
377 - # exceed the maximum number of sequences.
378 - num_new_seqs = seq_group.get_max_num_running_seqs()
379 - if (num_curr_seqs + num_new_seqs >
380 - self.scheduler_config.max_num_seqs):
381 - break
377 + It schedules swapped requests as long as it fits `budget` and
378 + curr_loras <= max_lora from the scheduling config. The input arguments
379 + `budget` and `curr_loras` are updated based on scheduled seq_groups.
380
383 - if lora_int_id > 0:
384 - curr_loras.add(lora_int_id)
385 - self.swapped.popleft()
386 - self._swap_in(seq_group, blocks_to_swap_in)
387 - self._append_slots(seq_group, blocks_to_copy)
388 - num_curr_seqs += num_new_seqs
389 - self.running.append(seq_group)
390 -
391 - self.swapped.extendleft(leftover_swapped)
392 -
393 - # Each sequence in the generation phase only takes one token slot.
394 - # Therefore, the number of batched tokens is equal to the number of
395 - # sequences in the RUNNING state.
396 - num_batched_tokens = sum(
397 - seq_group.num_seqs(status=SequenceStatus.RUNNING)
398 - for seq_group in self.running)
399 -
400 - scheduler_outputs = SchedulerOutputs(
401 - scheduled_seq_groups=[
402 - ScheduledSequenceGroup(seq_group=running_group,
403 - token_chunk_size=1)
404 - for running_group in self.running
405 - ],
406 - prompt_run=False,
407 - num_batched_tokens=num_batched_tokens,
381 + Args:
382 + swapped_queue: The queue that contains swapped out requests.
383 + The given arguments are NOT in-place modified.
384 + budget: The scheduling budget. The argument is in-place updated
385 + when any requests are swapped in.
386 + curr_loras: Currently batched lora request ids. The argument is
387 + in-place updated when any requests are swapped in.
388 + policy: The sorting policy to sort swapped_queue.
389 +
390 + Returns:
391 + A tuple of remaining swapped_queue after scheduling and
392 + SchedulerSwappedInOutputs.
393 + """
394 + # Blocks that need to be swapped or copied before model execution.
395 + blocks_to_swap_in: Dict[int, int] = {}
396 + blocks_to_copy: Dict[int, List[int]] = {}
397 + seq_groups: List[ScheduledSequenceGroup] = []
398 + now = time.time()
399 + swapped_queue = policy.sort_by_priority(now, swapped_queue)
400 +
401 + leftover_swapped = deque()
402 + while swapped_queue:
403 + seq_group = swapped_queue[0]
404 +
405 + # If the sequence group cannot be swapped in, stop.
406 + if not self.block_manager.can_swap_in(seq_group):
407 + break
408 +
409 + lora_int_id = 0
410 + if self.lora_enabled:
411 + lora_int_id = seq_group.lora_int_id
412 + if (lora_int_id > 0 and lora_int_id not in curr_loras
413 + and len(curr_loras) >= self.lora_config.max_loras):
414 + # We don't have a space for another LoRA, so
415 + # we ignore this request for now.
416 + leftover_swapped.appendleft(seq_group)
417 + swapped_queue.popleft()
418 + continue
419 +
420 + # The total number of sequences in the RUNNING state should not
421 + # exceed the maximum number of sequences.
422 + num_new_seqs = seq_group.get_max_num_running_seqs()
423 + num_new_tokens = (
424 + seq_group.num_seqs(status=SequenceStatus.SWAPPED) *
425 + self.num_decoding_tokens_per_seq)
426 +
427 + if not budget.can_schedule(num_new_tokens=num_new_tokens,
428 + num_new_seqs=num_new_seqs):
429 + break
430 +
431 + if lora_int_id > 0 and curr_loras is not None:
432 + curr_loras.add(lora_int_id)
433 + swapped_queue.popleft()
434 + self._swap_in(seq_group, blocks_to_swap_in)
435 + self._append_slots(seq_group, blocks_to_copy)
436 + seq_groups.append(
437 + ScheduledSequenceGroup(seq_group, token_chunk_size=1))
438 + budget.num_batched_tokens += num_new_tokens
439 + budget.num_curr_seqs += num_new_seqs
440 +
441 + swapped_queue.extendleft(leftover_swapped)
442 +
443 + return swapped_queue, SchedulerSwappedInOutputs(
444 + seq_groups=seq_groups,
445 blocks_to_swap_in=blocks_to_swap_in,
409 - blocks_to_swap_out=blocks_to_swap_out,
446 blocks_to_copy=blocks_to_copy,
411 - ignored_seq_groups=[],
447 num_lookahead_slots=self._get_num_lookahead_slots(
413 - is_prefill=False),
448 + is_prefill=False))
449 +
450 + def _schedule_prefills(
451 + self,
452 + waiting_queue: deque,
453 + budget: SchedulingBudget,
454 + curr_loras: Optional[Set[int]],
455 + ) -> Tuple[deque, SchedulerPrefillOutputs]:
456 + """Schedule sequence groups that are in prefill stage.
457 +
458 + Note that the current scheduler treats PREEMPTED_FOR_RECOMPUTE
459 + as a new prefill (that starts from beginning -> most recently generated
460 + tokens).
461 +
462 + It schedules waiting requests as long as it fits `budget` and
463 + curr_loras <= max_lora from the scheduling config. The input arguments
464 + `budget` and `curr_loras` are updated based on scheduled seq_groups.
465 +
466 + Args:
467 + waiting_queue: The queue that contains prefill requests.
468 + The given arguments are NOT in-place modified.
469 + budget: The scheduling budget. The argument is in-place updated
470 + when any requests are scheduled.
471 + curr_loras: Currently batched lora request ids. The argument is
472 + in-place updated when any requests are scheduled.
473 +
474 + Returns:
475 + A tuple of remaining waiting_queue after scheduling and
476 + SchedulerSwappedInOutputs.
477 + """
478 + ignored_seq_groups: List[SequenceGroup] = []
479 + seq_groups: List[SequenceGroup] = []
480 + # We don't sort waiting queue because we assume it is sorted.
481 + # Copy the queue so that the input queue is not modified.
482 + waiting_queue = deque([s for s in waiting_queue])
483 +
484 + leftover_waiting_sequences = deque()
485 + while self._passed_delay(time.time()) and waiting_queue:
486 + seq_group = waiting_queue[0]
487 +
488 + waiting_seqs = seq_group.get_seqs(status=SequenceStatus.WAITING)
489 + assert len(waiting_seqs) == 1, (
490 + "Waiting sequence group should have only one prompt "
491 + "sequence.")
492 +
493 + num_prompt_tokens = waiting_seqs[0].get_len()
494 + if num_prompt_tokens > self.prompt_limit:
495 + logger.warning(
496 + f"Input prompt ({num_prompt_tokens} tokens) is too long"
497 + f" and exceeds limit of {self.prompt_limit}")
498 + for seq in waiting_seqs:
499 + seq.status = SequenceStatus.FINISHED_IGNORED
500 + ignored_seq_groups.append(seq_group)
501 + waiting_queue.popleft()
502 + continue
503 +
504 + # If the sequence group cannot be allocated, stop.
505 + can_allocate = self.block_manager.can_allocate(seq_group)
506 + if can_allocate == AllocStatus.LATER:
507 + break
508 + elif can_allocate == AllocStatus.NEVER:
509 + logger.warning(
510 + f"Input prompt ({num_prompt_tokens} tokens) is too long"
511 + f" and exceeds the capacity of block_manager")
512 + for seq in waiting_seqs:
513 + seq.status = SequenceStatus.FINISHED_IGNORED
514 + ignored_seq_groups.append(seq_group)
515 + waiting_queue.popleft()
516 + continue
517 +
518 + lora_int_id = 0
519 + if self.lora_enabled:
520 + lora_int_id = seq_group.lora_int_id
521 + if (self.lora_enabled and lora_int_id > 0
522 + and lora_int_id not in curr_loras
523 + and len(curr_loras) >= self.lora_config.max_loras):
524 + # We don't have a space for another LoRA, so
525 + # we ignore this request for now.
526 + leftover_waiting_sequences.appendleft(seq_group)
527 + waiting_queue.popleft()
528 + continue
529 +
530 + num_new_seqs = seq_group.get_max_num_running_seqs()
531 + if not budget.can_schedule(num_new_tokens=num_prompt_tokens,
532 + num_new_seqs=num_new_seqs):
533 + break
534 +
535 + # Can schedule this request.
536 + if curr_loras is not None and lora_int_id > 0:
537 + curr_loras.add(lora_int_id)
538 + waiting_queue.popleft()
539 + self._allocate_and_set_running(seq_group)
540 + seq_groups.append(
541 + ScheduledSequenceGroup(seq_group=seq_group,
542 + token_chunk_size=num_prompt_tokens))
543 + budget.num_batched_tokens += num_prompt_tokens
544 + budget.num_curr_seqs += num_new_seqs
545 +
546 + # Queue requests that couldn't be scheduled.
547 + waiting_queue.extendleft(leftover_waiting_sequences)
548 + if len(seq_groups) > 0:
549 + self.prev_prompt = True
550 +
551 + return waiting_queue, SchedulerPrefillOutputs(
552 + seq_groups=seq_groups,
553 + ignored_seq_groups=ignored_seq_groups,
554 + num_lookahead_slots=self._get_num_lookahead_slots(is_prefill=True))
555 +
556 + def _schedule(self) -> SchedulerOutputs:
557 + """Batch requests that are queued..
558 +
559 + The current policy is designed to opimimize the throughput. First,
560 + it batches as many prefill requests as possible. And it schedules
561 + decodes. If there's a pressure on GPU memory, decode requests can
562 + be swapped or preempted.
563 + """
564 + # Include running requests to the budget.
565 + budget = SchedulingBudget(
566 + num_batched_tokens=sum(
567 + seq_group.num_seqs(status=SequenceStatus.RUNNING)
568 + for seq_group in self.running),
569 + num_curr_seqs=sum(seq_group.get_max_num_running_seqs()
570 + for seq_group in self.running),
571 + token_budget=self.scheduler_config.max_num_batched_tokens,
572 + max_num_seqs=self.scheduler_config.max_num_seqs,
573 + )
574 + curr_loras = set(
575 + seq_group.lora_int_id
576 + for seq_group in self.running) if self.lora_enabled else None
577 +
578 + remaining_waiting, prefills = (self.waiting,
579 + SchedulerPrefillOutputs.create_empty())
580 + remaining_running, decodes = (self.running,
581 + SchedulerDecodeOutputs.create_empty())
582 + remaining_swapped, swapped_in = (
583 + self.swapped, SchedulerSwappedInOutputs.create_empty())
584 +
585 + # If any requests are swapped, prioritized swapped requests.
586 + if not self.swapped:
587 + remaining_waiting, prefills = self._schedule_prefills(
588 + self.waiting, budget, curr_loras)
589 +
590 + # Don't schedule decodes if prefills are scheduled.
591 + if len(prefills.seq_groups) == 0:
592 + remaining_running, decodes = self._schedule_decodes(
593 + self.running, budget, curr_loras, self.policy)
594 + # If any sequence group is preempted, do not swap in any sequence
595 + # group. because it means there's no slot for new running requests.
596 + if len(decodes.preempted) + len(decodes.swapped_out) == 0:
597 + remaining_swapped, swapped_in = self._schedule_swapped(
598 + self.swapped, budget, curr_loras, self.policy)
599 +
600 + assert (budget.num_batched_tokens <=
601 + self.scheduler_config.max_num_batched_tokens)
602 + assert budget.num_curr_seqs <= self.scheduler_config.max_num_seqs
603 +
604 + # Update waiting requests.
605 + self.waiting = remaining_waiting
606 + self.waiting.extendleft(decodes.preempted)
607 + # Update new running requests.
608 + self.running = remaining_running
609 + self.running.extend([s.seq_group for s in prefills.seq_groups])
610 + self.running.extend([s.seq_group for s in decodes.seq_groups])
611 + self.running.extend([s.seq_group for s in swapped_in.seq_groups])
612 + # Update swapped requests.
613 + self.swapped = remaining_swapped
614 + self.swapped.extend(decodes.swapped_out)
615 +
616 + return SchedulerOutputs(
617 + scheduled_seq_groups=prefills.seq_groups + decodes.seq_groups +
618 + swapped_in.seq_groups,
619 + num_prefill_groups=len(prefills.seq_groups),
620 + num_batched_tokens=budget.num_batched_tokens,
621 + blocks_to_swap_in=swapped_in.blocks_to_swap_in,
622 + blocks_to_swap_out=decodes.blocks_to_swap_out,
623 + blocks_to_copy=merge_dicts(decodes.blocks_to_copy,
624 + swapped_in.blocks_to_copy),
625 + ignored_seq_groups=prefills.ignored_seq_groups,
626 + num_lookahead_slots=(prefills.num_lookahead_slots +
627 + decodes.num_lookahead_slots +
628 + swapped_in.num_lookahead_slots),
629 )
415 - return scheduler_outputs
630
631 def _can_append_slots(self, seq_group: SequenceGroup) -> bool:
632 """Determine whether or not we have enough space in the KV cache to
@@ -444,7 +658,8 @@ class Scheduler:
658
659 # Create input data structures.
660 seq_group_metadata_list: List[SequenceGroupMetadata] = []
447 - for scheduled_seq_group in scheduler_outputs.scheduled_seq_groups:
661 + for i, scheduled_seq_group in enumerate(
662 + scheduler_outputs.scheduled_seq_groups):
663 seq_group = scheduled_seq_group.seq_group
664 token_chunk_size = scheduled_seq_group.token_chunk_size
665 seq_group.maybe_set_first_scheduled_time(now)
@@ -464,9 +679,12 @@ class Scheduler:
679 self.block_manager.get_common_computed_block_ids(
680 seq_group.get_seqs(status=SequenceStatus.RUNNING)))
681
682 + # It assumes the scheduled_seq_groups is ordered by
683 + # prefill < decoding.
684 + is_prompt = i < scheduler_outputs.num_prefill_groups
685 seq_group_metadata = SequenceGroupMetadata(
686 request_id=seq_group.request_id,
469 - is_prompt=scheduler_outputs.prompt_run,
687 + is_prompt=is_prompt,
688 seq_data=seq_data,
689 sampling_params=seq_group.sampling_params,
690 block_tables=block_tables,
@@ -479,7 +697,7 @@ class Scheduler:
697 # the subsequent comms can still use delta, but
698 # `multi_modal_data` will be None.
699 multi_modal_data=seq_group.multi_modal_data
482 - if scheduler_outputs.prompt_run else None,
700 + if scheduler_outputs.num_prefill_groups > 0 else None,
701 )
702 seq_group_metadata_list.append(seq_group_metadata)
703
@@ -504,7 +722,7 @@ class Scheduler:
722 self.running = deque(seq_group for seq_group in self.running
723 if not seq_group.is_finished())
724
507 - def _allocate(self, seq_group: SequenceGroup) -> None:
725 + def _allocate_and_set_running(self, seq_group: SequenceGroup) -> None:
726 self.block_manager.allocate(seq_group)
727 for seq in seq_group.get_seqs(status=SequenceStatus.WAITING):
728 seq.status = SequenceStatus.RUNNING
@@ -539,7 +757,7 @@ class Scheduler:
757 seq_group: SequenceGroup,
758 blocks_to_swap_out: Dict[int, int],
759 preemption_mode: Optional[PreemptionMode] = None,
542 - ) -> None:
760 + ) -> PreemptionMode:
761 # If preemption mode is not specified, we determine the mode as follows:
762 # We use recomputation by default since it incurs lower overhead than
763 # swapping. However, when the sequence group has multiple sequences
@@ -562,6 +780,7 @@ class Scheduler:
780 self._preempt_by_swap(seq_group, blocks_to_swap_out)
781 else:
782 raise AssertionError("Invalid preemption mode.")
783 + return preemption_mode
784
785 def _preempt_by_recompute(
786 self,
@@ -573,9 +792,6 @@ class Scheduler:
792 seq.status = SequenceStatus.WAITING
793 self.free_seq(seq)
794 seq.reset_state_for_recompute()
576 - # NOTE: For FCFS, we insert the preempted sequence group to the front
577 - # of the waiting queue.
578 - self.waiting.appendleft(seq_group)
795
796 def _preempt_by_swap(
797 self,
@@ -583,7 +799,6 @@ class Scheduler:
799 blocks_to_swap_out: Dict[int, int],
800 ) -> None:
801 self._swap_out(seq_group, blocks_to_swap_out)
586 - self.swapped.append(seq_group)
802
803 def _swap_in(
804 self,
vllm/engine/llm_engine.py
+1 -1
@@ -728,7 +728,7 @@ class LLMEngine:
728 time_per_output_tokens = []
729 time_e2e_requests = []
730 if scheduler_outputs is not None:
731 - prompt_run = scheduler_outputs.prompt_run
731 + prompt_run = scheduler_outputs.num_prefill_groups > 0
732
733 # Number of Tokens.
734 if prompt_run:
vllm/utils.py
+18 -1
@@ -6,7 +6,7 @@ import socket
6 import subprocess
7 import uuid
8 import warnings
9 -from collections import OrderedDict
9 +from collections import OrderedDict, defaultdict
10 from functools import lru_cache, partial
11 from platform import uname
12 from typing import (Any, Awaitable, Callable, Generic, Hashable, List,
@@ -450,3 +450,20 @@ def maybe_expand_dim(tensor: torch.Tensor,
450 if tensor.ndim < target_dims:
451 tensor = tensor.view(-1, *([size] * (target_dims - tensor.ndim)))
452 return tensor
453 +
454 +
455 +def merge_dicts(dict1: dict[Any, list[Any]],
456 + dict2: dict[Any, list[Any]]) -> dict[Any, list[Any]]:
457 + """Merge 2 dicts that have key -> List of items.
458 +
459 + When a key conflicts, the values in dict1 is prioritized.
460 + """
461 + merged_dict = defaultdict(list)
462 +
463 + for key, value in dict1.items():
464 + merged_dict[key].extend(value)
465 +
466 + for key, value in dict2.items():
467 + merged_dict[key].extend(value)
468 +
469 + return dict(merged_dict)