@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 050f285ff6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Core] Scheduling optimization 2 (#4280)

SangBin Cho committed Apr 23, 2024 at 17:02 UTC 050f285ff6e7bbe898ee751770b2571972166bef
3 files changed +15 -3
tests/core/test_scheduler.py
+2 -1
@@ -563,7 +563,8 @@ def test_decode_schedule_preempted():
563 assert len(output.preempted) == 2
564 # Verify budgets are updated.
565 assert budget.num_batched_tokens == 1
566 - assert budget.num_curr_seqs == 1
566 + # NOTE: When enable_chunk is False, num_seqs budget is not updated.
567 + # assert budget.num_curr_seqs == 1
568 # Both should be preempted, not swapped.
569 assert output.blocks_to_swap_out == {}
570 # Nothing is copied.
vllm/core/scheduler.py
+8 -2
@@ -395,12 +395,12 @@ class Scheduler:
395 # We can have up to 1 running prefill at any given time in running
396 # queue, which means we can guarantee chunk size is at least 1.
397 assert num_running_tokens != 0
398 - num_running_seqs = seq_group.get_max_num_running_seqs()
398
399 running_queue.popleft()
400 while not self._can_append_slots(seq_group):
401 budget.subtract_num_batched_tokens(seq_group.request_id,
402 num_running_tokens)
403 + num_running_seqs = seq_group.get_max_num_running_seqs()
404 budget.subtract_num_seqs(seq_group.request_id,
405 num_running_seqs)
406 if curr_loras is not None and seq_group.lora_int_id > 0:
@@ -439,7 +439,13 @@ class Scheduler:
439 token_chunk_size=1))
440 budget.add_num_batched_tokens(seq_group.request_id,
441 num_running_tokens)
442 - budget.add_num_seqs(seq_group.request_id, num_running_seqs)
442 + # OPTIMIZATION: Note that get_max_num_running_seqs is
443 + # expensive. For the default scheduling chase where
444 + # enable_chunking is False, num_seqs are updated before running
445 + # this method, so we don't have to update it again here.
446 + if enable_chunking:
447 + num_running_seqs = seq_group.get_max_num_running_seqs()
448 + budget.add_num_seqs(seq_group.request_id, num_running_seqs)
449 if curr_loras is not None and seq_group.lora_int_id > 0:
450 curr_loras.add(seq_group.lora_int_id)
451
vllm/sequence.py
+5
@@ -508,6 +508,11 @@ class SequenceGroup:
508 return num_uncomputed_tokens
509
510 def num_seqs(self, status: Optional[SequenceStatus] = None) -> int:
511 + # Optimization. We don't need to call get_seqs if we don't need to
512 + # filter by states.
513 + if status is None:
514 + return len(self.seqs_dict)
515 +
516 return len(self.get_seqs(status))
517
518 def num_unfinished_seqs(self) -> int: