A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Core] Scheduling optimization 2 (#4280)
SangBin Cho committed
Apr 23, 2024 at 17:02 UTC
050f285ff6e7bbe898ee751770b2571972166bef
3 files changed
+15
-3
tests/core/test_scheduler.py
+2
-1
@@ -563,7 +563,8 @@ def test_decode_schedule_preempted():
563
assert len(output.preempted) == 2
564
# Verify budgets are updated.
565
assert budget.num_batched_tokens == 1
566
- assert budget.num_curr_seqs == 1
566
+ # NOTE: When enable_chunk is False, num_seqs budget is not updated.
567
+ # assert budget.num_curr_seqs == 1
568
# Both should be preempted, not swapped.
569
assert output.blocks_to_swap_out == {}
570
# Nothing is copied.
vllm/core/scheduler.py
+8
-2
@@ -395,12 +395,12 @@ class Scheduler:
395
# We can have up to 1 running prefill at any given time in running
396
# queue, which means we can guarantee chunk size is at least 1.
397
assert num_running_tokens != 0
398
- num_running_seqs = seq_group.get_max_num_running_seqs()
398
399
running_queue.popleft()
400
while not self._can_append_slots(seq_group):
401
budget.subtract_num_batched_tokens(seq_group.request_id,
402
num_running_tokens)
403
+ num_running_seqs = seq_group.get_max_num_running_seqs()
404
budget.subtract_num_seqs(seq_group.request_id,
405
num_running_seqs)
406
if curr_loras is not None and seq_group.lora_int_id > 0:
@@ -439,7 +439,13 @@ class Scheduler:
439
token_chunk_size=1))
440
budget.add_num_batched_tokens(seq_group.request_id,
441
num_running_tokens)
442
- budget.add_num_seqs(seq_group.request_id, num_running_seqs)
442
+ # OPTIMIZATION: Note that get_max_num_running_seqs is
443
+ # expensive. For the default scheduling chase where
444
+ # enable_chunking is False, num_seqs are updated before running
445
+ # this method, so we don't have to update it again here.
446
+ if enable_chunking:
447
+ num_running_seqs = seq_group.get_max_num_running_seqs()
448
+ budget.add_num_seqs(seq_group.request_id, num_running_seqs)
449
if curr_loras is not None and seq_group.lora_int_id > 0:
450
curr_loras.add(seq_group.lora_int_id)
451
vllm/sequence.py
+5
@@ -508,6 +508,11 @@ class SequenceGroup:
508
return num_uncomputed_tokens
509
510
def num_seqs(self, status: Optional[SequenceStatus] = None) -> int:
511
+ # Optimization. We don't need to call get_seqs if we don't need to
512
+ # filter by states.
513
+ if status is None:
514
+ return len(self.seqs_dict)
515
+
516
return len(self.get_seqs(status))
517
518
def num_unfinished_seqs(self) -> int: