A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Misc] Fix typos in scheduler.py (#7285)
Signed-off-by: Rui Qiao <ruisearch42@gmail.com>
Rui Qiao committed
Aug 7, 2024 at 17:06 UTC
746709642c81aa22926765aef67e086a15aef076
1 file changed
+3
-3
vllm/core/scheduler.py
+3
-3
@@ -678,7 +678,7 @@ class Scheduler:
678
all tokens.
679
680
Returns:
681
- SchedulerSwappedInOutputs.
681
+ SchedulerPrefillOutputs.
682
"""
683
ignored_seq_groups: List[SequenceGroup] = []
684
seq_groups: List[SequenceGroup] = []
@@ -851,7 +851,7 @@ class Scheduler:
851
preempted=preempted,
852
)
853
854
- def _schedule_chunked_prefill(self):
854
+ def _schedule_chunked_prefill(self) -> SchedulerOutputs:
855
"""Schedule queued requests.
856
857
Chunked prefill allows to chunk prefill requests, batch them together
@@ -862,7 +862,7 @@ class Scheduler:
862
863
The policy can sustain the high GPU utilization because it can put
864
prefill and decodes requests to the same batch, while it improves
865
- inter token latency because decodes requests don't need to blocked
865
+ inter token latency because decodes requests don't need to be blocked
866
by prefill requests.
867
"""
868
budget = SchedulingBudget(