A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[misc] fix comment and variable name (#9139)
youkaichao committed
Oct 7, 2024 at 16:07 UTC
fa45513a5189b3a9f73a59730c9ac65d061e1311
1 file changed
+4
-3
vllm/core/scheduler.py
+4
-3
@@ -1202,10 +1202,11 @@ class Scheduler:
1202
seq_group=seq_group, num_lookahead_slots=num_lookahead_slots)
1203
1204
def _allow_async_output_proc(self, seq_group: SequenceGroup) -> bool:
1205
- # TODO: does it work with parallel sampling?
1206
- no_beam_search = seq_group.sampling_params is None or (
1205
+ # async_output_proc is allowed only when we have a single sequence
1206
+ # in the sequence group
1207
+ no_single_seq = seq_group.sampling_params is None or (
1208
seq_group.sampling_params.best_of == 1)
1208
- return no_beam_search
1209
+ return no_single_seq
1210
1211
def schedule(
1212
self