@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fa45513a51
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[misc] fix comment and variable name (#9139)

youkaichao committed Oct 7, 2024 at 16:07 UTC fa45513a5189b3a9f73a59730c9ac65d061e1311
1 file changed +4 -3
vllm/core/scheduler.py
+4 -3
@@ -1202,10 +1202,11 @@ class Scheduler:
1202 seq_group=seq_group, num_lookahead_slots=num_lookahead_slots)
1203
1204 def _allow_async_output_proc(self, seq_group: SequenceGroup) -> bool:
1205 - # TODO: does it work with parallel sampling?
1206 - no_beam_search = seq_group.sampling_params is None or (
1205 + # async_output_proc is allowed only when we have a single sequence
1206 + # in the sequence group
1207 + no_single_seq = seq_group.sampling_params is None or (
1208 seq_group.sampling_params.best_of == 1)
1208 - return no_beam_search
1209 + return no_single_seq
1210
1211 def schedule(
1212 self