@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0fbc6696c2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Bugfix] Fix single output condition in output processor (#7881)

Woosuk Kwon committed Sep 2, 2024 at 20:35 UTC 0fbc6696c28f41009d8493c57e74f5971d6f5026
1 file changed +1 -1
vllm/engine/output_processor/single_step.py
+1 -1
@@ -113,7 +113,7 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
113 outputs: SequenceGroupOutput,
114 is_async: bool) -> None:
115 sampling_params = seq_group.sampling_params
116 - if sampling_params.n == 1 and not sampling_params.use_beam_search:
116 + if sampling_params.best_of == 1 and not sampling_params.use_beam_search:
117 # only have one output sample
118 sample = outputs.samples[0]
119 # only have one sequence