A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Bugfix] Fix single output condition in output processor (#7881)
Woosuk Kwon committed
Sep 2, 2024 at 20:35 UTC
0fbc6696c28f41009d8493c57e74f5971d6f5026
1 file changed
+1
-1
vllm/engine/output_processor/single_step.py
+1
-1
@@ -113,7 +113,7 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
113
outputs: SequenceGroupOutput,
114
is_async: bool) -> None:
115
sampling_params = seq_group.sampling_params
116
- if sampling_params.n == 1 and not sampling_params.use_beam_search:
116
+ if sampling_params.best_of == 1 and not sampling_params.use_beam_search:
117
# only have one output sample
118
sample = outputs.samples[0]
119
# only have one sequence