A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[Fix] Fix best_of behavior when n=1 (#3298)
Nick Hill committed
Mar 10, 2024 at 19:17 UTC
4b59f00e917679337169c88c981f268e6ab96cd6
1 file changed
+3
-3
vllm/outputs.py
+3
-3
@@ -87,12 +87,12 @@ class RequestOutput:
87
88
@classmethod
89
def from_seq_group(cls, seq_group: SequenceGroup) -> "RequestOutput":
90
- # Get the top-n sequences.
91
- n = seq_group.sampling_params.n
90
seqs = seq_group.get_seqs()
93
- if n == 1:
91
+ if len(seqs) == 1:
92
top_n_seqs = seqs
93
else:
94
+ # Get the top-n sequences.
95
+ n = seq_group.sampling_params.n
96
if seq_group.sampling_params.use_beam_search:
97
sorting_key = lambda seq: seq.get_beam_search_score(
98
seq_group.sampling_params.length_penalty)