@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4b59f00e91
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[Fix] Fix best_of behavior when n=1 (#3298)

Nick Hill committed Mar 10, 2024 at 19:17 UTC 4b59f00e917679337169c88c981f268e6ab96cd6
1 file changed +3 -3
vllm/outputs.py
+3 -3
@@ -87,12 +87,12 @@ class RequestOutput:
87
88 @classmethod
89 def from_seq_group(cls, seq_group: SequenceGroup) -> "RequestOutput":
90 - # Get the top-n sequences.
91 - n = seq_group.sampling_params.n
90 seqs = seq_group.get_seqs()
93 - if n == 1:
91 + if len(seqs) == 1:
92 top_n_seqs = seqs
93 else:
94 + # Get the top-n sequences.
95 + n = seq_group.sampling_params.n
96 if seq_group.sampling_params.use_beam_search:
97 sorting_key = lambda seq: seq.get_beam_search_score(
98 seq_group.sampling_params.length_penalty)