@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8db957ee3a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[bugfix] fixed parameter “n” when set parameter “bestof” > 1 (#10854)

Signed-off-by: jianzheng <57654625+o2363286@users.noreply.github.com>

jianzheng committed Dec 4, 2024 at 16:48 UTC 8db957ee3a8234574430d9e570e520501d8539e9
1 file changed +3 -2
vllm/sampling_params.py
+3 -2
@@ -293,8 +293,9 @@ class SamplingParams(
293 raise ValueError(
294 f"best_of must be greater than or equal to n, "
295 f"got n={self.n} and best_of={self.best_of}.")
296 - self._real_n = self.n
297 - self.n = self.best_of
296 + if not self._real_n:
297 + self._real_n = self.n
298 + self.n = self.best_of
299
300 if 0 < self.temperature < _MAX_TEMP:
301 logger.warning(