@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 480b7f40cf
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Misc] Improve validation errors around best_of and n (#9167)

Signed-off-by: Travis Johnson <tsjohnso@us.ibm.com>

Travis Johnson committed Oct 8, 2024 at 22:54 UTC 480b7f40cfa9a900e03ea4e825abc1a46b5d085b
1 file changed +7 -4
vllm/sampling_params.py
+7 -4
@@ -330,8 +330,8 @@ class SamplingParams(
330 if self.n < 1:
331 raise ValueError(f"n must be at least 1, got {self.n}.")
332 if not isinstance(self.best_of, int):
333 - raise ValueError(f'best_of must be an int, but is of '
334 - f'type {type(self.best_of)}')
333 + raise ValueError(f"best_of must be an int, but is of "
334 + f"type {type(self.best_of)}")
335 if self.best_of < self.n:
336 raise ValueError(f"best_of must be greater than or equal to n, "
337 f"got n={self.n} and best_of={self.best_of}.")
@@ -390,10 +390,13 @@ class SamplingParams(
390 raise ValueError("best_of must equal n to use output_kind=DELTA")
391
392 def _verify_greedy_sampling(self) -> None:
393 + if self.n > 1:
394 + raise ValueError("n must be 1 when using greedy sampling, "
395 + f"got {self.n}.")
396 assert isinstance(self.best_of, int)
397 if self.best_of > 1:
395 - raise ValueError("best_of must be 1 when using greedy sampling."
396 - f"Got {self.best_of}.")
398 + raise ValueError("best_of must be 1 when using greedy sampling, "
399 + f"got {self.best_of}.")
400
401 def update_from_generation_config(
402 self,