A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Misc] Improve validation errors around best_of and n (#9167)
Signed-off-by: Travis Johnson <tsjohnso@us.ibm.com>
Travis Johnson committed
Oct 8, 2024 at 22:54 UTC
480b7f40cfa9a900e03ea4e825abc1a46b5d085b
1 file changed
+7
-4
vllm/sampling_params.py
+7
-4
@@ -330,8 +330,8 @@ class SamplingParams(
330
if self.n < 1:
331
raise ValueError(f"n must be at least 1, got {self.n}.")
332
if not isinstance(self.best_of, int):
333
- raise ValueError(f'best_of must be an int, but is of '
334
- f'type {type(self.best_of)}')
333
+ raise ValueError(f"best_of must be an int, but is of "
334
+ f"type {type(self.best_of)}")
335
if self.best_of < self.n:
336
raise ValueError(f"best_of must be greater than or equal to n, "
337
f"got n={self.n} and best_of={self.best_of}.")
@@ -390,10 +390,13 @@ class SamplingParams(
390
raise ValueError("best_of must equal n to use output_kind=DELTA")
391
392
def _verify_greedy_sampling(self) -> None:
393
+ if self.n > 1:
394
+ raise ValueError("n must be 1 when using greedy sampling, "
395
+ f"got {self.n}.")
396
assert isinstance(self.best_of, int)
397
if self.best_of > 1:
395
- raise ValueError("best_of must be 1 when using greedy sampling."
396
- f"Got {self.best_of}.")
398
+ raise ValueError("best_of must be 1 when using greedy sampling, "
399
+ f"got {self.best_of}.")
400
401
def update_from_generation_config(
402
self,