A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Bugfix] Validate SamplingParam n is an int (#8548)
saumya-saran committed
Sep 20, 2024 at 12:46 UTC
b28298f2f4bd4ec6d1020c10b923a9eb7993dc89
1 file changed
+6
-1
vllm/sampling_params.py
+6
-1
@@ -273,9 +273,14 @@ class SamplingParams(
273
self._all_stop_token_ids = set(self.stop_token_ids)
274
275
def _verify_args(self) -> None:
276
+ if not isinstance(self.n, int):
277
+ raise ValueError(f"n must be an int, but is of "
278
+ f"type {type(self.n)}")
279
if self.n < 1:
280
raise ValueError(f"n must be at least 1, got {self.n}.")
278
- assert isinstance(self.best_of, int)
281
+ if not isinstance(self.best_of, int):
282
+ raise ValueError(f'best_of must be an int, but is of '
283
+ f'type {type(self.best_of)}')
284
if self.best_of < self.n:
285
raise ValueError(f"best_of must be greater than or equal to n, "
286
f"got n={self.n} and best_of={self.best_of}.")