@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b28298f2f4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Bugfix] Validate SamplingParam n is an int (#8548)

saumya-saran committed Sep 20, 2024 at 12:46 UTC b28298f2f4bd4ec6d1020c10b923a9eb7993dc89
1 file changed +6 -1
vllm/sampling_params.py
+6 -1
@@ -273,9 +273,14 @@ class SamplingParams(
273 self._all_stop_token_ids = set(self.stop_token_ids)
274
275 def _verify_args(self) -> None:
276 + if not isinstance(self.n, int):
277 + raise ValueError(f"n must be an int, but is of "
278 + f"type {type(self.n)}")
279 if self.n < 1:
280 raise ValueError(f"n must be at least 1, got {self.n}.")
278 - assert isinstance(self.best_of, int)
281 + if not isinstance(self.best_of, int):
282 + raise ValueError(f'best_of must be an int, but is of '
283 + f'type {type(self.best_of)}')
284 if self.best_of < self.n:
285 raise ValueError(f"best_of must be greater than or equal to n, "
286 f"got n={self.n} and best_of={self.best_of}.")