@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7b261092de
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[BUGFIX]: top_k is expected to be an integer. (#7227)

Atilla Akkuş committed Aug 7, 2024 at 10:32 UTC 7b261092de3b008f7a2c218e338f4f8a025c93ee
1 file changed +3
vllm/sampling_params.py
+3
@@ -224,6 +224,9 @@ class SamplingParams:
224 if self.top_k < -1 or self.top_k == 0:
225 raise ValueError(f"top_k must be -1 (disable), or at least 1, "
226 f"got {self.top_k}.")
227 + if not isinstance(self.top_k, int):
228 + raise TypeError(
229 + f"top_k must be an integer, got {type(self.top_k).__name__}")
230 if not 0.0 <= self.min_p <= 1.0:
231 raise ValueError("min_p must be in [0, 1], got "
232 f"{self.min_p}.")