@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4cea74c73b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Set top_p=0 and top_k=-1 in greedy sampling (#1748)

ljss committed Nov 23, 2023 at 04:51 UTC 4cea74c73b2e0981aadfefb3a00e8186d065c897
1 file changed +2 -4
vllm/sampling_params.py
+2 -4
@@ -147,6 +147,8 @@ class SamplingParams:
147 self._verify_non_beam_search()
148 if self.temperature < _SAMPLING_EPS:
149 # Zero temperature means greedy sampling.
150 + self.top_p = 1.0
151 + self.top_k = -1
152 self._verify_greedy_sampling()
153
154 def _verify_args(self) -> None:
@@ -214,10 +216,6 @@ class SamplingParams:
216 if self.best_of > 1:
217 raise ValueError("best_of must be 1 when using greedy sampling."
218 f"Got {self.best_of}.")
217 - if self.top_p < 1.0 - _SAMPLING_EPS:
218 - raise ValueError("top_p must be 1 when using greedy sampling.")
219 - if self.top_k != -1:
220 - raise ValueError("top_k must be -1 when using greedy sampling.")
219
220 @cached_property
221 def sampling_type(self) -> SamplingType: