A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Set top_p=0 and top_k=-1 in greedy sampling (#1748)
ljss committed
Nov 23, 2023 at 04:51 UTC
4cea74c73b2e0981aadfefb3a00e8186d065c897
1 file changed
+2
-4
vllm/sampling_params.py
+2
-4
@@ -147,6 +147,8 @@ class SamplingParams:
147
self._verify_non_beam_search()
148
if self.temperature < _SAMPLING_EPS:
149
# Zero temperature means greedy sampling.
150
+ self.top_p = 1.0
151
+ self.top_k = -1
152
self._verify_greedy_sampling()
153
154
def _verify_args(self) -> None:
@@ -214,10 +216,6 @@ class SamplingParams:
216
if self.best_of > 1:
217
raise ValueError("best_of must be 1 when using greedy sampling."
218
f"Got {self.best_of}.")
217
- if self.top_p < 1.0 - _SAMPLING_EPS:
218
- raise ValueError("top_p must be 1 when using greedy sampling.")
219
- if self.top_k != -1:
220
- raise ValueError("top_k must be -1 when using greedy sampling.")
219
220
@cached_property
221
def sampling_type(self) -> SamplingType: