A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Minor code cleaning for SamplingParams (#99)
Woosuk Kwon committed
May 12, 2023 at 18:07 UTC
6208d622ca74789f329fb4e9041a600e1f96659b
1 file changed
+50
-49
cacheflow/sampling_params.py
+50
-49
@@ -1,4 +1,4 @@
1
-from typing import Dict, Set
1
+from typing import Set
2
3
4
class SamplingParams:
@@ -16,54 +16,6 @@ class SamplingParams:
16
max_tokens: int = 16,
17
logprobs: int = 0,
18
) -> None:
19
- if n < 1:
20
- raise ValueError(f"n must be at least 1, got {n}.")
21
- if not -2.0 <= presence_penalty <= 2.0:
22
- raise ValueError(
23
- f"presence_penalty must be in [-2, 2], got {presence_penalty}.")
24
- if not -2.0 <= frequency_penalty <= 2.0:
25
- raise ValueError(
26
- f"frequency_penalty must be in [-2, 2], got {frequency_penalty}.")
27
- if temperature < 0.0:
28
- raise ValueError(
29
- f"temperature must be non-negative, got {temperature}.")
30
- if not 0.0 < top_p <= 1.0:
31
- raise ValueError(f"top_p must be in (0, 1], got {top_p}.")
32
- if top_k < -1 or top_k == 0:
33
- raise ValueError(f"top_k must be -1 (disable), or at least 1, "
34
- f"got {top_k}.")
35
- if max_tokens < 1:
36
- raise ValueError(
37
- f"max_tokens must be at least 1, got {max_tokens}.")
38
- if logprobs < 0:
39
- raise ValueError(
40
- f"logprobs must be non-negative, got {logprobs}.")
41
-
42
- if use_beam_search:
43
- if n == 1:
44
- raise ValueError(
45
- "n must be greater than 1 when using beam search.")
46
- if temperature > 0.0:
47
- raise ValueError(
48
- "temperature must be 0 when using beam search.")
49
- if top_p < 1.0:
50
- raise ValueError(
51
- "top_p must be 1 when using beam search.")
52
- if top_k != -1:
53
- raise ValueError(
54
- "top_k must be -1 when using beam search.")
55
- elif temperature == 0.0:
56
- # Zero temperature means greedy sampling.
57
- if n > 1:
58
- raise ValueError(
59
- "n must be 1 when using greedy sampling.")
60
- if top_p < 1.0:
61
- raise ValueError(
62
- "top_p must be 1 when using greedy sampling.")
63
- if top_k != -1:
64
- raise ValueError(
65
- "top_k must be -1 when using greedy sampling.")
66
-
19
self.n = n
20
self.presence_penalty = presence_penalty
21
self.frequency_penalty = frequency_penalty
@@ -75,6 +27,55 @@ class SamplingParams:
27
self.max_tokens = max_tokens
28
self.logprobs = logprobs
29
30
+ self._verify_args()
31
+ if self.use_beam_search:
32
+ self._verity_beam_search()
33
+ elif self.temperature == 0.0:
34
+ # Zero temperature means greedy sampling.
35
+ self._verify_greedy_sampling()
36
+
37
+ def _verify_args(self) -> None:
38
+ if self.n < 1:
39
+ raise ValueError(f"n must be at least 1, got {self.n}.")
40
+ if not -2.0 <= self.presence_penalty <= 2.0:
41
+ raise ValueError("presence_penalty must be in [-2, 2], got "
42
+ f"{self.presence_penalty}.")
43
+ if not -2.0 <= self.frequency_penalty <= 2.0:
44
+ raise ValueError("frequency_penalty must be in [-2, 2], got "
45
+ f"{self.frequency_penalty}.")
46
+ if self.temperature < 0.0:
47
+ raise ValueError(
48
+ f"temperature must be non-negative, got {self.temperature}.")
49
+ if not 0.0 < self.top_p <= 1.0:
50
+ raise ValueError(f"top_p must be in (0, 1], got {self.top_p}.")
51
+ if self.top_k < -1 or self.top_k == 0:
52
+ raise ValueError(f"top_k must be -1 (disable), or at least 1, "
53
+ f"got {self.top_k}.")
54
+ if self.max_tokens < 1:
55
+ raise ValueError(
56
+ f"max_tokens must be at least 1, got {self.max_tokens}.")
57
+ if self.logprobs < 0:
58
+ raise ValueError(
59
+ f"logprobs must be non-negative, got {self.logprobs}.")
60
+
61
+ def _verity_beam_search(self) -> None:
62
+ if self.n == 1:
63
+ raise ValueError("n must be greater than 1 when using beam search.")
64
+ if self.temperature > 0.0:
65
+ raise ValueError("temperature must be 0 when using beam search.")
66
+ if self.top_p < 1.0:
67
+ raise ValueError("top_p must be 1 when using beam search.")
68
+ if self.top_k != -1:
69
+ raise ValueError("top_k must be -1 when using beam search.")
70
+
71
+ def _verify_greedy_sampling(self) -> None:
72
+ if self.n > 1:
73
+ raise ValueError("n must be 1 when using greedy sampling.")
74
+ if self.top_p < 1.0:
75
+ raise ValueError("top_p must be 1 when using greedy sampling.")
76
+ if self.top_k != -1:
77
+ raise ValueError("top_k must be -1 when using greedy sampling.")
78
+
79
def __repr__(self) -> str:
80
return (f"SamplingParams(n={self.n}, "
81
f"presence_penalty={self.presence_penalty}, "