A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Add decoding params
Woosuk Kwon committed
Feb 9, 2023 at 11:27 UTC
3c2b47fcae9cd60c03d599462380e27c8195b15e
1 file changed
+30
cacheflow/decoding.py
new
+30
@@ -0,0 +1,30 @@
1
+from typing import List
2
+
3
+
4
+class DecodingParams:
5
+
6
+ def __init__(
7
+ self,
8
+ n: int = 1,
9
+ temperature: float = 1.0,
10
+ top_p: float = 1.0,
11
+ use_beam_search: bool = False,
12
+ stop_token_ids: List[int] = [],
13
+ ) -> None:
14
+ assert n >= 1
15
+ assert temperature >= 0.0
16
+ assert 0.0 < top_p <= 1.0
17
+ if use_beam_search:
18
+ assert n > 1
19
+ assert temperature > 0.0
20
+ assert top_p == 1.0
21
+ elif temperature == 0.0:
22
+ # Zero temperature means greedy decoding.
23
+ assert n == 1
24
+ assert top_p == 1.0
25
+
26
+ self.n = n
27
+ self.temperature = temperature
28
+ self.top_p = top_p
29
+ self.use_beam_search = use_beam_search
30
+ self.stop_token_ids = stop_token_ids