@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3c2b47fcae
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Add decoding params

Woosuk Kwon committed Feb 9, 2023 at 11:27 UTC 3c2b47fcae9cd60c03d599462380e27c8195b15e
1 file changed +30
cacheflow/decoding.py new
+30
@@ -0,0 +1,30 @@
1 +from typing import List
2 +
3 +
4 +class DecodingParams:
5 +
6 + def __init__(
7 + self,
8 + n: int = 1,
9 + temperature: float = 1.0,
10 + top_p: float = 1.0,
11 + use_beam_search: bool = False,
12 + stop_token_ids: List[int] = [],
13 + ) -> None:
14 + assert n >= 1
15 + assert temperature >= 0.0
16 + assert 0.0 < top_p <= 1.0
17 + if use_beam_search:
18 + assert n > 1
19 + assert temperature > 0.0
20 + assert top_p == 1.0
21 + elif temperature == 0.0:
22 + # Zero temperature means greedy decoding.
23 + assert n == 1
24 + assert top_p == 1.0
25 +
26 + self.n = n
27 + self.temperature = temperature
28 + self.top_p = top_p
29 + self.use_beam_search = use_beam_search
30 + self.stop_token_ids = stop_token_ids