A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Add max_num_steps to SamplingParams
Woosuk Kwon committed
Feb 24, 2023 at 11:44 UTC
46ce1356f7108f27af9ebb153ddbe02d1a3fa97d
1 file changed
+3
cacheflow/sampling_params.py
+3
@@ -10,6 +10,7 @@ class SamplingParams:
10
top_p: float = 1.0,
11
use_beam_search: bool = False,
12
stop_token_ids: Set[int] = [],
13
+ max_num_steps: int = 16, # From OpenAI API.
14
max_context_len: Optional[int] = None,
15
) -> None:
16
assert n >= 1
@@ -23,6 +24,7 @@ class SamplingParams:
24
# Zero temperature means greedy decoding.
25
assert n == 1
26
assert top_p == 1.0
27
+ assert max_num_steps >= 1
28
assert max_context_len is None or max_context_len >= 0
29
30
self.n = n
@@ -30,4 +32,5 @@ class SamplingParams:
32
self.top_p = top_p
33
self.use_beam_search = use_beam_search
34
self.stop_token_ids = stop_token_ids
35
+ self.max_num_steps = max_num_steps
36
self.max_context_len = max_context_len