@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 46ce1356f7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Add max_num_steps to SamplingParams

Woosuk Kwon committed Feb 24, 2023 at 11:44 UTC 46ce1356f7108f27af9ebb153ddbe02d1a3fa97d
1 file changed +3
cacheflow/sampling_params.py
+3
@@ -10,6 +10,7 @@ class SamplingParams:
10 top_p: float = 1.0,
11 use_beam_search: bool = False,
12 stop_token_ids: Set[int] = [],
13 + max_num_steps: int = 16, # From OpenAI API.
14 max_context_len: Optional[int] = None,
15 ) -> None:
16 assert n >= 1
@@ -23,6 +24,7 @@ class SamplingParams:
24 # Zero temperature means greedy decoding.
25 assert n == 1
26 assert top_p == 1.0
27 + assert max_num_steps >= 1
28 assert max_context_len is None or max_context_len >= 0
29
30 self.n = n
@@ -30,4 +32,5 @@ class SamplingParams:
32 self.top_p = top_p
33 self.use_beam_search = use_beam_search
34 self.stop_token_ids = stop_token_ids
35 + self.max_num_steps = max_num_steps
36 self.max_context_len = max_context_len