A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Move max_context_len
Woosuk Kwon committed
Feb 23, 2023 at 04:57 UTC
d094512296ad18968efffd925c372533e9dd12e3
2 files changed
+5
-4
cacheflow/decoding.py
+5
-2
@@ -1,4 +1,4 @@
1
-from typing import List
1
+from typing import Optional, Set
2
3
4
class DecodingParams:
@@ -9,7 +9,8 @@ class DecodingParams:
9
temperature: float = 1.0,
10
top_p: float = 1.0,
11
use_beam_search: bool = False,
12
- stop_token_ids: List[int] = [],
12
+ stop_token_ids: Set[int] = [],
13
+ max_context_len: Optional[int] = None,
14
) -> None:
15
assert n >= 1
16
assert temperature >= 0.0
@@ -22,9 +23,11 @@ class DecodingParams:
23
# Zero temperature means greedy decoding.
24
assert n == 1
25
assert top_p == 1.0
26
+ assert max_context_len is None or max_context_len >= 0
27
28
self.n = n
29
self.temperature = temperature
30
self.top_p = top_p
31
self.use_beam_search = use_beam_search
32
self.stop_token_ids = stop_token_ids
33
+ self.max_context_len = max_context_len
cacheflow/sequence.py
-2
@@ -17,11 +17,9 @@ class Sequence:
17
self,
18
seq_id: int,
19
token_ids: List[int],
20
- max_context_len: int,
20
block_size: int,
21
) -> None:
22
self.seq_id = seq_id
24
- self.max_context_len = max_context_len
23
self.block_size = block_size
24
25
self.logical_token_blocks: List[LogicalTokenBlock] = []