@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d094512296
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Move max_context_len

Woosuk Kwon committed Feb 23, 2023 at 04:57 UTC d094512296ad18968efffd925c372533e9dd12e3
2 files changed +5 -4
cacheflow/decoding.py
+5 -2
@@ -1,4 +1,4 @@
1 -from typing import List
1 +from typing import Optional, Set
2
3
4 class DecodingParams:
@@ -9,7 +9,8 @@ class DecodingParams:
9 temperature: float = 1.0,
10 top_p: float = 1.0,
11 use_beam_search: bool = False,
12 - stop_token_ids: List[int] = [],
12 + stop_token_ids: Set[int] = [],
13 + max_context_len: Optional[int] = None,
14 ) -> None:
15 assert n >= 1
16 assert temperature >= 0.0
@@ -22,9 +23,11 @@ class DecodingParams:
23 # Zero temperature means greedy decoding.
24 assert n == 1
25 assert top_p == 1.0
26 + assert max_context_len is None or max_context_len >= 0
27
28 self.n = n
29 self.temperature = temperature
30 self.top_p = top_p
31 self.use_beam_search = use_beam_search
32 self.stop_token_ids = stop_token_ids
33 + self.max_context_len = max_context_len
cacheflow/sequence.py
-2
@@ -17,11 +17,9 @@ class Sequence:
17 self,
18 seq_id: int,
19 token_ids: List[int],
20 - max_context_len: int,
20 block_size: int,
21 ) -> None:
22 self.seq_id = seq_id
24 - self.max_context_len = max_context_len
23 self.block_size = block_size
24
25 self.logical_token_blocks: List[LogicalTokenBlock] = []