A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Add get_len
Woosuk Kwon committed
Feb 23, 2023 at 05:58 UTC
af16c050742979475fa8fe842e9916eb389d872a
1 file changed
+3
cacheflow/sequence.py
+3
@@ -49,6 +49,9 @@ class Sequence:
49
last_block.append(token_ids[:num_empty_slots])
50
token_ids = token_ids[num_empty_slots:]
51
52
+ def get_len(self) -> int:
53
+ return sum(block.num_tokens for block in self.logical_token_blocks)
54
+
55
def get_token_ids(self) -> List[int]:
56
token_ids: List[int] = []
57
for block in self.logical_token_blocks: