@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / af16c05074
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Add get_len

Woosuk Kwon committed Feb 23, 2023 at 05:58 UTC af16c050742979475fa8fe842e9916eb389d872a
1 file changed +3
cacheflow/sequence.py
+3
@@ -49,6 +49,9 @@ class Sequence:
49 last_block.append(token_ids[:num_empty_slots])
50 token_ids = token_ids[num_empty_slots:]
51
52 + def get_len(self) -> int:
53 + return sum(block.num_tokens for block in self.logical_token_blocks)
54 +
55 def get_token_ids(self) -> List[int]:
56 token_ids: List[int] = []
57 for block in self.logical_token_blocks: