@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e441bad674
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Optimization] use a pool to reuse LogicalTokenBlock.token_ids (#5584)

youkaichao committed Jun 17, 2024 at 15:08 UTC e441bad674e6dca076a145ca63f72100318c51e5
1 file changed +39 -2
vllm/block.py
+39 -2
@@ -1,5 +1,7 @@
1 """Token blocks."""
2 -from typing import List
2 +import weakref
3 +from collections import defaultdict
4 +from typing import Dict, List
5
6 from vllm.utils import Device
7
@@ -7,6 +9,35 @@ _BLANK_TOKEN_ID = -1
9
10 DEFAULT_LAST_ACCESSED_TIME = -1
11
12 +TokensBlock = List[int]
13 +
14 +
15 +class BlockPool:
16 + """A pool of physical blocks.
17 + When requests come, we create a lot of logical blocks;
18 + when requests are done, we destroy a lot of logical blocks.
19 + It turns out that creating and destroying logical blocks can be expensive,
20 + especially for the `token_ids` field, which is a list of integers.
21 + To avoid this overhead, we use a pool to manage the logical blocks.
22 + When an old request is done and a new request comes, we can reuse the
23 + logical blocks from the old request to feed the new request.
24 + """
25 +
26 + def __init__(self) -> None:
27 + # block size to list of token blocks
28 + self.pool: Dict[int, List[TokensBlock]] = defaultdict(list)
29 +
30 + def alloc_block(self, block_size: int) -> TokensBlock:
31 + if block_size in self.pool and self.pool[block_size]:
32 + return self.pool[block_size].pop()
33 + return [_BLANK_TOKEN_ID] * block_size
34 +
35 + def del_block(self, block: TokensBlock) -> None:
36 + self.pool[len(block)].append(block)
37 +
38 +
39 +_BLOCK_POOL = BlockPool()
40 +
41
42 class LogicalTokenBlock:
43 """A block that stores a contiguous chunk of tokens from left to right.
@@ -23,7 +54,13 @@ class LogicalTokenBlock:
54 self.block_number = block_number
55 self.block_size = block_size
56
26 - self.token_ids = [_BLANK_TOKEN_ID] * block_size
57 + self.token_ids = _BLOCK_POOL.alloc_block(block_size)
58 + # this finalizer is used to return the block to the pool when the object is deleted # noqa
59 + # NOTE: don't use __del__ because it cannot guarantee the order of finalization, # noqa
60 + # i.e. `self.token_ids` may be deleted before `self`, and we lose
61 + # the opportunity to return the block to the pool
62 + self._finalizer = weakref.finalize(self, _BLOCK_POOL.del_block,
63 + self.token_ids)
64 self.num_tokens = 0
65
66 def is_empty(self) -> bool: