A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Misc] Remove unused vllm/block.py (#11336)
yangzhibin committed
Dec 20, 2024 at 01:54 UTC
e461c262f0d4c9911f1bf75bea723f8ae17219be
2 files changed
+2
-90
vllm/block.py
deleted
-88
@@ -1,88 +0,0 @@
1
-"""Token blocks."""
2
-from typing import TYPE_CHECKING, Iterator, List, Optional
3
-
4
-from vllm.utils import Device
5
-
6
-DEFAULT_LAST_ACCESSED_TIME: float = -1
7
-
8
-
9
-class PhysicalTokenBlock:
10
- """Represents the state of a block in the KV cache."""
11
-
12
- def __init__(
13
- self,
14
- device: Device,
15
- block_number: int,
16
- block_size: int,
17
- block_hash: int,
18
- num_hashed_tokens: int,
19
- ) -> None:
20
- self.device = device
21
- self.block_number = block_number
22
- self.block_size = block_size
23
- self.block_hash = block_hash
24
- self.num_hashed_tokens = num_hashed_tokens
25
-
26
- self.ref_count = 0
27
- self.last_accessed = DEFAULT_LAST_ACCESSED_TIME
28
-
29
- self.computed = False
30
-
31
- def __repr__(self) -> str:
32
- return (f'PhysicalTokenBlock(device={self.device}, '
33
- f'block_number={self.block_number}, '
34
- f'num_hashed_tokens={self.num_hashed_tokens}, '
35
- f'ref_count={self.ref_count}, '
36
- f'last_accessed={self.last_accessed}, '
37
- f'computed={self.computed})')
38
-
39
-
40
-class BlockTable:
41
- """Holds a list of blocks with caching of their associated block_ids
42
- """
43
-
44
- def __init__(self, blocks: Optional[List[PhysicalTokenBlock]] = None):
45
- self._blocks: List[PhysicalTokenBlock] = []
46
- self._block_ids: List[int] = []
47
-
48
- if blocks is not None:
49
- for block in blocks:
50
- self.append(block)
51
-
52
- def append(self, block: PhysicalTokenBlock):
53
- self._blocks.append(block)
54
- self._block_ids.append(block.block_number)
55
-
56
- def __len__(self) -> int:
57
- return len(self._blocks)
58
-
59
- def __getitem__(self, key):
60
- return self._blocks[key]
61
-
62
- if TYPE_CHECKING:
63
-
64
- def __iter__(self) -> Iterator[PhysicalTokenBlock]:
65
- raise RuntimeError("Method should be automatically generated")
66
-
67
- def __setitem__(self, key, value):
68
- if isinstance(key, slice):
69
- blocks = value
70
- self._blocks[key] = blocks
71
- self._block_ids[key] = [b.block_number for b in blocks]
72
- else:
73
- block = value
74
- self._blocks[key] = block
75
- self._block_ids[key] = block.block_number
76
-
77
- def reset(self):
78
- self._blocks = []
79
- self._block_ids = []
80
-
81
- def copy(self) -> "BlockTable":
82
- return BlockTable(self._blocks)
83
-
84
- def list(self) -> List[PhysicalTokenBlock]:
85
- return self._blocks
86
-
87
- def ids(self) -> List[int]:
88
- return self._block_ids
vllm/core/evictor.py
+2
-2
@@ -13,7 +13,7 @@ class EvictionPolicy(enum.Enum):
13
14
class Evictor(ABC):
15
"""The Evictor subclasses should be used by the BlockAllocator class to
16
- handle eviction of freed PhysicalTokenBlocks.
16
+ handle eviction of freed Blocks.
17
"""
18
19
@abstractmethod
@@ -70,7 +70,7 @@ class BlockMetaData:
70
71
class LRUEvictor(Evictor):
72
"""Evicts in a least-recently-used order using the last_accessed timestamp
73
- that's recorded in the PhysicalTokenBlock. If there are multiple blocks with
73
+ that's recorded in the Block. If there are multiple blocks with
74
the same last_accessed time, then the one with the largest num_hashed_tokens
75
will be evicted. If two blocks each have the lowest last_accessed time and
76
highest num_hashed_tokens value, then one will be chose arbitrarily