A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[Misc]add some explanations for BlockHashType (#11847)
WangErXiao committed
Jan 9, 2025 at 02:21 UTC
47de8821d3cdd32fce7df6312318223aee591fd2
1 file changed
+4
-2
vllm/v1/core/kv_cache_utils.py
+4
-2
@@ -11,8 +11,10 @@ logger = init_logger(__name__)
11
12
class BlockHashType(NamedTuple):
13
"""Hash value of a block (int), the token IDs in the block, and extra keys.
14
- The reason we keep a tuple of token IDs and extra keys is to make sure
15
- no hash collision happens when the hash value is the same.
14
+ We keep a tuple of token IDs and extra keys to reduce the likelihood of
15
+ hash collisions when the hash value is the same. But please note that
16
+ hash collisions can still theoretically occur, albeit with an extremely
17
+ low probability.
18
"""
19
# Hash value of the block in an integer.
20
hash_value: int