@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c31d4a57a6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Core] support LoRA and prompt adapter in content-based hashing for Block Manager v2 prefix caching (#8240)

Sungjae Lee committed Dec 14, 2024 at 00:51 UTC c31d4a57a6b639900a7c70b6e844db0116c2f9f6
10 files changed +246 -55
tests/core/block/test_prefix_caching_block.py
+63 -2
@@ -5,7 +5,7 @@ from unittest.mock import MagicMock
5
6 import pytest
7
8 -from tests.core.utils import create_dummy_sequence
8 +from tests.core.utils import create_dummy_lora_sequence, create_dummy_sequence
9 from vllm.core.block.cpu_gpu_block_allocator import CpuGpuBlockAllocator
10 from vllm.core.block.interfaces import Block, BlockAllocator
11 from vllm.core.block.prefix_caching_block import (ComputedBlocksTracker,
@@ -801,6 +801,7 @@ class TestPrefixCachingBlockAllocator:
801 block_size: int,
802 token_ids: List[int],
803 allocator: PrefixCachingBlockAllocator,
804 + extra_hash: Optional[int] = None,
805 ) -> List[PrefixCachingBlock]:
806 """Helper method which creates a chain of blocks.
807 """
@@ -816,7 +817,9 @@ class TestPrefixCachingBlockAllocator:
817 block_size:(block_number + 1) *
818 block_size]
819 prev_block = allocator.allocate_immutable_block(
819 - prev_block=prev_block, token_ids=block_token_ids)
820 + prev_block=prev_block,
821 + token_ids=block_token_ids,
822 + extra_hash=extra_hash)
823 blocks.append(prev_block)
824
825 return blocks
@@ -931,3 +934,61 @@ class TestComputedBlocksTracker:
934 allocator.mark_blocks_as_computed([])
935
936 assert tracker.get_num_cached_tokens(seq) == len(tokens)
937 +
938 + @staticmethod
939 + def test_correct_extra_hash():
940 + """
941 + Test that the block hash is correctly computed based on the extra hash,
942 + ensuring it matches the allocator's block hash, specifically for the
943 + LoRA case, and that the correct number of cached tokens is retrieved.
944 + """
945 + block_size = 4
946 + allocator = CpuGpuBlockAllocator.create(
947 + allocator_type="prefix_caching",
948 + num_gpu_blocks=16,
949 + num_cpu_blocks=16,
950 + block_size=block_size,
951 + )
952 + gpu_allocator = allocator._allocators[Device.GPU]
953 +
954 + tracker = ComputedBlocksTracker(
955 + allocator=allocator,
956 + block_size=block_size,
957 + enable_caching=True,
958 + )
959 +
960 + tokens = list(range(block_size * 4))
961 +
962 + # Create a dummy LoRA sequence with a specific LoRA ID.
963 + lora_seq = create_dummy_lora_sequence(request_id=0,
964 + token_ids=tokens,
965 + block_size=block_size,
966 + lora_int_id=1)
967 +
968 + _ = TestPrefixCachingBlockAllocator.create_immutable_chain(
969 + block_size=block_size,
970 + token_ids=tokens,
971 + allocator=gpu_allocator,
972 + extra_hash=lora_seq.extra_hash(),
973 + )
974 +
975 + allocator.mark_blocks_as_computed([])
976 +
977 + # Create different dummy sequences that have the same token IDs
978 + # but different LoRA IDs.
979 + seq = create_dummy_sequence(request_id=1,
980 + token_ids=tokens,
981 + block_size=block_size)
982 +
983 + different_lora_seq = create_dummy_lora_sequence(request_id=2,
984 + token_ids=tokens,
985 + block_size=block_size,
986 + lora_int_id=2)
987 +
988 + # Due to the different LoRA IDs, corresponding blocks are not cached.
989 + assert tracker.get_num_cached_tokens(seq) == 0
990 + assert tracker.get_num_cached_tokens(different_lora_seq) == 0
991 +
992 + # The number of cached tokens matches the length of the tokens
993 + # for the cached LoRA sequence.
994 + assert tracker.get_num_cached_tokens(lora_seq) == len(tokens)
tests/core/utils.py
+10
@@ -46,6 +46,16 @@ def create_dummy_prompt(
46 return prompt, seq_group
47
48
49 +def create_dummy_lora_sequence(request_id: int, token_ids: List[int],
50 + block_size: int, lora_int_id: int) -> Sequence:
51 + return Sequence(seq_id=request_id,
52 + inputs=token_inputs(token_ids),
53 + block_size=block_size,
54 + lora_request=LoRARequest(lora_name="dummy",
55 + lora_path="/dummy",
56 + lora_int_id=lora_int_id))
57 +
58 +
59 def create_dummy_sequence(request_id: int, token_ids: List[int],
60 block_size: int) -> Sequence:
61 return Sequence(
vllm/core/block/block_table.py
+34 -12
@@ -80,7 +80,8 @@ class BlockTable:
80
81 def allocate(self,
82 token_ids: List[int],
83 - device: Device = Device.GPU) -> None:
83 + device: Device = Device.GPU,
84 + extra_hash: Optional[int] = None) -> None:
85 """Allocates memory blocks for storing the given sequence of token IDs.
86
87 This method allocates the required number of blocks to store the given
@@ -90,12 +91,16 @@ class BlockTable:
91 token_ids (List[int]): The sequence of token IDs to be stored.
92 device (Device, optional): The device on which the blocks should be
93 allocated. Defaults to Device.GPU.
94 + extra_hash (Optional[int]): The hash value of additional
95 + factors, such as adapters, that influence the block hash
96 + in the prefixcaching block.
97 """
98 assert not self._is_allocated
99 assert token_ids
100 blocks = self._allocate_blocks_for_token_ids(prev_block=None,
101 token_ids=token_ids,
98 - device=device)
102 + device=device,
103 + extra_hash=extra_hash)
104 self.update(blocks)
105 self._num_full_slots = len(token_ids)
106
@@ -108,7 +113,8 @@ class BlockTable:
113 def append_token_ids(self,
114 token_ids: List[int],
115 num_lookahead_slots: int = 0,
111 - num_computed_slots: Optional[int] = None) -> None:
116 + num_computed_slots: Optional[int] = None,
117 + extra_hash: Optional[int] = None) -> None:
118 """Appends a sequence of token IDs to the existing blocks in the
119 BlockTable.
120
@@ -130,6 +136,9 @@ class BlockTable:
136 Without sliding window, None can be passed.
137 Without chunked prefill, it should be the same as
138 _num_full_slots.
139 + extra_hash (Optional[int]): The hash value of additional
140 + factors such as adapters that influence the block, apart
141 + from the token_ids.
142 """
143 assert self._is_allocated, "no blocks have been allocated"
144 assert len(self._blocks) > 0
@@ -149,7 +158,8 @@ class BlockTable:
158 # Ensure there are enough empty slots for the new tokens plus
159 # lookahead slots
160 self.ensure_num_empty_slots(num_empty_slots=len(token_ids) +
152 - num_lookahead_slots)
161 + num_lookahead_slots,
162 + extra_hash=extra_hash)
163
164 # Update the blocks with the new tokens
165 first_block_idx = self._num_full_slots // self._block_size
@@ -160,7 +170,9 @@ class BlockTable:
170
171 self._num_full_slots += len(token_ids)
172
163 - def ensure_num_empty_slots(self, num_empty_slots: int) -> None:
173 + def ensure_num_empty_slots(self,
174 + num_empty_slots: int,
175 + extra_hash: Optional[int] = None) -> None:
176 """Ensures that the BlockTable has at least the specified number of
177 empty slots available.
178
@@ -171,6 +183,9 @@ class BlockTable:
183
184 Args:
185 num_empty_slots (int): The minimum number of empty slots required.
186 + extra_hash (Optional[int]): The hash value of additional
187 + factors such as adapters that influence the block, apart
188 + from the token_ids.
189 """
190 # Currently the block table only supports
191 # appending tokens to GPU blocks.
@@ -187,7 +202,9 @@ class BlockTable:
202 assert len(self._blocks) > 0
203 self._blocks.append(
204 self._allocator.allocate_mutable_block(
190 - prev_block=self._blocks[-1], device=device))
205 + prev_block=self._blocks[-1],
206 + device=device,
207 + extra_hash=extra_hash))
208
209 def fork(self) -> "BlockTable":
210 """Creates a new BlockTable instance with a copy of the blocks from the
@@ -259,9 +276,12 @@ class BlockTable:
276 # ones after the appended ones.
277 return sequence_token_ids[self.num_full_slots:]
278
262 - def _allocate_blocks_for_token_ids(self, prev_block: Optional[Block],
263 - token_ids: List[int],
264 - device: Device) -> List[Block]:
279 + def _allocate_blocks_for_token_ids(
280 + self,
281 + prev_block: Optional[Block],
282 + token_ids: List[int],
283 + device: Device,
284 + extra_hash: Optional[int] = None) -> List[Block]:
285 blocks: List[Block] = []
286
287 block_token_ids = []
@@ -275,8 +295,10 @@ class BlockTable:
295 if block_token_ids:
296 blocks.extend(
297 self._allocator.allocate_immutable_blocks(
278 - prev_block, block_token_ids=block_token_ids,
279 - device=device))
298 + prev_block,
299 + block_token_ids=block_token_ids,
300 + device=device,
301 + extra_hash=extra_hash))
302 prev_block = blocks[-1]
303
304 if tail_token_ids:
@@ -284,7 +306,7 @@ class BlockTable:
306 cur_token_ids = tail_token_ids[0]
307
308 block = self._allocator.allocate_mutable_block(
287 - prev_block=prev_block, device=device)
309 + prev_block=prev_block, device=device, extra_hash=extra_hash)
310 block.append_token_ids(cur_token_ids)
311
312 blocks.append(block)
vllm/core/block/common.py
+13 -6
@@ -177,7 +177,8 @@ class BlockPool:
177 token_ids=[],
178 block_size=self._block_size,
179 allocator=self._allocator,
180 - block_id=None))
180 + block_id=None,
181 + extra_hash=None))
182
183 def increase_pool(self):
184 """Doubles the internal pool size
@@ -194,10 +195,15 @@ class BlockPool:
195 token_ids=[],
196 block_size=self._block_size,
197 allocator=self._allocator,
197 - block_id=None))
198 -
199 - def init_block(self, prev_block: Optional[Block], token_ids: List[int],
200 - block_size: int, physical_block_id: Optional[int]) -> Block:
198 + block_id=None,
199 + extra_hash=None))
200 +
201 + def init_block(self,
202 + prev_block: Optional[Block],
203 + token_ids: List[int],
204 + block_size: int,
205 + physical_block_id: Optional[int],
206 + extra_hash: Optional[int] = None) -> Block:
207 if len(self._free_ids) == 0:
208 self.increase_pool()
209 assert len(self._free_ids) > 0
@@ -210,7 +216,8 @@ class BlockPool:
216 token_ids=token_ids,
217 block_size=block_size,
218 allocator=block._allocator, # type: ignore[attr-defined]
213 - block_id=physical_block_id)
219 + block_id=physical_block_id,
220 + extra_hash=extra_hash)
221 block.pool_id = pool_id # type: ignore[attr-defined]
222 return block
223
vllm/core/block/cpu_gpu_block_allocator.py
+32 -11
@@ -121,23 +121,32 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
121 self.allocate_mutable_block(None, Device.GPU))
122 return self._null_block
123
124 - def allocate_mutable_block(self, prev_block: Optional[Block],
125 - device: Device) -> Block:
124 + def allocate_mutable_block(self,
125 + prev_block: Optional[Block],
126 + device: Device,
127 + extra_hash: Optional[int] = None) -> Block:
128 """Allocates a new mutable block on the specified device.
129
130 Args:
131 prev_block (Optional[Block]): The previous block to in the sequence.
132 Used for prefix hashing.
133 device (Device): The device on which to allocate the new block.
134 + extra_hash (Optional[int]): The hash value of additional
135 + factors, such as adapters, that influence the block hash
136 + in the prefix caching block.
137
138 Returns:
139 Block: The newly allocated mutable block.
140 """
136 - return self._allocators[device].allocate_mutable_block(prev_block)
137 -
138 - def allocate_immutable_blocks(self, prev_block: Optional[Block],
139 - block_token_ids: List[List[int]],
140 - device: Device) -> List[Block]:
141 + return self._allocators[device].allocate_mutable_block(
142 + prev_block, extra_hash=extra_hash)
143 +
144 + def allocate_immutable_blocks(
145 + self,
146 + prev_block: Optional[Block],
147 + block_token_ids: List[List[int]],
148 + device: Device,
149 + extra_hash: Optional[int] = None) -> List[Block]:
150 """Allocates a new group of immutable blocks with the provided block
151 token IDs on the specified device.
152
@@ -147,17 +156,22 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
156 block_token_ids (List[int]): The list of block token IDs to be
157 stored in the new blocks.
158 device (Device): The device on which to allocate the new block.
159 + extra_hash (Optional[int]): The hash value of additional
160 + factors, such as adapters, that influence the block hash
161 + in the prefix caching block.
162
163 Returns:
164 List[Block]: The newly allocated list of immutable blocks
165 containing the provided block token IDs.
166 """
167 return self._allocators[device].allocate_immutable_blocks(
156 - prev_block, block_token_ids)
168 + prev_block, block_token_ids, extra_hash=extra_hash)
169
158 - def allocate_immutable_block(self, prev_block: Optional[Block],
170 + def allocate_immutable_block(self,
171 + prev_block: Optional[Block],
172 token_ids: List[int],
160 - device: Device) -> Block:
173 + device: Device,
174 + extra_hash: Optional[int] = None) -> Block:
175 """Allocates a new immutable block with the provided token IDs on the
176 specified device.
177
@@ -167,13 +181,16 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
181 token_ids (List[int]): The list of token IDs to be stored in the new
182 block.
183 device (Device): The device on which to allocate the new block.
184 + extra_hash (Optional[int]): The hash value of additional
185 + factors, such as adapters, that influence the block hash
186 + in the prefix caching block.
187
188 Returns:
189 Block: The newly allocated immutable block containing the provided
190 token IDs.
191 """
192 return self._allocators[device].allocate_immutable_block(
176 - prev_block, token_ids)
193 + prev_block, token_ids, extra_hash=extra_hash)
194
195 def free(self, block: Block) -> None:
196 """Frees the memory occupied by the given block.
@@ -387,6 +404,10 @@ class NullBlock(Block):
404 def prev_block(self):
405 return self._proxy.prev_block
406
407 + @property
408 + def extra_hash(self):
409 + return None
410 +
411 @property
412 def computed(self):
413 return self._proxy.computed
vllm/core/block/interfaces.py
+23 -9
@@ -50,6 +50,11 @@ class Block(ABC):
50 def prev_block(self) -> Optional["Block"]:
51 pass
52
53 + @property
54 + @abstractmethod
55 + def extra_hash(self) -> Optional[int]:
56 + return None
57 +
58 @property
59 @abstractmethod
60 def computed(self) -> bool:
@@ -81,6 +86,8 @@ class Block(ABC):
86 block_size: int,
87 allocator: "BlockAllocator",
88 block_id: Optional[int] = None,
89 + computed: bool = False,
90 + extra_hash: Optional[int] = None,
91 ) -> "Block":
92 pass
93
@@ -99,18 +106,20 @@ class Block(ABC):
106 class BlockAllocator(ABC):
107
108 @abstractmethod
102 - def allocate_mutable_block(self, prev_block: Optional[Block]) -> Block:
109 + def allocate_mutable_block(self, prev_block: Optional[Block],
110 + extra_hash: Optional[int]) -> Block:
111 pass
112
113 @abstractmethod
114 def allocate_immutable_block(self, prev_block: Optional[Block],
107 - token_ids: List[int]) -> Block:
115 + token_ids: List[int],
116 + extra_hash: Optional[int]) -> Block:
117 pass
118
119 @abstractmethod
111 - def allocate_immutable_blocks(
112 - self, prev_block: Optional[Block],
113 - block_token_ids: List[List[int]]) -> List[Block]:
120 + def allocate_immutable_blocks(self, prev_block: Optional[Block],
121 + block_token_ids: List[List[int]],
122 + extra_hash: Optional[int]) -> List[Block]:
123 pass
124
125 @abstractmethod
@@ -197,14 +206,18 @@ class BlockAllocator(ABC):
206 class DeviceAwareBlockAllocator(ABC):
207
208 @abstractmethod
200 - def allocate_mutable_block(self, prev_block: Optional[Block],
201 - device: Device) -> Block:
209 + def allocate_mutable_block(self,
210 + prev_block: Optional[Block],
211 + device: Device,
212 + extra_hash: Optional[int] = None) -> Block:
213 pass
214
215 @abstractmethod
205 - def allocate_immutable_block(self, prev_block: Optional[Block],
216 + def allocate_immutable_block(self,
217 + prev_block: Optional[Block],
218 token_ids: List[int],
207 - device: Device) -> Block:
219 + device: Device,
220 + extra_hash: Optional[int] = None) -> Block:
221 pass
222
223 @abstractmethod
@@ -213,6 +226,7 @@ class DeviceAwareBlockAllocator(ABC):
226 prev_block: Optional[Block],
227 block_token_ids: List[List[int]],
228 device: Device,
229 + extra_hash: Optional[int] = None,
230 ) -> List[Block]:
231 pass
232
vllm/core/block/naive_block.py
+9 -1
@@ -63,6 +63,7 @@ class NaiveBlockAllocator(BlockAllocator):
63 def allocate_immutable_block(self,
64 prev_block: Optional[Block],
65 token_ids: List[int],
66 + extra_hash: Optional[int] = None,
67 device: Optional[Device] = None) -> Block:
68 """Allocates a new immutable block with the given token IDs, linked to
69 the previous block.
@@ -85,6 +86,7 @@ class NaiveBlockAllocator(BlockAllocator):
86 self,
87 prev_block: Optional[Block],
88 block_token_ids: List[List[int]],
89 + extra_hash: Optional[int] = None,
90 device: Optional[Device] = None) -> List[Block]:
91 assert device is None
92 num_blocks = len(block_token_ids)
@@ -106,6 +108,7 @@ class NaiveBlockAllocator(BlockAllocator):
108
109 def allocate_mutable_block(self,
110 prev_block: Optional[Block],
111 + extra_hash: Optional[int] = None,
112 device: Optional[Device] = None) -> Block:
113 """Allocates a new mutable block, linked to the previous block.
114
@@ -355,7 +358,8 @@ class NaiveBlock(Block):
358 block_size: int,
359 allocator: BlockAllocator,
360 block_id: Optional[int] = None,
358 - _cow_target: Optional[Block] = None):
361 + _cow_target: Optional[Block] = None,
362 + extra_hash: Optional[int] = None):
363 self._token_ids: List[int] = []
364 self._block_size = block_size
365 self._prev_block = prev_block
@@ -441,6 +445,10 @@ class NaiveBlock(Block):
445 def prev_block(self) -> Optional["Block"]:
446 return self._prev_block
447
448 + @property
449 + def extra_hash(self):
450 + return None
451 +
452 @property
453 def content_hash(self) -> Optional[int]:
454 return None
vllm/core/block/prefix_caching_block.py
+42 -13
@@ -126,6 +126,7 @@ class PrefixCachingBlockAllocator(BlockAllocator):
126 allocator: BlockAllocator,
127 block_id: Optional[int] = None,
128 computed: bool = False,
129 + extra_hash: Optional[int] = None,
130 ) -> Block:
131 # Bind block to self.
132 allocator = self
@@ -137,11 +138,13 @@ class PrefixCachingBlockAllocator(BlockAllocator):
138 block_id=block_id,
139 allocator=allocator,
140 computed=computed,
141 + extra_hash=extra_hash,
142 )
143
144 def allocate_immutable_block(self,
145 prev_block: Optional[Block],
146 token_ids: List[int],
147 + extra_hash: Optional[int] = None,
148 device: Optional[Device] = None) -> Block:
149 """Allocates an immutable block with the given token IDs, reusing cached
150 blocks if possible.
@@ -160,7 +163,8 @@ class PrefixCachingBlockAllocator(BlockAllocator):
163 block = self._block_pool.init_block(prev_block=prev_block,
164 token_ids=token_ids,
165 block_size=self._block_size,
163 - physical_block_id=None)
166 + physical_block_id=None,
167 + extra_hash=extra_hash)
168 assert block.content_hash is not None
169
170 cached_block_id = self._cached_blocks.get(block.content_hash, None)
@@ -173,7 +177,7 @@ class PrefixCachingBlockAllocator(BlockAllocator):
177 self._block_pool.free_block(block)
178
179 # No cached block => Allocate a new block
176 - block = self.allocate_mutable_block(prev_block)
180 + block = self.allocate_mutable_block(prev_block, extra_hash=extra_hash)
181 block.append_token_ids(token_ids)
182 return block
183
@@ -181,17 +185,20 @@ class PrefixCachingBlockAllocator(BlockAllocator):
185 self,
186 prev_block: Optional[Block],
187 block_token_ids: List[List[int]],
188 + extra_hash: Optional[int] = None,
189 device: Optional[Device] = None) -> List[Block]:
190 blocks = []
191 for token_ids in block_token_ids:
192 prev_block = self.allocate_immutable_block(prev_block=prev_block,
193 token_ids=token_ids,
189 - device=device)
194 + device=device,
195 + extra_hash=extra_hash)
196 blocks.append(prev_block)
197 return blocks
198
199 def allocate_mutable_block(self,
200 prev_block: Optional[Block],
201 + extra_hash: Optional[int] = None,
202 device: Optional[Device] = None) -> Block:
203 """Allocates a mutable block. If there are no free blocks, this will
204 evict unused cached blocks.
@@ -210,7 +217,8 @@ class PrefixCachingBlockAllocator(BlockAllocator):
217 block = self._block_pool.init_block(prev_block=prev_block,
218 token_ids=[],
219 block_size=self._block_size,
213 - physical_block_id=block_id)
220 + physical_block_id=block_id,
221 + extra_hash=extra_hash)
222 assert not block.computed
223 assert block.content_hash is None
224 return block
@@ -382,7 +390,8 @@ class PrefixCachingBlockAllocator(BlockAllocator):
390 prev_block=prev_block,
391 token_ids=block.token_ids,
392 block_size=self._block_size,
385 - physical_block_id=block_id)
393 + physical_block_id=block_id,
394 + extra_hash=block.extra_hash)
395
396 forked_blocks.append(forked_block)
397 prev_block = forked_blocks[-1]
@@ -608,10 +617,12 @@ class PrefixCachingBlockAllocator(BlockAllocator):
617 # existing "block" object
618 if block.is_full:
619 tmp_block = self.allocate_immutable_block(
611 - prev_block=block.prev_block, token_ids=block.token_ids)
620 + prev_block=block.prev_block,
621 + token_ids=block.token_ids,
622 + extra_hash=block.extra_hash)
623 else:
624 tmp_block = self.allocate_mutable_block(
614 - prev_block=block.prev_block)
625 + prev_block=block.prev_block, extra_hash=block.extra_hash)
626 tmp_block.append_token_ids(block.token_ids)
627
628 block_id = tmp_block.block_id
@@ -679,6 +690,8 @@ class PrefixCachingBlock(Block):
690 caching block allocator associated with this block.
691 block_id (Optional[int], optional): The physical block index
692 of this block. Defaults to None.
693 + extra_hash (Optional[int]): The hash value of additional factors
694 + such as adapters that influence the block, apart from the token_ids.
695 """
696
697 def __init__(
@@ -689,6 +702,7 @@ class PrefixCachingBlock(Block):
702 allocator: BlockAllocator,
703 block_id: Optional[int] = None,
704 computed: bool = False,
705 + extra_hash: Optional[int] = None,
706 ):
707 assert isinstance(allocator, PrefixCachingBlockAllocator), (
708 "Currently this class is only tested with "
@@ -702,6 +716,7 @@ class PrefixCachingBlock(Block):
716 self._allocator = allocator
717 self._last_accessed: float = _DEFAULT_LAST_ACCESSED_TIME
718 self._computed = computed
719 + self._extra_hash = extra_hash
720
721 # On the first time, we create the block object, and next we only
722 # reinitialize it
@@ -811,6 +826,10 @@ class PrefixCachingBlock(Block):
826 def prev_block(self) -> Optional[Block]:
827 return self._prev_block
828
829 + @property
830 + def extra_hash(self) -> Optional[int]:
831 + return self._extra_hash
832 +
833 @property
834 def content_hash(self) -> Optional[int]:
835 """Return the content-based hash of the current block, or None if it is
@@ -841,18 +860,19 @@ class PrefixCachingBlock(Block):
860 self._cached_content_hash = PrefixCachingBlock.hash_block_tokens(
861 is_first_block,
862 prev_block_hash,
844 - cur_block_token_ids=self.token_ids)
863 + cur_block_token_ids=self.token_ids,
864 + extra_hash=self._extra_hash)
865 return self._cached_content_hash
866
867 @staticmethod
848 - def hash_block_tokens(is_first_block: bool, prev_block_hash: Optional[int],
849 - cur_block_token_ids: List[int]) -> int:
868 + def hash_block_tokens(is_first_block: bool,
869 + prev_block_hash: Optional[int],
870 + cur_block_token_ids: List[int],
871 + extra_hash: Optional[int] = None) -> int:
872 """Computes a hash value corresponding to the contents of a block and
873 the contents of the preceding block(s). The hash value is used for
874 prefix caching.
875
854 - NOTE: Content-based hashing does not yet support LoRA.
855 -
876 Parameters:
877 - is_first_block (bool): A flag indicating if the block is the first in
878 the sequence.
@@ -860,12 +880,15 @@ class PrefixCachingBlock(Block):
880 if this is the first block.
881 - cur_block_token_ids (List[int]): A list of token ids in the current
882 block. The current block is assumed to be full.
883 + - extra_hash (Optional[int]): The hash value of additional factors
884 + such as adapters that influence the block, apart from the token_ids.
885
886 Returns:
887 - int: The computed hash value for the block.
888 """
889 assert (prev_block_hash is None) == is_first_block
868 - return hash((is_first_block, prev_block_hash, *cur_block_token_ids))
890 + return hash((is_first_block, prev_block_hash, *cur_block_token_ids,
891 + extra_hash))
892
893
894 class ComputedBlocksTracker:
@@ -935,12 +958,18 @@ class ComputedBlocksTracker:
958 assert len(token_ids) >= (i + 1) * self._block_size
959 block_token_ids = token_ids[i * self._block_size:(i + 1) *
960 self._block_size]
961 +
962 + # NOTE: If there are any factors affecting the block besides
963 + # token_ids, they should be added as input to extra_hash.
964 + extra_hash = seq.extra_hash()
965 +
966 # This has to be kept in sync with the allocator's hash
967 # calculation.
968 block_hash = PrefixCachingBlock.hash_block_tokens(
969 is_first_block=prev_block_hash is None,
970 prev_block_hash=prev_block_hash,
971 cur_block_token_ids=block_token_ids,
972 + extra_hash=extra_hash,
973 )
974 block_hashes_recorded.append(block_hash)
975 prev_block_hash = block_hash
vllm/core/block_manager.py
+7 -1
@@ -151,8 +151,13 @@ class SelfAttnBlockSpaceManager(BlockSpaceManager):
151 max_block_sliding_window=self.max_block_sliding_window,
152 )
153 if seq.get_token_ids():
154 + # NOTE: If there are any factors affecting the block besides
155 + # token_ids, they should be added as input to extra_hash.
156 + extra_hash = seq.extra_hash()
157 +
158 # Add blocks to the block table only if the sequence is non empty.
155 - block_table.allocate(seq.get_token_ids())
159 + block_table.allocate(token_ids=seq.get_token_ids(),
160 + extra_hash=extra_hash)
161
162 return block_table
163
@@ -238,6 +243,7 @@ class SelfAttnBlockSpaceManager(BlockSpaceManager):
243 token_ids=block_table.get_unseen_token_ids(seq.get_token_ids()),
244 num_lookahead_slots=num_lookahead_slots,
245 num_computed_slots=seq.data.get_num_computed_tokens(),
246 + extra_hash=seq.extra_hash(),
247 )
248 # Return any new copy-on-writes.
249 new_cows = self.block_allocator.clear_copy_on_writes()
vllm/sequence.py
+13
@@ -527,6 +527,19 @@ class Sequence:
527 hashed_tokens = self.data.get_prefix_token_ids(num_tokens)
528 return hash((hashed_tokens, self.lora_int_id))
529
530 + def extra_hash(self) -> Optional[int]:
531 + """
532 + This function computes an extra hash for a sequence, specifically
533 + designed for prefix caching mode. The final sequence hash is determined
534 + by applying token_ids from the sequence's blocks.
535 + """
536 + if self.prompt_adapter_id == 0 and self.lora_int_id == 0:
537 + return None
538 +
539 + # NOTE: If there are additional factors influencing the block aside from
540 + # token_ids, include them as input parameters to the hash.
541 + return hash((self.prompt_adapter_id, self.lora_int_id))
542 +
543 def num_hashed_tokens_of_block(self, logical_idx: int):
544 return logical_idx * self.block_size + self.block_size
545