A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Core] support LoRA and prompt adapter in content-based hashing for Block Manager v2 prefix caching (#8240)
Sungjae Lee committed
Dec 14, 2024 at 00:51 UTC
c31d4a57a6b639900a7c70b6e844db0116c2f9f6
10 files changed
+246
-55
tests/core/block/test_prefix_caching_block.py
+63
-2
@@ -5,7 +5,7 @@ from unittest.mock import MagicMock
5
6
import pytest
7
8
-from tests.core.utils import create_dummy_sequence
8
+from tests.core.utils import create_dummy_lora_sequence, create_dummy_sequence
9
from vllm.core.block.cpu_gpu_block_allocator import CpuGpuBlockAllocator
10
from vllm.core.block.interfaces import Block, BlockAllocator
11
from vllm.core.block.prefix_caching_block import (ComputedBlocksTracker,
@@ -801,6 +801,7 @@ class TestPrefixCachingBlockAllocator:
801
block_size: int,
802
token_ids: List[int],
803
allocator: PrefixCachingBlockAllocator,
804
+ extra_hash: Optional[int] = None,
805
) -> List[PrefixCachingBlock]:
806
"""Helper method which creates a chain of blocks.
807
"""
@@ -816,7 +817,9 @@ class TestPrefixCachingBlockAllocator:
817
block_size:(block_number + 1) *
818
block_size]
819
prev_block = allocator.allocate_immutable_block(
819
- prev_block=prev_block, token_ids=block_token_ids)
820
+ prev_block=prev_block,
821
+ token_ids=block_token_ids,
822
+ extra_hash=extra_hash)
823
blocks.append(prev_block)
824
825
return blocks
@@ -931,3 +934,61 @@ class TestComputedBlocksTracker:
934
allocator.mark_blocks_as_computed([])
935
936
assert tracker.get_num_cached_tokens(seq) == len(tokens)
937
+
938
+ @staticmethod
939
+ def test_correct_extra_hash():
940
+ """
941
+ Test that the block hash is correctly computed based on the extra hash,
942
+ ensuring it matches the allocator's block hash, specifically for the
943
+ LoRA case, and that the correct number of cached tokens is retrieved.
944
+ """
945
+ block_size = 4
946
+ allocator = CpuGpuBlockAllocator.create(
947
+ allocator_type="prefix_caching",
948
+ num_gpu_blocks=16,
949
+ num_cpu_blocks=16,
950
+ block_size=block_size,
951
+ )
952
+ gpu_allocator = allocator._allocators[Device.GPU]
953
+
954
+ tracker = ComputedBlocksTracker(
955
+ allocator=allocator,
956
+ block_size=block_size,
957
+ enable_caching=True,
958
+ )
959
+
960
+ tokens = list(range(block_size * 4))
961
+
962
+ # Create a dummy LoRA sequence with a specific LoRA ID.
963
+ lora_seq = create_dummy_lora_sequence(request_id=0,
964
+ token_ids=tokens,
965
+ block_size=block_size,
966
+ lora_int_id=1)
967
+
968
+ _ = TestPrefixCachingBlockAllocator.create_immutable_chain(
969
+ block_size=block_size,
970
+ token_ids=tokens,
971
+ allocator=gpu_allocator,
972
+ extra_hash=lora_seq.extra_hash(),
973
+ )
974
+
975
+ allocator.mark_blocks_as_computed([])
976
+
977
+ # Create different dummy sequences that have the same token IDs
978
+ # but different LoRA IDs.
979
+ seq = create_dummy_sequence(request_id=1,
980
+ token_ids=tokens,
981
+ block_size=block_size)
982
+
983
+ different_lora_seq = create_dummy_lora_sequence(request_id=2,
984
+ token_ids=tokens,
985
+ block_size=block_size,
986
+ lora_int_id=2)
987
+
988
+ # Due to the different LoRA IDs, corresponding blocks are not cached.
989
+ assert tracker.get_num_cached_tokens(seq) == 0
990
+ assert tracker.get_num_cached_tokens(different_lora_seq) == 0
991
+
992
+ # The number of cached tokens matches the length of the tokens
993
+ # for the cached LoRA sequence.
994
+ assert tracker.get_num_cached_tokens(lora_seq) == len(tokens)
tests/core/utils.py
+10
@@ -46,6 +46,16 @@ def create_dummy_prompt(
46
return prompt, seq_group
47
48
49
+def create_dummy_lora_sequence(request_id: int, token_ids: List[int],
50
+ block_size: int, lora_int_id: int) -> Sequence:
51
+ return Sequence(seq_id=request_id,
52
+ inputs=token_inputs(token_ids),
53
+ block_size=block_size,
54
+ lora_request=LoRARequest(lora_name="dummy",
55
+ lora_path="/dummy",
56
+ lora_int_id=lora_int_id))
57
+
58
+
59
def create_dummy_sequence(request_id: int, token_ids: List[int],
60
block_size: int) -> Sequence:
61
return Sequence(
vllm/core/block/block_table.py
+34
-12
@@ -80,7 +80,8 @@ class BlockTable:
80
81
def allocate(self,
82
token_ids: List[int],
83
- device: Device = Device.GPU) -> None:
83
+ device: Device = Device.GPU,
84
+ extra_hash: Optional[int] = None) -> None:
85
"""Allocates memory blocks for storing the given sequence of token IDs.
86
87
This method allocates the required number of blocks to store the given
@@ -90,12 +91,16 @@ class BlockTable:
91
token_ids (List[int]): The sequence of token IDs to be stored.
92
device (Device, optional): The device on which the blocks should be
93
allocated. Defaults to Device.GPU.
94
+ extra_hash (Optional[int]): The hash value of additional
95
+ factors, such as adapters, that influence the block hash
96
+ in the prefixcaching block.
97
"""
98
assert not self._is_allocated
99
assert token_ids
100
blocks = self._allocate_blocks_for_token_ids(prev_block=None,
101
token_ids=token_ids,
98
- device=device)
102
+ device=device,
103
+ extra_hash=extra_hash)
104
self.update(blocks)
105
self._num_full_slots = len(token_ids)
106
@@ -108,7 +113,8 @@ class BlockTable:
113
def append_token_ids(self,
114
token_ids: List[int],
115
num_lookahead_slots: int = 0,
111
- num_computed_slots: Optional[int] = None) -> None:
116
+ num_computed_slots: Optional[int] = None,
117
+ extra_hash: Optional[int] = None) -> None:
118
"""Appends a sequence of token IDs to the existing blocks in the
119
BlockTable.
120
@@ -130,6 +136,9 @@ class BlockTable:
136
Without sliding window, None can be passed.
137
Without chunked prefill, it should be the same as
138
_num_full_slots.
139
+ extra_hash (Optional[int]): The hash value of additional
140
+ factors such as adapters that influence the block, apart
141
+ from the token_ids.
142
"""
143
assert self._is_allocated, "no blocks have been allocated"
144
assert len(self._blocks) > 0
@@ -149,7 +158,8 @@ class BlockTable:
158
# Ensure there are enough empty slots for the new tokens plus
159
# lookahead slots
160
self.ensure_num_empty_slots(num_empty_slots=len(token_ids) +
152
- num_lookahead_slots)
161
+ num_lookahead_slots,
162
+ extra_hash=extra_hash)
163
164
# Update the blocks with the new tokens
165
first_block_idx = self._num_full_slots // self._block_size
@@ -160,7 +170,9 @@ class BlockTable:
170
171
self._num_full_slots += len(token_ids)
172
163
- def ensure_num_empty_slots(self, num_empty_slots: int) -> None:
173
+ def ensure_num_empty_slots(self,
174
+ num_empty_slots: int,
175
+ extra_hash: Optional[int] = None) -> None:
176
"""Ensures that the BlockTable has at least the specified number of
177
empty slots available.
178
@@ -171,6 +183,9 @@ class BlockTable:
183
184
Args:
185
num_empty_slots (int): The minimum number of empty slots required.
186
+ extra_hash (Optional[int]): The hash value of additional
187
+ factors such as adapters that influence the block, apart
188
+ from the token_ids.
189
"""
190
# Currently the block table only supports
191
# appending tokens to GPU blocks.
@@ -187,7 +202,9 @@ class BlockTable:
202
assert len(self._blocks) > 0
203
self._blocks.append(
204
self._allocator.allocate_mutable_block(
190
- prev_block=self._blocks[-1], device=device))
205
+ prev_block=self._blocks[-1],
206
+ device=device,
207
+ extra_hash=extra_hash))
208
209
def fork(self) -> "BlockTable":
210
"""Creates a new BlockTable instance with a copy of the blocks from the
@@ -259,9 +276,12 @@ class BlockTable:
276
# ones after the appended ones.
277
return sequence_token_ids[self.num_full_slots:]
278
262
- def _allocate_blocks_for_token_ids(self, prev_block: Optional[Block],
263
- token_ids: List[int],
264
- device: Device) -> List[Block]:
279
+ def _allocate_blocks_for_token_ids(
280
+ self,
281
+ prev_block: Optional[Block],
282
+ token_ids: List[int],
283
+ device: Device,
284
+ extra_hash: Optional[int] = None) -> List[Block]:
285
blocks: List[Block] = []
286
287
block_token_ids = []
@@ -275,8 +295,10 @@ class BlockTable:
295
if block_token_ids:
296
blocks.extend(
297
self._allocator.allocate_immutable_blocks(
278
- prev_block, block_token_ids=block_token_ids,
279
- device=device))
298
+ prev_block,
299
+ block_token_ids=block_token_ids,
300
+ device=device,
301
+ extra_hash=extra_hash))
302
prev_block = blocks[-1]
303
304
if tail_token_ids:
@@ -284,7 +306,7 @@ class BlockTable:
306
cur_token_ids = tail_token_ids[0]
307
308
block = self._allocator.allocate_mutable_block(
287
- prev_block=prev_block, device=device)
309
+ prev_block=prev_block, device=device, extra_hash=extra_hash)
310
block.append_token_ids(cur_token_ids)
311
312
blocks.append(block)
vllm/core/block/common.py
+13
-6
@@ -177,7 +177,8 @@ class BlockPool:
177
token_ids=[],
178
block_size=self._block_size,
179
allocator=self._allocator,
180
- block_id=None))
180
+ block_id=None,
181
+ extra_hash=None))
182
183
def increase_pool(self):
184
"""Doubles the internal pool size
@@ -194,10 +195,15 @@ class BlockPool:
195
token_ids=[],
196
block_size=self._block_size,
197
allocator=self._allocator,
197
- block_id=None))
198
-
199
- def init_block(self, prev_block: Optional[Block], token_ids: List[int],
200
- block_size: int, physical_block_id: Optional[int]) -> Block:
198
+ block_id=None,
199
+ extra_hash=None))
200
+
201
+ def init_block(self,
202
+ prev_block: Optional[Block],
203
+ token_ids: List[int],
204
+ block_size: int,
205
+ physical_block_id: Optional[int],
206
+ extra_hash: Optional[int] = None) -> Block:
207
if len(self._free_ids) == 0:
208
self.increase_pool()
209
assert len(self._free_ids) > 0
@@ -210,7 +216,8 @@ class BlockPool:
216
token_ids=token_ids,
217
block_size=block_size,
218
allocator=block._allocator, # type: ignore[attr-defined]
213
- block_id=physical_block_id)
219
+ block_id=physical_block_id,
220
+ extra_hash=extra_hash)
221
block.pool_id = pool_id # type: ignore[attr-defined]
222
return block
223
vllm/core/block/cpu_gpu_block_allocator.py
+32
-11
@@ -121,23 +121,32 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
121
self.allocate_mutable_block(None, Device.GPU))
122
return self._null_block
123
124
- def allocate_mutable_block(self, prev_block: Optional[Block],
125
- device: Device) -> Block:
124
+ def allocate_mutable_block(self,
125
+ prev_block: Optional[Block],
126
+ device: Device,
127
+ extra_hash: Optional[int] = None) -> Block:
128
"""Allocates a new mutable block on the specified device.
129
130
Args:
131
prev_block (Optional[Block]): The previous block to in the sequence.
132
Used for prefix hashing.
133
device (Device): The device on which to allocate the new block.
134
+ extra_hash (Optional[int]): The hash value of additional
135
+ factors, such as adapters, that influence the block hash
136
+ in the prefix caching block.
137
138
Returns:
139
Block: The newly allocated mutable block.
140
"""
136
- return self._allocators[device].allocate_mutable_block(prev_block)
137
-
138
- def allocate_immutable_blocks(self, prev_block: Optional[Block],
139
- block_token_ids: List[List[int]],
140
- device: Device) -> List[Block]:
141
+ return self._allocators[device].allocate_mutable_block(
142
+ prev_block, extra_hash=extra_hash)
143
+
144
+ def allocate_immutable_blocks(
145
+ self,
146
+ prev_block: Optional[Block],
147
+ block_token_ids: List[List[int]],
148
+ device: Device,
149
+ extra_hash: Optional[int] = None) -> List[Block]:
150
"""Allocates a new group of immutable blocks with the provided block
151
token IDs on the specified device.
152
@@ -147,17 +156,22 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
156
block_token_ids (List[int]): The list of block token IDs to be
157
stored in the new blocks.
158
device (Device): The device on which to allocate the new block.
159
+ extra_hash (Optional[int]): The hash value of additional
160
+ factors, such as adapters, that influence the block hash
161
+ in the prefix caching block.
162
163
Returns:
164
List[Block]: The newly allocated list of immutable blocks
165
containing the provided block token IDs.
166
"""
167
return self._allocators[device].allocate_immutable_blocks(
156
- prev_block, block_token_ids)
168
+ prev_block, block_token_ids, extra_hash=extra_hash)
169
158
- def allocate_immutable_block(self, prev_block: Optional[Block],
170
+ def allocate_immutable_block(self,
171
+ prev_block: Optional[Block],
172
token_ids: List[int],
160
- device: Device) -> Block:
173
+ device: Device,
174
+ extra_hash: Optional[int] = None) -> Block:
175
"""Allocates a new immutable block with the provided token IDs on the
176
specified device.
177
@@ -167,13 +181,16 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
181
token_ids (List[int]): The list of token IDs to be stored in the new
182
block.
183
device (Device): The device on which to allocate the new block.
184
+ extra_hash (Optional[int]): The hash value of additional
185
+ factors, such as adapters, that influence the block hash
186
+ in the prefix caching block.
187
188
Returns:
189
Block: The newly allocated immutable block containing the provided
190
token IDs.
191
"""
192
return self._allocators[device].allocate_immutable_block(
176
- prev_block, token_ids)
193
+ prev_block, token_ids, extra_hash=extra_hash)
194
195
def free(self, block: Block) -> None:
196
"""Frees the memory occupied by the given block.
@@ -387,6 +404,10 @@ class NullBlock(Block):
404
def prev_block(self):
405
return self._proxy.prev_block
406
407
+ @property
408
+ def extra_hash(self):
409
+ return None
410
+
411
@property
412
def computed(self):
413
return self._proxy.computed
vllm/core/block/interfaces.py
+23
-9
@@ -50,6 +50,11 @@ class Block(ABC):
50
def prev_block(self) -> Optional["Block"]:
51
pass
52
53
+ @property
54
+ @abstractmethod
55
+ def extra_hash(self) -> Optional[int]:
56
+ return None
57
+
58
@property
59
@abstractmethod
60
def computed(self) -> bool:
@@ -81,6 +86,8 @@ class Block(ABC):
86
block_size: int,
87
allocator: "BlockAllocator",
88
block_id: Optional[int] = None,
89
+ computed: bool = False,
90
+ extra_hash: Optional[int] = None,
91
) -> "Block":
92
pass
93
@@ -99,18 +106,20 @@ class Block(ABC):
106
class BlockAllocator(ABC):
107
108
@abstractmethod
102
- def allocate_mutable_block(self, prev_block: Optional[Block]) -> Block:
109
+ def allocate_mutable_block(self, prev_block: Optional[Block],
110
+ extra_hash: Optional[int]) -> Block:
111
pass
112
113
@abstractmethod
114
def allocate_immutable_block(self, prev_block: Optional[Block],
107
- token_ids: List[int]) -> Block:
115
+ token_ids: List[int],
116
+ extra_hash: Optional[int]) -> Block:
117
pass
118
119
@abstractmethod
111
- def allocate_immutable_blocks(
112
- self, prev_block: Optional[Block],
113
- block_token_ids: List[List[int]]) -> List[Block]:
120
+ def allocate_immutable_blocks(self, prev_block: Optional[Block],
121
+ block_token_ids: List[List[int]],
122
+ extra_hash: Optional[int]) -> List[Block]:
123
pass
124
125
@abstractmethod
@@ -197,14 +206,18 @@ class BlockAllocator(ABC):
206
class DeviceAwareBlockAllocator(ABC):
207
208
@abstractmethod
200
- def allocate_mutable_block(self, prev_block: Optional[Block],
201
- device: Device) -> Block:
209
+ def allocate_mutable_block(self,
210
+ prev_block: Optional[Block],
211
+ device: Device,
212
+ extra_hash: Optional[int] = None) -> Block:
213
pass
214
215
@abstractmethod
205
- def allocate_immutable_block(self, prev_block: Optional[Block],
216
+ def allocate_immutable_block(self,
217
+ prev_block: Optional[Block],
218
token_ids: List[int],
207
- device: Device) -> Block:
219
+ device: Device,
220
+ extra_hash: Optional[int] = None) -> Block:
221
pass
222
223
@abstractmethod
@@ -213,6 +226,7 @@ class DeviceAwareBlockAllocator(ABC):
226
prev_block: Optional[Block],
227
block_token_ids: List[List[int]],
228
device: Device,
229
+ extra_hash: Optional[int] = None,
230
) -> List[Block]:
231
pass
232
vllm/core/block/naive_block.py
+9
-1
@@ -63,6 +63,7 @@ class NaiveBlockAllocator(BlockAllocator):
63
def allocate_immutable_block(self,
64
prev_block: Optional[Block],
65
token_ids: List[int],
66
+ extra_hash: Optional[int] = None,
67
device: Optional[Device] = None) -> Block:
68
"""Allocates a new immutable block with the given token IDs, linked to
69
the previous block.
@@ -85,6 +86,7 @@ class NaiveBlockAllocator(BlockAllocator):
86
self,
87
prev_block: Optional[Block],
88
block_token_ids: List[List[int]],
89
+ extra_hash: Optional[int] = None,
90
device: Optional[Device] = None) -> List[Block]:
91
assert device is None
92
num_blocks = len(block_token_ids)
@@ -106,6 +108,7 @@ class NaiveBlockAllocator(BlockAllocator):
108
109
def allocate_mutable_block(self,
110
prev_block: Optional[Block],
111
+ extra_hash: Optional[int] = None,
112
device: Optional[Device] = None) -> Block:
113
"""Allocates a new mutable block, linked to the previous block.
114
@@ -355,7 +358,8 @@ class NaiveBlock(Block):
358
block_size: int,
359
allocator: BlockAllocator,
360
block_id: Optional[int] = None,
358
- _cow_target: Optional[Block] = None):
361
+ _cow_target: Optional[Block] = None,
362
+ extra_hash: Optional[int] = None):
363
self._token_ids: List[int] = []
364
self._block_size = block_size
365
self._prev_block = prev_block
@@ -441,6 +445,10 @@ class NaiveBlock(Block):
445
def prev_block(self) -> Optional["Block"]:
446
return self._prev_block
447
448
+ @property
449
+ def extra_hash(self):
450
+ return None
451
+
452
@property
453
def content_hash(self) -> Optional[int]:
454
return None
vllm/core/block/prefix_caching_block.py
+42
-13
@@ -126,6 +126,7 @@ class PrefixCachingBlockAllocator(BlockAllocator):
126
allocator: BlockAllocator,
127
block_id: Optional[int] = None,
128
computed: bool = False,
129
+ extra_hash: Optional[int] = None,
130
) -> Block:
131
# Bind block to self.
132
allocator = self
@@ -137,11 +138,13 @@ class PrefixCachingBlockAllocator(BlockAllocator):
138
block_id=block_id,
139
allocator=allocator,
140
computed=computed,
141
+ extra_hash=extra_hash,
142
)
143
144
def allocate_immutable_block(self,
145
prev_block: Optional[Block],
146
token_ids: List[int],
147
+ extra_hash: Optional[int] = None,
148
device: Optional[Device] = None) -> Block:
149
"""Allocates an immutable block with the given token IDs, reusing cached
150
blocks if possible.
@@ -160,7 +163,8 @@ class PrefixCachingBlockAllocator(BlockAllocator):
163
block = self._block_pool.init_block(prev_block=prev_block,
164
token_ids=token_ids,
165
block_size=self._block_size,
163
- physical_block_id=None)
166
+ physical_block_id=None,
167
+ extra_hash=extra_hash)
168
assert block.content_hash is not None
169
170
cached_block_id = self._cached_blocks.get(block.content_hash, None)
@@ -173,7 +177,7 @@ class PrefixCachingBlockAllocator(BlockAllocator):
177
self._block_pool.free_block(block)
178
179
# No cached block => Allocate a new block
176
- block = self.allocate_mutable_block(prev_block)
180
+ block = self.allocate_mutable_block(prev_block, extra_hash=extra_hash)
181
block.append_token_ids(token_ids)
182
return block
183
@@ -181,17 +185,20 @@ class PrefixCachingBlockAllocator(BlockAllocator):
185
self,
186
prev_block: Optional[Block],
187
block_token_ids: List[List[int]],
188
+ extra_hash: Optional[int] = None,
189
device: Optional[Device] = None) -> List[Block]:
190
blocks = []
191
for token_ids in block_token_ids:
192
prev_block = self.allocate_immutable_block(prev_block=prev_block,
193
token_ids=token_ids,
189
- device=device)
194
+ device=device,
195
+ extra_hash=extra_hash)
196
blocks.append(prev_block)
197
return blocks
198
199
def allocate_mutable_block(self,
200
prev_block: Optional[Block],
201
+ extra_hash: Optional[int] = None,
202
device: Optional[Device] = None) -> Block:
203
"""Allocates a mutable block. If there are no free blocks, this will
204
evict unused cached blocks.
@@ -210,7 +217,8 @@ class PrefixCachingBlockAllocator(BlockAllocator):
217
block = self._block_pool.init_block(prev_block=prev_block,
218
token_ids=[],
219
block_size=self._block_size,
213
- physical_block_id=block_id)
220
+ physical_block_id=block_id,
221
+ extra_hash=extra_hash)
222
assert not block.computed
223
assert block.content_hash is None
224
return block
@@ -382,7 +390,8 @@ class PrefixCachingBlockAllocator(BlockAllocator):
390
prev_block=prev_block,
391
token_ids=block.token_ids,
392
block_size=self._block_size,
385
- physical_block_id=block_id)
393
+ physical_block_id=block_id,
394
+ extra_hash=block.extra_hash)
395
396
forked_blocks.append(forked_block)
397
prev_block = forked_blocks[-1]
@@ -608,10 +617,12 @@ class PrefixCachingBlockAllocator(BlockAllocator):
617
# existing "block" object
618
if block.is_full:
619
tmp_block = self.allocate_immutable_block(
611
- prev_block=block.prev_block, token_ids=block.token_ids)
620
+ prev_block=block.prev_block,
621
+ token_ids=block.token_ids,
622
+ extra_hash=block.extra_hash)
623
else:
624
tmp_block = self.allocate_mutable_block(
614
- prev_block=block.prev_block)
625
+ prev_block=block.prev_block, extra_hash=block.extra_hash)
626
tmp_block.append_token_ids(block.token_ids)
627
628
block_id = tmp_block.block_id
@@ -679,6 +690,8 @@ class PrefixCachingBlock(Block):
690
caching block allocator associated with this block.
691
block_id (Optional[int], optional): The physical block index
692
of this block. Defaults to None.
693
+ extra_hash (Optional[int]): The hash value of additional factors
694
+ such as adapters that influence the block, apart from the token_ids.
695
"""
696
697
def __init__(
@@ -689,6 +702,7 @@ class PrefixCachingBlock(Block):
702
allocator: BlockAllocator,
703
block_id: Optional[int] = None,
704
computed: bool = False,
705
+ extra_hash: Optional[int] = None,
706
):
707
assert isinstance(allocator, PrefixCachingBlockAllocator), (
708
"Currently this class is only tested with "
@@ -702,6 +716,7 @@ class PrefixCachingBlock(Block):
716
self._allocator = allocator
717
self._last_accessed: float = _DEFAULT_LAST_ACCESSED_TIME
718
self._computed = computed
719
+ self._extra_hash = extra_hash
720
721
# On the first time, we create the block object, and next we only
722
# reinitialize it
@@ -811,6 +826,10 @@ class PrefixCachingBlock(Block):
826
def prev_block(self) -> Optional[Block]:
827
return self._prev_block
828
829
+ @property
830
+ def extra_hash(self) -> Optional[int]:
831
+ return self._extra_hash
832
+
833
@property
834
def content_hash(self) -> Optional[int]:
835
"""Return the content-based hash of the current block, or None if it is
@@ -841,18 +860,19 @@ class PrefixCachingBlock(Block):
860
self._cached_content_hash = PrefixCachingBlock.hash_block_tokens(
861
is_first_block,
862
prev_block_hash,
844
- cur_block_token_ids=self.token_ids)
863
+ cur_block_token_ids=self.token_ids,
864
+ extra_hash=self._extra_hash)
865
return self._cached_content_hash
866
867
@staticmethod
848
- def hash_block_tokens(is_first_block: bool, prev_block_hash: Optional[int],
849
- cur_block_token_ids: List[int]) -> int:
868
+ def hash_block_tokens(is_first_block: bool,
869
+ prev_block_hash: Optional[int],
870
+ cur_block_token_ids: List[int],
871
+ extra_hash: Optional[int] = None) -> int:
872
"""Computes a hash value corresponding to the contents of a block and
873
the contents of the preceding block(s). The hash value is used for
874
prefix caching.
875
854
- NOTE: Content-based hashing does not yet support LoRA.
855
-
876
Parameters:
877
- is_first_block (bool): A flag indicating if the block is the first in
878
the sequence.
@@ -860,12 +880,15 @@ class PrefixCachingBlock(Block):
880
if this is the first block.
881
- cur_block_token_ids (List[int]): A list of token ids in the current
882
block. The current block is assumed to be full.
883
+ - extra_hash (Optional[int]): The hash value of additional factors
884
+ such as adapters that influence the block, apart from the token_ids.
885
886
Returns:
887
- int: The computed hash value for the block.
888
"""
889
assert (prev_block_hash is None) == is_first_block
868
- return hash((is_first_block, prev_block_hash, *cur_block_token_ids))
890
+ return hash((is_first_block, prev_block_hash, *cur_block_token_ids,
891
+ extra_hash))
892
893
894
class ComputedBlocksTracker:
@@ -935,12 +958,18 @@ class ComputedBlocksTracker:
958
assert len(token_ids) >= (i + 1) * self._block_size
959
block_token_ids = token_ids[i * self._block_size:(i + 1) *
960
self._block_size]
961
+
962
+ # NOTE: If there are any factors affecting the block besides
963
+ # token_ids, they should be added as input to extra_hash.
964
+ extra_hash = seq.extra_hash()
965
+
966
# This has to be kept in sync with the allocator's hash
967
# calculation.
968
block_hash = PrefixCachingBlock.hash_block_tokens(
969
is_first_block=prev_block_hash is None,
970
prev_block_hash=prev_block_hash,
971
cur_block_token_ids=block_token_ids,
972
+ extra_hash=extra_hash,
973
)
974
block_hashes_recorded.append(block_hash)
975
prev_block_hash = block_hash
vllm/core/block_manager.py
+7
-1
@@ -151,8 +151,13 @@ class SelfAttnBlockSpaceManager(BlockSpaceManager):
151
max_block_sliding_window=self.max_block_sliding_window,
152
)
153
if seq.get_token_ids():
154
+ # NOTE: If there are any factors affecting the block besides
155
+ # token_ids, they should be added as input to extra_hash.
156
+ extra_hash = seq.extra_hash()
157
+
158
# Add blocks to the block table only if the sequence is non empty.
155
- block_table.allocate(seq.get_token_ids())
159
+ block_table.allocate(token_ids=seq.get_token_ids(),
160
+ extra_hash=extra_hash)
161
162
return block_table
163
@@ -238,6 +243,7 @@ class SelfAttnBlockSpaceManager(BlockSpaceManager):
243
token_ids=block_table.get_unseen_token_ids(seq.get_token_ids()),
244
num_lookahead_slots=num_lookahead_slots,
245
num_computed_slots=seq.data.get_num_computed_tokens(),
246
+ extra_hash=seq.extra_hash(),
247
)
248
# Return any new copy-on-writes.
249
new_cows = self.block_allocator.clear_copy_on_writes()
vllm/sequence.py
+13
@@ -527,6 +527,19 @@ class Sequence:
527
hashed_tokens = self.data.get_prefix_token_ids(num_tokens)
528
return hash((hashed_tokens, self.lora_int_id))
529
530
+ def extra_hash(self) -> Optional[int]:
531
+ """
532
+ This function computes an extra hash for a sequence, specifically
533
+ designed for prefix caching mode. The final sequence hash is determined
534
+ by applying token_ids from the sequence's blocks.
535
+ """
536
+ if self.prompt_adapter_id == 0 and self.lora_int_id == 0:
537
+ return None
538
+
539
+ # NOTE: If there are additional factors influencing the block aside from
540
+ # token_ids, include them as input parameters to the hash.
541
+ return hash((self.prompt_adapter_id, self.lora_int_id))
542
+
543
def num_hashed_tokens_of_block(self, logical_idx: int):
544
return logical_idx * self.block_size + self.block_size
545