A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Add Automatic Prefix Caching (#2762)
Co-authored-by: ElizaWszola <eliza@neuralmagic.com> Co-authored-by: Michael Goin <michael@neuralmagic.com>
Sage Moore committed
Mar 2, 2024 at 03:50 UTC
ce4f5a29fb3e35041842518fefe999847b8326b9
18 files changed
+618
-292
benchmarks/benchmark_throughput.py
+16
-14
@@ -73,21 +73,21 @@ def run_vllm(
73
enforce_eager: bool,
74
kv_cache_dtype: str,
75
device: str,
76
+ enable_prefix_caching: bool,
77
) -> float:
78
from vllm import LLM, SamplingParams
78
- llm = LLM(
79
- model=model,
80
- tokenizer=tokenizer,
81
- quantization=quantization,
82
- tensor_parallel_size=tensor_parallel_size,
83
- seed=seed,
84
- trust_remote_code=trust_remote_code,
85
- dtype=dtype,
86
- max_model_len=max_model_len,
87
- enforce_eager=enforce_eager,
88
- kv_cache_dtype=kv_cache_dtype,
89
- device=device,
90
- )
79
+ llm = LLM(model=model,
80
+ tokenizer=tokenizer,
81
+ quantization=quantization,
82
+ tensor_parallel_size=tensor_parallel_size,
83
+ seed=seed,
84
+ trust_remote_code=trust_remote_code,
85
+ dtype=dtype,
86
+ max_model_len=max_model_len,
87
+ enforce_eager=enforce_eager,
88
+ kv_cache_dtype=kv_cache_dtype,
89
+ device=device,
90
+ enable_prefix_caching=enable_prefix_caching)
91
92
# Add the requests to the engine.
93
for prompt, _, output_len in requests:
@@ -211,7 +211,8 @@ def main(args: argparse.Namespace):
211
args.seed, args.n, args.use_beam_search,
212
args.trust_remote_code, args.dtype,
213
args.max_model_len, args.enforce_eager,
214
- args.kv_cache_dtype, args.device)
214
+ args.kv_cache_dtype, args.device,
215
+ args.enable_prefix_caching)
216
elif args.backend == "hf":
217
assert args.tensor_parallel_size == 1
218
elapsed_time = run_hf(requests, args.model, tokenizer, args.n,
@@ -302,6 +303,7 @@ if __name__ == "__main__":
303
default="cuda",
304
choices=["cuda"],
305
help='device type for vLLM execution, supporting CUDA only currently.')
306
+ parser.add_argument("--enable_prefix_caching", action='store_true')
307
args = parser.parse_args()
308
if args.tokenizer is None:
309
args.tokenizer = args.model
docs/source/models/engine_args.rst
+4
@@ -81,6 +81,10 @@ Below, you can find an explanation of every engine argument for vLLM:
81
82
Token block size for contiguous chunks of tokens.
83
84
+.. option:: --enable-prefix-caching
85
+
86
+ Enables automatic prefix caching
87
+
88
.. option:: --seed <seed>
89
90
Random seed for operations.
examples/offline_inference_with_prefix.py
+2
-9
@@ -37,20 +37,13 @@ for output in outputs:
37
38
print("-" * 80)
39
40
-# -1 since the last token can change when concatenating prompts.
41
-prefix_pos = len(llm.llm_engine.tokenizer.encode(prefix)) - 1
42
-
40
# The llm.generate call will batch all prompts and send the batch at once if resources allow.
41
# The prefix will only be cached after the first batch is processed, so we need to call generate once
42
# to calculate the prefix and cache it.
46
-outputs = llm.generate(generating_prompts[0],
47
- sampling_params,
48
- prefix_pos=[prefix_pos])
43
+outputs = llm.generate(generating_prompts[0], sampling_params)
44
45
# Subsequent batches can leverage the cached prefix
51
-outputs = llm.generate(generating_prompts,
52
- sampling_params,
53
- prefix_pos=[prefix_pos] * len(generating_prompts))
46
+outputs = llm.generate(generating_prompts, sampling_params)
47
48
# Print the outputs. You should see the same outputs as before
49
for output in outputs:
tests/prefix_caching/test_prefix_caching.py
+69
-34
@@ -4,38 +4,73 @@ Run `pytest tests/prefix_caching/test_prefix_caching.py`.
4
"""
5
import pytest
6
7
-from vllm import LLM, SamplingParams
8
-
9
-prefix = (
10
- "You are an expert school principal, skilled in effectively managing "
11
- "faculty and staff. Draft 10-15 questions for a potential first grade "
12
- "Head Teacher for my K-12, all-girls', independent school that emphasizes "
13
- "community, joyful discovery, and life-long learning. The candidate is "
14
- "coming in for a first-round panel interview for a 8th grade Math "
15
- "teaching role. They have 5 years of previous teaching experience "
16
- "as an assistant teacher at a co-ed, public school with experience "
17
- "in middle school math teaching. Based on these information, fulfill "
18
- "the following paragraph: ")
19
-
20
-
21
-@pytest.mark.parametrize("model", ["facebook/opt-125m"])
22
-@pytest.mark.parametrize("max_tokens", [16])
23
-def test_prefix_caching(
24
- example_prompts,
25
- model: str,
26
- max_tokens: int,
7
+from vllm.core.block_manager import BlockAllocator
8
+from vllm.utils import Device
9
+
10
+
11
+@pytest.mark.parametrize("block_size", [16])
12
+@pytest.mark.parametrize("num_blocks", [16])
13
+def test_block_allocator(
14
+ block_size: int,
15
+ num_blocks: int,
16
):
28
- llm = LLM(model=model)
29
- # -1 since the last token can change when concatenating prompts.
30
- prefix_pos = len(llm.llm_engine.tokenizer.encode(prefix)) - 1
31
- prompts = [prefix + prompt for prompt in example_prompts]
32
- sampling_params = SamplingParams(temperature=0.0, max_tokens=max_tokens)
33
- outputs_without_prefix = llm.generate(prompts, sampling_params)
34
- outputs_with_prefix = llm.generate(prompts,
35
- sampling_params,
36
- prefix_pos=[prefix_pos] * len(prompts))
37
- for output_without_prefix, output_with_prefix in zip(
38
- outputs_without_prefix, outputs_with_prefix):
39
- assert (output_without_prefix.outputs[0].token_ids ==
40
- output_with_prefix.outputs[0].token_ids)
41
- assert len(llm.llm_engine.scheduler.prefix_pool.prefixes) == 1
17
+ block_hash = 1
18
+ block_allocator = BlockAllocator(Device.CPU,
19
+ block_size,
20
+ num_blocks,
21
+ enable_caching=True)
22
+
23
+ # Allocate two PysicalTokenBlocks with the same hash and check that they are the same PhysicalTokenBlock
24
+ first_block = block_allocator.allocate(block_hash, 0)
25
+ second_block = block_allocator.allocate(block_hash, 0)
26
+ assert (first_block == second_block)
27
+ assert (second_block.ref_count == 2)
28
+
29
+ # Free the first_block and confirm that the ref_count is correctly decremented on the second block
30
+ block_allocator.free(first_block)
31
+ assert (second_block.ref_count == 1)
32
+
33
+ # Free the second block
34
+ block_allocator.free(second_block)
35
+
36
+ # Reallocate the first block and confirm that, even after the block had its ref_count go to 0, we still get the same block back
37
+ first_block = block_allocator.allocate(block_hash, 0)
38
+ assert (first_block == second_block)
39
+ assert (first_block.block_hash == block_hash)
40
+
41
+
42
+@pytest.mark.parametrize("num_blocks", [16])
43
+def test_eviction(num_blocks: int, ):
44
+ block_size = 16
45
+ block_allocator = BlockAllocator(Device.CPU,
46
+ block_size,
47
+ num_blocks,
48
+ enable_caching=True)
49
+ blocks = []
50
+
51
+ for i in range(num_blocks):
52
+ # use i as the block_hash
53
+ blocks.append(block_allocator.allocate(i, 0))
54
+
55
+ #Free all blocks
56
+ for block in blocks:
57
+ block_allocator.free(block)
58
+
59
+ # Allocate a new block and confirm that it's the first block freed. I.E The Least Recently Used block
60
+ new_block_hash = block_size
61
+ new_block = block_allocator.allocate(new_block_hash, 0)
62
+ assert (new_block == blocks[0])
63
+ assert (new_block.block_hash == new_block_hash)
64
+
65
+ # Reallocate the second in blocks to remove it from the free list
66
+ realloc_block_hash = 1
67
+ realloc_block = block_allocator.allocate(realloc_block_hash, 0)
68
+ assert (realloc_block == blocks[realloc_block_hash])
69
+ assert (realloc_block.block_hash == realloc_block_hash)
70
+
71
+ # Allocate a new block and confirm that it's not the realloc_block, since the realloc_block shouldn't be in the free list
72
+ new_block_hash = block_size + 1
73
+ new_block = block_allocator.allocate(new_block_hash, 0)
74
+ assert (realloc_block != new_block)
75
+ assert (new_block.block_hash == new_block_hash)
76
+ assert (new_block.block_number == 2)
tests/test_cache_block_hashing.py
new
+76
@@ -0,0 +1,76 @@
1
+"""Test hashing of cache blocks.
2
+
3
+Run `pytest tests/test_cache_block_hashing.py`.
4
+"""
5
+import pytest
6
+
7
+from vllm.transformers_utils.tokenizer import TokenizerGroup
8
+from vllm.sequence import Sequence
9
+
10
+# Make two prefixes with different first blocks.
11
+prefix_start = [("You are an expert"), ("You are a")]
12
+prefix_common = (
13
+ " school principal, skilled in effectively managing "
14
+ "faculty and staff. Draft 10-15 questions for a potential first grade "
15
+ "Head Teacher for my K-12, all-girls', independent school that emphasizes "
16
+ "community, joyful discovery, and life-long learning. The candidate is "
17
+ "coming in for a first-round panel interview for a 8th grade Math "
18
+ "teaching role. They have 5 years of previous teaching experience "
19
+ "as an assistant teacher at a co-ed, public school with experience "
20
+ "in middle school math teaching. Based on this, fulfill "
21
+ "the following: ")
22
+prefixes = [start + prefix_common for start in prefix_start]
23
+
24
+# Sample prompts.
25
+sample_prompts = [
26
+ "Hello, my name is", "The president of the United States is",
27
+ "The capital of France is", "The future of AI is"
28
+]
29
+
30
+
31
+# Helper function.
32
+def flatten_2d(li):
33
+ return [lss for ls in li for lss in ls]
34
+
35
+
36
+@pytest.mark.parametrize("model", ["facebook/opt-125m"])
37
+@pytest.mark.parametrize("block_size", [16])
38
+@pytest.mark.parametrize("max_num_seqs", [256])
39
+def test_auto_prefix_caching(model: str, block_size: int, max_num_seqs: int):
40
+
41
+ tokenizer = TokenizerGroup(
42
+ tokenizer_id="facebook/opt-125m",
43
+ enable_lora=False,
44
+ max_num_seqs=max_num_seqs,
45
+ max_input_length=None,
46
+ )
47
+
48
+ hashes = []
49
+
50
+ for prefix in prefixes:
51
+ hashes.append([])
52
+ prompts = [prefix + prompt for prompt in sample_prompts]
53
+ seq_id = 0
54
+ for prompt in prompts:
55
+ hashes[-1].append([])
56
+ prompt_token_ids = tokenizer.encode(prompt)
57
+ seq = Sequence(seq_id, prompt, prompt_token_ids, block_size)
58
+
59
+ num_blocks = len(prompt_token_ids) // block_size
60
+ for idx in range(num_blocks):
61
+ hashes[-1][-1].append(seq.hash_of_block(idx))
62
+
63
+ seq_id += 1
64
+
65
+ # Check that hashes made with two prefixes with different first blocks are
66
+ # different everywhere.
67
+ for hash0, hash1 in zip(flatten_2d(hashes[0]), flatten_2d(hashes[1])):
68
+ assert (hash0 != hash1)
69
+
70
+ # Check that hashes of different prompts made with the same prefix are the
71
+ # same until the hashes that contain the prompt.
72
+ for hash_pref in hashes:
73
+ same_hashes = [tuple(h[:-1]) for h in hash_pref]
74
+ different_hashes = [h[-1] for h in hash_pref]
75
+ assert (len(set(same_hashes)) == 1)
76
+ assert (len(set(different_hashes)) == len(different_hashes))
vllm/block.py
+13
-1
@@ -5,6 +5,8 @@ from vllm.utils import Device
5
6
_BLANK_TOKEN_ID = -1
7
8
+DEFAULT_LAST_ACCESSED_TIME = -1
9
+
10
11
class LogicalTokenBlock:
12
"""A block that stores a contiguous chunk of tokens from left to right.
@@ -55,17 +57,27 @@ class PhysicalTokenBlock:
57
device: Device,
58
block_number: int,
59
block_size: int,
60
+ block_hash: int,
61
+ num_hashed_tokens: int,
62
) -> None:
63
self.device = device
64
self.block_number = block_number
65
self.block_size = block_size
66
+ self.block_hash = block_hash
67
+ self.num_hashed_tokens = num_hashed_tokens
68
69
self.ref_count = 0
70
+ self.last_accessed = DEFAULT_LAST_ACCESSED_TIME
71
+
72
+ self.computed = False
73
74
def __repr__(self) -> str:
75
return (f'PhysicalTokenBlock(device={self.device}, '
76
f'block_number={self.block_number}, '
68
- f'ref_count={self.ref_count})')
77
+ f'num_hashed_tokens={self.num_hashed_tokens}, '
78
+ f'ref_count={self.ref_count}, '
79
+ f'last_accessed={self.last_accessed}, '
80
+ f'computed={self.computed})')
81
82
83
# Mapping: logical block number -> physical block.
vllm/config.py
+2
@@ -303,12 +303,14 @@ class CacheConfig:
303
swap_space: int,
304
cache_dtype: str,
305
sliding_window: Optional[int] = None,
306
+ enable_prefix_caching: bool = False,
307
) -> None:
308
self.block_size = block_size
309
self.gpu_memory_utilization = gpu_memory_utilization
310
self.swap_space_bytes = swap_space * _GB
311
self.cache_dtype = cache_dtype
312
self.sliding_window = sliding_window
313
+ self.enable_prefix_caching = enable_prefix_caching
314
self._verify_args()
315
self._verify_cache_dtype()
316
vllm/core/block_manager.py
+213
-72
@@ -1,10 +1,13 @@
1
"""A block manager that manages token blocks."""
2
import enum
3
+from itertools import count
4
+from os.path import commonprefix
5
from typing import Dict, List, Optional, Set, Tuple
6
7
from vllm.block import BlockTable, PhysicalTokenBlock
8
from vllm.sequence import Sequence, SequenceGroup, SequenceStatus
9
from vllm.utils import Device
10
+from vllm.core.evictor import Evictor, EvictionPolicy, make_evictor
11
12
13
class BlockAllocator:
@@ -15,29 +18,68 @@ class BlockAllocator:
18
the reference count becomes zero, the block is added back to the free list.
19
"""
20
18
- def __init__(
19
- self,
20
- device: Device,
21
- block_size: int,
22
- num_blocks: int,
23
- ) -> None:
21
+ def __init__(self,
22
+ device: Device,
23
+ block_size: int,
24
+ num_blocks: int,
25
+ eviction_policy: EvictionPolicy = EvictionPolicy.LRU,
26
+ enable_caching: bool = False) -> None:
27
self.device = device
28
self.block_size = block_size
29
self.num_blocks = num_blocks
30
+ self.enable_caching = enable_caching
31
+
32
+ self.current_num_blocks = 0
33
+ self.cached_blocks: Dict[int, PhysicalTokenBlock] = {}
34
+
35
+ # Switch over to FIFO eviction when caching is disabled
36
+ if not self.enable_caching:
37
+ eviction_policy = EvictionPolicy.FIFO
38
+ self.evictor: Evictor = make_evictor(eviction_policy)
39
+
40
+ self.default_hash_ctr = count()
41
+
42
+ def allocate_block(self, block_hash: int,
43
+ num_hashed_tokens: int) -> PhysicalTokenBlock:
44
+ if self.current_num_blocks == self.num_blocks:
45
+ block = self.evictor.evict()
46
+ block.block_hash = block_hash
47
+ block.num_hashed_tokens = num_hashed_tokens
48
+ return block
49
+ block = PhysicalTokenBlock(device=self.device,
50
+ block_number=self.current_num_blocks,
51
+ block_size=self.block_size,
52
+ block_hash=block_hash,
53
+ num_hashed_tokens=num_hashed_tokens)
54
+ self.current_num_blocks += 1
55
+ return block
56
28
- # Initialize the free blocks.
29
- self.free_blocks: BlockTable = []
30
- for i in range(num_blocks):
31
- block = PhysicalTokenBlock(device=device,
32
- block_number=i,
33
- block_size=block_size)
34
- self.free_blocks.append(block)
35
-
36
- def allocate(self) -> PhysicalTokenBlock:
37
- if not self.free_blocks:
38
- raise ValueError("Out of memory! No free blocks are available.")
39
- block = self.free_blocks.pop()
40
- block.ref_count = 1
57
+ def allocate(self,
58
+ block_hash: Optional[int] = None,
59
+ num_hashed_tokens: int = 0) -> PhysicalTokenBlock:
60
+ # If caching is disabled, just allocate a new block and return it
61
+ if not self.enable_caching:
62
+ block = self.allocate_block(next(self.default_hash_ctr),
63
+ num_hashed_tokens)
64
+ block.ref_count += 1
65
+ return block
66
+
67
+ if block_hash is None:
68
+ block_hash = next(self.default_hash_ctr)
69
+ if block_hash in self.evictor:
70
+ assert block_hash not in self.cached_blocks
71
+ block = self.evictor.remove(block_hash)
72
+ assert block.ref_count == 0
73
+ self.cached_blocks[block_hash] = block
74
+ block.ref_count += 1
75
+ assert block.block_hash == block_hash
76
+ return block
77
+ if block_hash not in self.cached_blocks:
78
+ self.cached_blocks[block_hash] = self.allocate_block(
79
+ block_hash, num_hashed_tokens)
80
+ block = self.cached_blocks[block_hash]
81
+ assert block.block_hash == block_hash
82
+ block.ref_count += 1
83
return block
84
85
def free(self, block: PhysicalTokenBlock) -> None:
@@ -45,10 +87,27 @@ class BlockAllocator:
87
raise ValueError(f"Double free! {block} is already freed.")
88
block.ref_count -= 1
89
if block.ref_count == 0:
48
- self.free_blocks.append(block)
90
+ assert block.block_hash not in self.evictor
91
+ self.evictor.add(block)
92
+
93
+ # If caching is enabled, remove the block from the cached_blocks
94
+ if self.enable_caching:
95
+ del self.cached_blocks[block.block_hash]
96
97
def get_num_free_blocks(self) -> int:
51
- return len(self.free_blocks)
98
+ return self.num_blocks - self.current_num_blocks + self.evictor.num_blocks
99
+
100
+ def contains_block(self, block_hash: int) -> bool:
101
+ return block_hash in self.cached_blocks or block_hash in self.evictor
102
+
103
+ def update_hash(self, block_hash: int, block: PhysicalTokenBlock):
104
+ # If caching is enabled, update the hash of block and the cached_blocks dictionary.
105
+ if self.enable_caching:
106
+ assert not self.contains_block(block_hash)
107
+ old_hash = block.block_hash
108
+ block.block_hash = block_hash
109
+ del self.cached_blocks[old_hash]
110
+ self.cached_blocks[block_hash] = block
111
112
113
class AllocStatus(enum.Enum):
@@ -75,6 +134,7 @@ class BlockSpaceManager:
134
num_cpu_blocks: int,
135
watermark: float = 0.01,
136
sliding_window: Optional[int] = None,
137
+ enable_caching: bool = False,
138
) -> None:
139
self.block_size = block_size
140
self.num_total_gpu_blocks = num_gpu_blocks
@@ -89,11 +149,17 @@ class BlockSpaceManager:
149
self.watermark = watermark
150
assert watermark >= 0.0
151
152
+ self.enable_caching = enable_caching
153
+
154
self.watermark_blocks = int(watermark * num_gpu_blocks)
93
- self.gpu_allocator = BlockAllocator(Device.GPU, block_size,
94
- num_gpu_blocks)
95
- self.cpu_allocator = BlockAllocator(Device.CPU, block_size,
96
- num_cpu_blocks)
155
+ self.gpu_allocator = BlockAllocator(Device.GPU,
156
+ block_size,
157
+ num_gpu_blocks,
158
+ enable_caching=enable_caching)
159
+ self.cpu_allocator = BlockAllocator(Device.CPU,
160
+ block_size,
161
+ num_cpu_blocks,
162
+ enable_caching=enable_caching)
163
# Mapping: seq_id -> BlockTable.
164
self.block_tables: Dict[int, BlockTable] = {}
165
@@ -103,9 +169,6 @@ class BlockSpaceManager:
169
seq = seq_group.get_seqs(status=SequenceStatus.WAITING)[0]
170
num_required_blocks = len(seq.logical_token_blocks)
171
106
- if seq_group.prefix is not None and seq_group.prefix.allocated:
107
- num_required_blocks -= seq_group.prefix.get_num_blocks()
108
-
172
if self.block_sliding_window is not None:
173
num_required_blocks = min(num_required_blocks,
174
self.block_sliding_window)
@@ -129,36 +192,16 @@ class BlockSpaceManager:
192
num_prompt_blocks = len(seq.logical_token_blocks)
193
194
block_table: BlockTable = []
132
- prefix_block_table: BlockTable = []
133
- num_prefix_blocks = 0
134
-
135
- prefix = seq_group.prefix
136
- if prefix is not None and prefix.allocated:
137
- # Prefix has already been allocated. Use the existing block table.
138
- num_prompt_blocks -= prefix.get_num_blocks()
139
- for block in prefix.block_table:
140
- block.ref_count += seq_group.num_seqs()
141
- block_table.append(block)
142
-
195
for logical_idx in range(num_prompt_blocks):
196
if (self.block_sliding_window is not None
197
and logical_idx >= self.block_sliding_window):
198
block = block_table[logical_idx % self.block_sliding_window]
199
else:
148
- block = self.gpu_allocator.allocate()
149
- # Set the reference counts of the token blocks.
150
- block.ref_count = seq_group.num_seqs()
200
+ block = self.gpu_allocator.allocate(
201
+ seq.hash_of_block(logical_idx),
202
+ seq.num_hashed_tokens_of_block(logical_idx))
203
block_table.append(block)
204
153
- if prefix is not None and not prefix.allocated:
154
- # Allocate blocks for the prefix, we will compute the prefix's
155
- # KV cache in this run.
156
- num_prefix_blocks = prefix.get_num_blocks()
157
- prefix_block_table = block_table[:num_prefix_blocks]
158
- for block in prefix_block_table:
159
- block.ref_count += 1
160
- prefix.set_block_table(prefix_block_table)
161
-
205
# Assign the block table for each sequence.
206
for seq in seq_group.get_seqs(status=SequenceStatus.WAITING):
207
self.block_tables[seq.seq_id] = block_table.copy()
@@ -170,12 +213,72 @@ class BlockSpaceManager:
213
num_seqs = seq_group.num_seqs(status=SequenceStatus.RUNNING)
214
return num_seqs <= num_free_gpu_blocks
215
173
- def append_slot(self, seq: Sequence) -> Optional[Tuple[int, int]]:
216
+ def _promote_last_block(
217
+ self,
218
+ seq: Sequence,
219
+ last_block: PhysicalTokenBlock,
220
+ ) -> PhysicalTokenBlock:
221
+ # Compute a new hash for the block so that it can be shared by other Sequences
222
+ new_hash = seq.hash_of_block(len(seq.logical_token_blocks) - 1)
223
+
224
+ # if new_hash is already in the cached table, then free last_block and return the cached version
225
+ if self.gpu_allocator.contains_block(new_hash):
226
+ self.gpu_allocator.free(last_block)
227
+ return self.gpu_allocator.allocate(new_hash)
228
+ else:
229
+ self.gpu_allocator.update_hash(new_hash, last_block)
230
+ return last_block
231
+
232
+ def _is_last_block_full(
233
+ self,
234
+ seq: Sequence,
235
+ ) -> bool:
236
+ token_ids_len = len(seq.data.get_token_ids())
237
+ return token_ids_len > 0 and token_ids_len % seq.block_size == 0
238
+
239
+ def _is_last_block(
240
+ self,
241
+ seq: Sequence,
242
+ index: int,
243
+ ) -> bool:
244
+ return index == len(seq.logical_token_blocks) - 1
245
+
246
+ def _maybe_promote_last_block(
247
+ self,
248
+ seq: Sequence,
249
+ last_block: PhysicalTokenBlock,
250
+ ) -> PhysicalTokenBlock:
251
+ if self._is_last_block_full(seq):
252
+ return self._promote_last_block(seq, last_block)
253
+ else:
254
+ return last_block
255
+
256
+ def _allocate_last_physical_block(
257
+ self,
258
+ seq: Sequence,
259
+ ) -> PhysicalTokenBlock:
260
+ block_hash: Optional[int] = None
261
+ if (self._is_last_block_full(seq)):
262
+ block_hash = seq.hash_of_block(len(seq.logical_token_blocks) - 1)
263
+ num_hashed_tokens = seq.num_hashed_tokens_of_block(
264
+ len(seq.logical_token_blocks) - 1)
265
+ new_block = self.gpu_allocator.allocate(block_hash, num_hashed_tokens)
266
+ if block_hash is None:
267
+ assert new_block.ref_count == 1
268
+ return new_block
269
+
270
+ def append_slot(
271
+ self,
272
+ seq: Sequence,
273
+ ) -> Optional[Tuple[int, int]]:
274
"""Allocate a physical slot for a new token."""
275
logical_blocks = seq.logical_token_blocks
276
block_table = self.block_tables[seq.seq_id]
177
-
277
+ # If we need to allocate a new physical block
278
if len(block_table) < len(logical_blocks):
279
+ # Currently this code only supports adding one physical block
280
+ assert len(block_table) == len(logical_blocks) - 1
281
+
282
if (self.block_sliding_window
283
and len(block_table) >= self.block_sliding_window):
284
# reuse a block
@@ -184,8 +287,8 @@ class BlockSpaceManager:
287
else:
288
# The sequence has a new logical block.
289
# Allocate a new physical block.
187
- block = self.gpu_allocator.allocate()
188
- block_table.append(block)
290
+ new_block = self._allocate_last_physical_block(seq)
291
+ block_table.append(new_block)
292
return None
293
294
# We want to append the token to the last physical block.
@@ -193,11 +296,15 @@ class BlockSpaceManager:
296
assert last_block.device == Device.GPU
297
if last_block.ref_count == 1:
298
# Not shared with other sequences. Appendable.
299
+ # If the last block is now complete, promote it to a full block so that it can be shared
300
+ new_block = self._maybe_promote_last_block(seq, last_block)
301
+ block_table[-1] = new_block
302
return None
303
else:
304
# The last block is shared with other sequences.
305
# Copy on Write: Allocate a new block and copy the tokens.
200
- new_block = self.gpu_allocator.allocate()
306
+ new_block = self._allocate_last_physical_block(seq)
307
+
308
block_table[-1] = new_block
309
self.gpu_allocator.free(last_block)
310
return last_block.block_number, new_block.block_number
@@ -233,25 +340,18 @@ class BlockSpaceManager:
340
341
def swap_in(self, seq_group: SequenceGroup) -> Dict[int, int]:
342
# CPU block -> GPU block.
236
- if seq_group.prefix is not None:
237
- # make sure to swap in the prefix first
238
- assert seq_group.prefix.allocated and seq_group.prefix.computed
239
-
343
mapping: Dict[PhysicalTokenBlock, PhysicalTokenBlock] = {}
344
for seq in seq_group.get_seqs(status=SequenceStatus.SWAPPED):
345
new_block_table: BlockTable = []
346
block_table = self.block_tables[seq.seq_id]
244
- if seq_group.prefix is not None:
245
- for block in seq_group.prefix.block_table:
246
- new_block_table.append(block)
247
- block.ref_count += 1
347
348
for cpu_block in block_table:
349
if cpu_block in mapping:
350
gpu_block = mapping[cpu_block]
351
gpu_block.ref_count += 1
352
else:
254
- gpu_block = self.gpu_allocator.allocate()
353
+ gpu_block = self.gpu_allocator.allocate(
354
+ cpu_block.block_hash, cpu_block.num_hashed_tokens)
355
mapping[cpu_block] = gpu_block
356
new_block_table.append(gpu_block)
357
# Free the CPU block swapped in to GPU.
@@ -276,17 +376,12 @@ class BlockSpaceManager:
376
block_table = self.block_tables[seq.seq_id]
377
378
for gpu_block in block_table:
279
- if (seq_group.prefix is not None
280
- and gpu_block in seq_group.prefix.block_table):
281
- # NOTE: We do not swap out the prefix blocks for now.
282
- self.gpu_allocator.free(gpu_block)
283
- continue
284
-
379
if gpu_block in mapping:
380
cpu_block = mapping[gpu_block]
381
cpu_block.ref_count += 1
382
else:
289
- cpu_block = self.cpu_allocator.allocate()
383
+ cpu_block = self.cpu_allocator.allocate(
384
+ gpu_block.block_hash, gpu_block.num_hashed_tokens)
385
mapping[gpu_block] = cpu_block
386
new_block_table.append(cpu_block)
387
# Free the GPU block swapped out to CPU.
@@ -328,3 +423,49 @@ class BlockSpaceManager:
423
424
def get_num_free_cpu_blocks(self) -> int:
425
return self.cpu_allocator.get_num_free_blocks()
426
+
427
+ def access_all_blocks_in_seq(
428
+ self,
429
+ seq: Sequence,
430
+ access_time: float,
431
+ ) -> None:
432
+ block_table = self.block_tables[seq.seq_id]
433
+ for block in block_table:
434
+ block.last_accessed = access_time
435
+
436
+ def compute_last_full_block_in_seq(self, seq: Sequence):
437
+ if seq.seq_id not in self.block_tables:
438
+ return
439
+ max_full_block = seq.get_len() // seq.block_size - 1
440
+ block_table = self.block_tables[seq.seq_id]
441
+ if max_full_block == -1:
442
+ return
443
+ block_table[max_full_block].computed = True
444
+
445
+ def get_all_block_ids_till_computed(self, seq: Sequence) -> List[int]:
446
+ if seq.seq_id not in self.block_tables:
447
+ return []
448
+ block_table = self.block_tables[seq.seq_id]
449
+ for block_idx in reversed(range(len(block_table))):
450
+ if block_table[block_idx].computed:
451
+ return [b.block_number for b in block_table[:block_idx + 1]]
452
+ return []
453
+
454
+ # Can return non-empty result only with prefix caching enabled.
455
+ def get_common_computed_block_ids(self,
456
+ seq_group: SequenceGroup) -> List[int]:
457
+ if not self.enable_caching:
458
+ return []
459
+
460
+ ids_list = [
461
+ self.get_all_block_ids_till_computed(seq)
462
+ for seq in iter(seq_group.seqs_dict.values())
463
+ ]
464
+ return commonprefix([ids for ids in ids_list if ids != []])
465
+
466
+ # We only mark the last full block because with prefix caching,
467
+ # all blocks until the marked one are guaranteed to be computed.
468
+ def mark_blocks_as_computed(self, seq_group: SequenceGroup):
469
+ if self.enable_caching:
470
+ for seq in seq_group.seqs_dict.values():
471
+ self.compute_last_full_block_in_seq(seq)
vllm/core/evictor.py
new
+161
@@ -0,0 +1,161 @@
1
+import enum
2
+from typing import Dict, List, Optional
3
+from abc import ABC, abstractmethod, abstractproperty
4
+
5
+from vllm.block import PhysicalTokenBlock
6
+
7
+
8
+class EvictionPolicy(enum.Enum):
9
+ """Enum for eviction policy used by make_evictor to instantiate the correct
10
+ Evictor subclass.
11
+ """
12
+ LRU = enum.auto()
13
+ FIFO = enum.auto()
14
+
15
+
16
+class Evictor(ABC):
17
+ """The Evictor subclasses should be used by the BlockAllocator class to
18
+ handle eviction of freed PhysicalTokenBlocks.
19
+ """
20
+
21
+ @abstractmethod
22
+ def __init__(self):
23
+ pass
24
+
25
+ @abstractmethod
26
+ def __contains__(self, block_hash: int) -> bool:
27
+ pass
28
+
29
+ @abstractmethod
30
+ def evict(self) -> PhysicalTokenBlock:
31
+ """Runs the eviction algorithm and returns the evicted block"""
32
+ pass
33
+
34
+ @abstractmethod
35
+ def add(self, block: PhysicalTokenBlock):
36
+ """Adds block to the evictor, making it a candidate for eviction"""
37
+ pass
38
+
39
+ @abstractmethod
40
+ def remove(self, block_hash: int) -> PhysicalTokenBlock:
41
+ """Simply removes the block with the hash value block_hash from the
42
+ evictor. Caller is responsible for making sure that block_hash is contained
43
+ in the evictor before calling remove. Should be used to "bring back" blocks
44
+ that have been freed but not evicted yet.
45
+ """
46
+ pass
47
+
48
+ @abstractproperty
49
+ def num_blocks(self) -> int:
50
+ pass
51
+
52
+
53
+class LRUEvictor(Evictor):
54
+ """Evicts in a least-recently-used order using the last_accessed timestamp
55
+ that's recorded in the PhysicalTokenBlock. If there are multiple blocks with
56
+ the same last_accessed time, then the one with the largest num_hashed_tokens
57
+ will be evicted. If two blocks each have the lowest last_accessed time and
58
+ highest num_hashed_tokens value, then one will be chose arbitrarily
59
+ """
60
+
61
+ def __init__(self):
62
+ self.free_table: Dict[int, PhysicalTokenBlock] = {}
63
+
64
+ def __contains__(self, block_hash: int) -> bool:
65
+ return block_hash in self.free_table
66
+
67
+ # TODO: The performance of this evict function can be optimized further.
68
+ def evict(self) -> PhysicalTokenBlock:
69
+ free_blocks: List[PhysicalTokenBlock] = list(self.free_table.values())
70
+ if len(free_blocks) == 0:
71
+ raise ValueError("No usable cache memory left")
72
+
73
+ # Find lowest timestamp
74
+ lowest_timestamp = free_blocks[0].last_accessed
75
+ for block in free_blocks:
76
+ if block.last_accessed < lowest_timestamp:
77
+ lowest_timestamp = block.last_accessed
78
+
79
+ # Find all blocks with the lowest timestamp
80
+ least_recent: List[PhysicalTokenBlock] = []
81
+ for block in free_blocks:
82
+ if block.last_accessed == lowest_timestamp:
83
+ least_recent.append(block)
84
+
85
+ # Find highest prefix count per block
86
+ highest_num_hashed_tokens = 0
87
+ for block in least_recent:
88
+ if block.num_hashed_tokens > highest_num_hashed_tokens:
89
+ highest_num_hashed_tokens = block.num_hashed_tokens
90
+
91
+ evicted_block: Optional[PhysicalTokenBlock] = None
92
+
93
+ # Find the first block with the lowest timestamp
94
+ for block in least_recent:
95
+ if block.num_hashed_tokens == highest_num_hashed_tokens:
96
+ evicted_block = block
97
+ break
98
+
99
+ assert evicted_block is not None
100
+
101
+ del self.free_table[evicted_block.block_hash]
102
+
103
+ evicted_block.computed = False
104
+ return evicted_block
105
+
106
+ def add(self, block: PhysicalTokenBlock):
107
+ self.free_table[block.block_hash] = block
108
+
109
+ def remove(self, block_hash: int) -> PhysicalTokenBlock:
110
+ if block_hash not in self.free_table:
111
+ raise ValueError(
112
+ "Attempting to remove block that's not in the evictor")
113
+ block: PhysicalTokenBlock = self.free_table[block_hash]
114
+ del self.free_table[block_hash]
115
+ return block
116
+
117
+ @property
118
+ def num_blocks(self) -> int:
119
+ return len(self.free_table)
120
+
121
+
122
+class RandomEvictor(Evictor):
123
+ """Evicts in a first-in-first-out order"""
124
+
125
+ def __init__(self):
126
+ self.free_table: Dict[int, PhysicalTokenBlock] = {}
127
+
128
+ def __contains__(self, block_hash: int) -> bool:
129
+ return block_hash in self.free_table
130
+
131
+ def evict(self) -> PhysicalTokenBlock:
132
+ if len(self.free_table) == 0:
133
+ raise ValueError("No usable cache memory left")
134
+ evicted_block = next(iter(self.free_table.values()))
135
+ evicted_block.computed = False
136
+ del self.free_table[evicted_block.block_hash]
137
+ return evicted_block
138
+
139
+ def add(self, block: PhysicalTokenBlock):
140
+ self.free_table[block.block_hash] = block
141
+
142
+ def remove(self, block_hash: int) -> PhysicalTokenBlock:
143
+ if block_hash not in self.free_table:
144
+ raise ValueError(
145
+ "Attempting to remove block that's not in the evictor")
146
+ block: PhysicalTokenBlock = self.free_table[block_hash]
147
+ del self.free_table[block_hash]
148
+ return block
149
+
150
+ @property
151
+ def num_blocks(self) -> int:
152
+ return len(self.free_table)
153
+
154
+
155
+def make_evictor(eviction_policy: EvictionPolicy) -> Evictor:
156
+ if eviction_policy == EvictionPolicy.LRU:
157
+ return LRUEvictor()
158
+ elif eviction_policy == EvictionPolicy.FIFO:
159
+ return RandomEvictor()
160
+ else:
161
+ raise ValueError(f"Unknown cache eviction policy: {eviction_policy}")
vllm/core/scheduler.py
+9
-6
@@ -10,7 +10,6 @@ from vllm.lora.request import LoRARequest
10
from vllm.logger import init_logger
11
from vllm.sequence import (Sequence, SequenceData, SequenceGroup,
12
SequenceGroupMetadata, SequenceStatus)
13
-from vllm.prefix import PrefixPool
13
14
logger = init_logger(__name__)
15
@@ -95,10 +94,8 @@ class Scheduler:
94
block_size=self.cache_config.block_size,
95
num_gpu_blocks=self.cache_config.num_gpu_blocks,
96
num_cpu_blocks=self.cache_config.num_cpu_blocks,
98
- sliding_window=self.cache_config.sliding_window)
99
-
100
- # Create the prefix pool to cache the prefixes.
101
- self.prefix_pool = PrefixPool(self.cache_config.block_size)
97
+ sliding_window=self.cache_config.sliding_window,
98
+ enable_caching=self.cache_config.enable_prefix_caching)
99
100
# Sequence groups in the WAITING state.
101
self.waiting: Deque[SequenceGroup] = deque()
@@ -374,10 +371,12 @@ class Scheduler:
371
372
seq_data: Dict[int, SequenceData] = {}
373
block_tables: Dict[int, List[int]] = {}
374
+
375
for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
376
seq_id = seq.seq_id
377
seq_data[seq_id] = seq.data
378
block_tables[seq_id] = self.block_manager.get_block_table(seq)
379
+ self.block_manager.access_all_blocks_in_seq(seq, now)
380
381
seq_group_metadata = SequenceGroupMetadata(
382
request_id=seq_group.request_id,
@@ -386,7 +385,8 @@ class Scheduler:
385
sampling_params=seq_group.sampling_params,
386
block_tables=block_tables,
387
lora_request=seq_group.lora_request,
389
- prefix=seq_group.prefix,
388
+ computed_block_nums=self.block_manager.
389
+ get_common_computed_block_ids(seq_group),
390
state=seq_group.state,
391
)
392
seq_group_metadata_list.append(seq_group_metadata)
@@ -496,3 +496,6 @@ class Scheduler:
496
blocks_to_swap_out.update(mapping)
497
for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
498
seq.status = SequenceStatus.SWAPPED
499
+
500
+ def mark_blocks_as_computed(self, seq_group: SequenceGroup):
501
+ self.block_manager.mark_blocks_as_computed(seq_group)
vllm/engine/arg_utils.py
+8
-1
@@ -25,6 +25,7 @@ class EngineArgs:
25
tensor_parallel_size: int = 1
26
max_parallel_loading_workers: Optional[int] = None
27
block_size: int = 16
28
+ enable_prefix_caching: bool = False
29
swap_space: int = 4 # GiB
30
gpu_memory_utilization: float = 0.90
31
max_num_batched_tokens: Optional[int] = None
@@ -173,6 +174,11 @@ class EngineArgs:
174
default=EngineArgs.block_size,
175
choices=[8, 16, 32, 128],
176
help='token block size')
177
+
178
+ parser.add_argument('--enable-prefix-caching',
179
+ action='store_true',
180
+ help='Enables automatic prefix caching')
181
+
182
parser.add_argument('--seed',
183
type=int,
184
default=EngineArgs.seed,
@@ -293,7 +299,8 @@ class EngineArgs:
299
cache_config = CacheConfig(self.block_size,
300
self.gpu_memory_utilization,
301
self.swap_space, self.kv_cache_dtype,
296
- model_config.get_sliding_window())
302
+ model_config.get_sliding_window(),
303
+ self.enable_prefix_caching)
304
parallel_config = ParallelConfig(self.pipeline_parallel_size,
305
self.tensor_parallel_size,
306
self.worker_use_ray,
vllm/engine/async_llm_engine.py
+1
-13
@@ -225,7 +225,6 @@ class _AsyncLLMEngine(LLMEngine):
225
prompt_token_ids: Optional[List[int]] = None,
226
arrival_time: Optional[float] = None,
227
lora_request: Optional[LoRARequest] = None,
228
- prefix_pos: Optional[int] = None,
228
) -> None:
229
if lora_request is not None and not self.lora_config:
230
raise ValueError(f"Got lora_request {lora_request} but LoRA is "
@@ -245,7 +244,6 @@ class _AsyncLLMEngine(LLMEngine):
244
sampling_params=sampling_params,
245
arrival_time=arrival_time,
246
lora_request=lora_request,
248
- prefix_pos=prefix_pos,
247
)
248
249
async def _run_workers_async(
@@ -422,7 +420,6 @@ class AsyncLLMEngine:
420
prompt_token_ids: Optional[List[int]] = None,
421
arrival_time: Optional[float] = None,
422
lora_request: Optional[LoRARequest] = None,
425
- prefix_pos: Optional[int] = None,
423
) -> AsyncStream:
424
if self.log_requests:
425
shortened_prompt = prompt
@@ -435,7 +432,6 @@ class AsyncLLMEngine:
432
max_log_len]
433
logger.info(f"Received request {request_id}: "
434
f"prompt: {shortened_prompt!r}, "
438
- f"prefix_pos: {prefix_pos},"
435
f"sampling_params: {sampling_params}, "
436
f"prompt_token_ids: {shortened_token_ids}, "
437
f"lora_request: {lora_request}.")
@@ -472,8 +468,7 @@ class AsyncLLMEngine:
468
sampling_params=sampling_params,
469
prompt_token_ids=prompt_token_ids,
470
arrival_time=arrival_time,
475
- lora_request=lora_request,
476
- prefix_pos=prefix_pos)
471
+ lora_request=lora_request)
472
473
return stream
474
@@ -484,7 +479,6 @@ class AsyncLLMEngine:
479
request_id: str,
480
prompt_token_ids: Optional[List[int]] = None,
481
lora_request: Optional[LoRARequest] = None,
487
- prefix_pos: Optional[int] = None,
482
) -> AsyncIterator[RequestOutput]:
483
"""Generate outputs for a request.
484
@@ -500,11 +494,6 @@ class AsyncLLMEngine:
494
prompt_token_ids: The token IDs of the prompt. If None, we
495
use the tokenizer to convert the prompts to token IDs.
496
lora_request: LoRA request to use for generation, if any.
503
- prefix_pos: If not None, we use the given position as the prefix
504
- position for each prompt. We will cache the prefix's KV
505
- cache and reuse it for the next request with the same prefix.
506
- This is an experimental feature, and may be replaced with
507
- automatic prefix caching in the future.
497
498
Yields:
499
The output `RequestOutput` objects from the LLMEngine for the
@@ -565,7 +554,6 @@ class AsyncLLMEngine:
554
prompt_token_ids=prompt_token_ids,
555
arrival_time=arrival_time,
556
lora_request=lora_request,
568
- prefix_pos=prefix_pos,
557
)
558
559
async for request_output in stream:
vllm/engine/llm_engine.py
+8
-18
@@ -415,7 +415,6 @@ class LLMEngine:
415
prompt_token_ids: Optional[List[int]] = None,
416
arrival_time: Optional[float] = None,
417
lora_request: Optional[LoRARequest] = None,
418
- prefix_pos: Optional[int] = None,
418
) -> None:
419
"""Add a request to the engine's request pool.
420
@@ -432,11 +431,6 @@ class LLMEngine:
431
use the tokenizer to convert the prompts to token IDs.
432
arrival_time: The arrival time of the request. If None, we use
433
the current monotonic time.
435
- prefix_pos: If not None, we use the given position as the prefix
436
- position for each prompt. We will cache the prefix's KV
437
- cache and reuse it for the next request with the same prefix.
438
- This is an experimental feature, and may be replaced with
439
- automatic prefix caching in the future.
434
435
Details:
436
- Set arrival_time to the current time if it is None.
@@ -479,18 +473,13 @@ class LLMEngine:
473
seq = Sequence(seq_id, prompt, prompt_token_ids, block_size,
474
lora_request)
475
482
- # Check whether the input specifies prefix
483
- prefix = self.scheduler.prefix_pool.add_or_get_prefix(
484
- prompt_token_ids[:prefix_pos], lora_request.lora_int_id
485
- if lora_request else 0) if prefix_pos is not None else None
486
-
476
# Defensive copy of SamplingParams, which are used by the sampler,
477
# this doesn't deep-copy LogitsProcessor objects
478
sampling_params = sampling_params.clone()
479
480
# Create the sequence group.
481
seq_group = SequenceGroup(request_id, [seq], sampling_params,
493
- arrival_time, lora_request, prefix)
482
+ arrival_time, lora_request)
483
484
# Add the sequence group to the scheduler.
485
self.scheduler.add_seq_group(seq_group)
@@ -752,6 +741,13 @@ class LLMEngine:
741
now = time.time()
742
# Update the scheduled sequence groups with the model outputs.
743
scheduled_seq_groups = scheduler_outputs.scheduled_seq_groups
744
+
745
+ # If prefix caching is enabled, mark all blocks in the sequence groups
746
+ # as completed so that future requests don't attempt to recompute them
747
+ if self.cache_config.enable_prefix_caching:
748
+ for seq_group in scheduled_seq_groups:
749
+ self.scheduler.mark_blocks_as_computed(seq_group)
750
+
751
for seq_group, outputs in zip(scheduled_seq_groups, output):
752
self._process_sequence_group_outputs(seq_group, outputs)
753
@@ -768,12 +764,6 @@ class LLMEngine:
764
request_output = RequestOutput.from_seq_group(seq_group)
765
request_outputs.append(request_output)
766
771
- # Update prefix state, now all the uncomputed prefixes are computed.
772
- for seq_group in scheduled_seq_groups:
773
- if (seq_group.prefix is not None and seq_group.prefix.allocated
774
- and not seq_group.prefix.computed):
775
- seq_group.prefix.computed = True
776
-
767
# Log stats.
768
if self.log_stats:
769
self.stat_logger.log(self._get_stats(scheduler_outputs))
vllm/entrypoints/api_server.py
+1
-5
@@ -39,15 +39,11 @@ async def generate(request: Request) -> Response:
39
"""
40
request_dict = await request.json()
41
prompt = request_dict.pop("prompt")
42
- prefix_pos = request_dict.pop("prefix_pos", None)
42
stream = request_dict.pop("stream", False)
43
sampling_params = SamplingParams(**request_dict)
44
request_id = random_uuid()
45
47
- results_generator = engine.generate(prompt,
48
- sampling_params,
49
- request_id,
50
- prefix_pos=prefix_pos)
46
+ results_generator = engine.generate(prompt, sampling_params, request_id)
47
48
# Streaming case
49
async def stream_results() -> AsyncGenerator[bytes, None]:
vllm/entrypoints/llm.py
+2
-12
@@ -124,7 +124,6 @@ class LLM:
124
prompts: Optional[Union[str, List[str]]] = None,
125
sampling_params: Optional[SamplingParams] = None,
126
prompt_token_ids: Optional[List[List[int]]] = None,
127
- prefix_pos: Optional[Union[int, List[int]]] = None,
127
use_tqdm: bool = True,
128
lora_request: Optional[LoRARequest] = None,
129
) -> List[RequestOutput]:
@@ -140,11 +139,6 @@ class LLM:
139
None, we use the default sampling parameters.
140
prompt_token_ids: A list of token IDs for the prompts. If None, we
141
use the tokenizer to convert the prompts to token IDs.
143
- prefix_pos: If not None, we use the given position as the prefix
144
- position for each prompt. We will cache the prefix's KV
145
- cache and reuse it for the next request with the same prefix.
146
- This is an experimental feature, and may be replaced with
147
- automatic prefix caching in the future.
142
use_tqdm: Whether to use tqdm to display the progress bar.
143
lora_request: LoRA request to use for generation, if any.
144
@@ -171,14 +165,12 @@ class LLM:
165
prompt_token_ids)
166
for i in range(num_requests):
167
prompt = prompts[i] if prompts is not None else None
174
- prefix_pos_i = prefix_pos[i] if prefix_pos is not None else None
168
token_ids = None if prompt_token_ids is None else prompt_token_ids[
169
i]
170
self._add_request(prompt,
171
sampling_params,
172
token_ids,
180
- lora_request=lora_request,
181
- prefix_pos=prefix_pos_i)
173
+ lora_request=lora_request)
174
return self._run_engine(use_tqdm)
175
176
def _add_request(
@@ -187,15 +179,13 @@ class LLM:
179
sampling_params: SamplingParams,
180
prompt_token_ids: Optional[List[int]],
181
lora_request: Optional[LoRARequest] = None,
190
- prefix_pos: Optional[int] = None,
182
) -> None:
183
request_id = str(next(self.request_counter))
184
self.llm_engine.add_request(request_id,
185
prompt,
186
sampling_params,
187
prompt_token_ids,
197
- lora_request=lora_request,
198
- prefix_pos=prefix_pos)
188
+ lora_request=lora_request)
189
190
def _run_engine(self, use_tqdm: bool) -> List[RequestOutput]:
191
# Initialize tqdm.
vllm/prefix.py
deleted
-87
@@ -1,87 +0,0 @@
1
-from typing import Dict, List, Sequence, Tuple, Optional
2
-
3
-from vllm.block import BlockTable
4
-
5
-
6
-class Prefix:
7
- """Data and states associated with a prefix of prompt tokens for multiple
8
- sequence groups.
9
-
10
- NOTE: This feature is experimental and may be replaced with automatic
11
- prefix caching in the future.
12
-
13
- Args:
14
- token_ids: The token ids of the prefix.
15
- block_size: The block size of the executed model.
16
- """
17
-
18
- def __init__(
19
- self,
20
- token_ids: Sequence[int],
21
- block_size: int,
22
- ) -> None:
23
- self.token_ids = tuple(token_ids)
24
- self.block_size = block_size
25
- self.length = len(token_ids)
26
- self.hash = hash(token_ids)
27
- assert self.length % block_size == 0
28
- self.block_table: Optional[BlockTable] = None
29
- self.computed = False
30
-
31
- @property
32
- def allocated(self) -> bool:
33
- return self.block_table is not None
34
-
35
- def get_num_blocks(self) -> int:
36
- return self.length // self.block_size
37
-
38
- def get_block_numbers(self) -> List[int]:
39
- return [block.block_number for block in self.block_table]
40
-
41
- def get_length(self) -> int:
42
- return self.length
43
-
44
- def __hash__(self) -> int:
45
- return self.hash
46
-
47
- def set_block_table(self, block_table: BlockTable) -> None:
48
- self.block_table = block_table.copy()
49
-
50
-
51
-class PrefixPool:
52
- """Manages all the prompt prefixes.
53
-
54
- NOTE: This feature is experimental and may be replaced with automatic
55
- prefix caching in the future.
56
-
57
- Args:
58
- block_size: The block size of the executed model.
59
-
60
- Attributes:
61
- prefixes: A list of all the prefixes.
62
- block_size: The block size of the executed model.
63
- """
64
-
65
- def __init__(
66
- self,
67
- block_size: int,
68
- ) -> None:
69
- # TODO(zhuohan): Add a capacity limit to the prefix pool.
70
- self.prefixes: Dict[int, Prefix] = {}
71
- self.block_size = block_size
72
-
73
- def _truncate_token_ids(self, token_ids: Sequence[int]) -> Tuple[int]:
74
- new_length = len(token_ids) // self.block_size * self.block_size
75
- return tuple(token_ids[:new_length])
76
-
77
- def add_or_get_prefix(self, token_ids: Sequence[int],
78
- lora_int_id: int) -> Optional[Prefix]:
79
- token_ids = self._truncate_token_ids(token_ids)
80
- if len(token_ids) == 0:
81
- # Prefix is empty.
82
- return None
83
- prefix = Prefix(token_ids, self.block_size)
84
- prefix_hash = hash((prefix, lora_int_id))
85
- if prefix_hash not in self.prefixes:
86
- self.prefixes[prefix_hash] = prefix
87
- return self.prefixes[prefix_hash]
vllm/sequence.py
+16
-7
@@ -5,7 +5,6 @@ from dataclasses import dataclass
5
from typing import Dict, List, Optional, Union
6
7
from vllm.block import LogicalTokenBlock
8
-from vllm.prefix import Prefix
8
from vllm.sampling_params import SamplingParams
9
from vllm.lora.request import LoRARequest
10
@@ -161,6 +160,16 @@ class Sequence:
160
def lora_int_id(self) -> int:
161
return self.lora_request.lora_int_id if self.lora_request else 0
162
163
+ # TODO The current hashing function is O(L^2). We should optimize this in
164
+ # the future.
165
+ def hash_of_block(self, logical_idx: int) -> int:
166
+ # Compute the number of tokens in the sequence
167
+ num_tokens = self.num_hashed_tokens_of_block(logical_idx)
168
+ return hash(tuple(self.data.get_token_ids()[0:num_tokens]))
169
+
170
+ def num_hashed_tokens_of_block(self, logical_idx: int):
171
+ return logical_idx * self.block_size + self.block_size
172
+
173
def _append_logical_block(self) -> None:
174
block = LogicalTokenBlock(
175
block_number=len(self.logical_token_blocks),
@@ -265,7 +274,6 @@ class SequenceGroup:
274
sampling_params: The sampling parameters used to generate the outputs.
275
arrival_time: The arrival time of the request.
276
lora_request: LoRA request.
268
- prefix: The prefix of the prompt of the sequence group.
277
"""
278
279
def __init__(
@@ -275,7 +283,6 @@ class SequenceGroup:
283
sampling_params: SamplingParams,
284
arrival_time: float,
285
lora_request: Optional[LoRARequest] = None,
278
- prefix: Optional[Prefix] = None,
286
) -> None:
287
self.request_id = request_id
288
self.seqs_dict = {seq.seq_id: seq for seq in seqs}
@@ -286,7 +293,6 @@ class SequenceGroup:
293
first_token_time=None,
294
time_in_queue=None)
295
self.lora_request = lora_request
289
- self.prefix: Optional[Prefix] = prefix
296
self.prompt_logprobs: Optional[PromptLogprobs] = None
297
self.state = SequenceGroupState()
298
@@ -302,6 +308,10 @@ class SequenceGroup:
308
# We use the prompt of an arbitrary sequence.
309
return next(iter(self.seqs_dict.values())).data.prompt_token_ids
310
311
+ @property
312
+ def block_size(self) -> int:
313
+ return next(iter(self.seqs_dict.values())).block_size
314
+
315
@property
316
def lora_int_id(self) -> int:
317
return self.lora_request.lora_int_id if self.lora_request else 0
@@ -408,7 +418,6 @@ class SequenceGroupMetadata:
418
numbers)
419
state: Internal state tied to this sequence group.
420
lora_request: LoRA request.
411
- prefix: The prefix of the prompt of the sequence group.
421
"""
422
423
def __init__(
@@ -419,7 +428,7 @@ class SequenceGroupMetadata:
428
sampling_params: SamplingParams,
429
block_tables: Dict[int, List[int]],
430
lora_request: Optional[LoRARequest] = None,
422
- prefix: Optional[Prefix] = None,
431
+ computed_block_nums: Optional[List[int]] = None,
432
state: Optional[SequenceGroupState] = None,
433
) -> None:
434
self.request_id = request_id
@@ -428,7 +437,7 @@ class SequenceGroupMetadata:
437
self.sampling_params = sampling_params
438
self.block_tables = block_tables
439
self.lora_request = lora_request
431
- self.prefix = prefix
440
+ self.computed_block_nums = computed_block_nums
441
self.state = SequenceGroupState() if state is None else state
442
443
@property
vllm/worker/model_runner.py
+17
-13
@@ -145,33 +145,37 @@ class ModelRunner:
145
prompt_tokens = seq_data.get_token_ids()
146
prompt_len = len(prompt_tokens)
147
prompt_lens.append(prompt_len)
148
- prefix_len = 0
149
- prefix = seq_group_metadata.prefix
150
- if prefix is not None and prefix.computed:
151
- prefix_len = prefix.get_length()
152
- prompt_tokens = prompt_tokens[prefix_len:]
153
- prefix_block_tables.append(prefix.get_block_numbers())
148
+ computed_len = 0
149
+
150
+ # NOTE: This only works for oooooooxxx style attention.
151
+ computed_block_nums = seq_group_metadata.computed_block_nums
152
+ if computed_block_nums is not None and len(
153
+ computed_block_nums) > 0 and self.sliding_window is None:
154
+ # Prefix is not supported with sliding_window
155
+ computed_len = len(computed_block_nums) * self.block_size
156
+ prompt_tokens = prompt_tokens[computed_len:]
157
+ prefix_block_tables.append(computed_block_nums)
158
else:
159
prefix_block_tables.append([])
160
# actual prompt lens
157
- context_lens.append(prefix_len)
158
- subquery_lens.append(prompt_len - prefix_len)
161
+ context_lens.append(computed_len)
162
+ subquery_lens.append(prompt_len - computed_len)
163
164
input_tokens.append(prompt_tokens)
165
# NOTE(woosuk): Here we assume that the first token in the prompt
166
# is always the first token in the sequence.
167
input_positions.append(
164
- list(range(prefix_len, prefix_len + len(prompt_tokens))))
168
+ list(range(computed_len, computed_len + len(prompt_tokens))))
169
170
lora_id = seq_group_metadata.lora_int_id
171
172
if lora_id > 0:
173
lora_requests.add(seq_group_metadata.lora_request)
174
171
- lora_index_mapping.append([lora_id] * (prompt_len - prefix_len))
175
+ lora_index_mapping.append([lora_id] * (prompt_len - computed_len))
176
lora_prompt_mapping.extend(
177
[lora_id] *
174
- (prompt_len - prefix_len
178
+ (prompt_len - computed_len
179
if seq_group_metadata.sampling_params.prompt_logprobs else 1))
180
181
if seq_group_metadata.block_tables is None:
@@ -190,11 +194,11 @@ class ModelRunner:
194
# mapping will be [-1, -1, 2, 3, 4, 5, 6, 7, 0, 1].
195
start_idx = 0
196
if self.sliding_window is not None:
193
- assert prefix_len == 0, (
197
+ assert computed_len == 0, (
198
"Prefix caching is currently not supported with "
199
"sliding window attention")
200
start_idx = max(0, prompt_len - self.sliding_window)
197
- for i in range(prefix_len, prompt_len):
201
+ for i in range(computed_len, prompt_len):
202
if i < start_idx:
203
slot_mapping[-1].append(_PAD_SLOT_ID)
204
continue