@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ce4f5a29fb
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Add Automatic Prefix Caching (#2762)

Co-authored-by: ElizaWszola <eliza@neuralmagic.com> Co-authored-by: Michael Goin <michael@neuralmagic.com>

Sage Moore committed Mar 2, 2024 at 03:50 UTC ce4f5a29fb3e35041842518fefe999847b8326b9
18 files changed +618 -292
benchmarks/benchmark_throughput.py
+16 -14
@@ -73,21 +73,21 @@ def run_vllm(
73 enforce_eager: bool,
74 kv_cache_dtype: str,
75 device: str,
76 + enable_prefix_caching: bool,
77 ) -> float:
78 from vllm import LLM, SamplingParams
78 - llm = LLM(
79 - model=model,
80 - tokenizer=tokenizer,
81 - quantization=quantization,
82 - tensor_parallel_size=tensor_parallel_size,
83 - seed=seed,
84 - trust_remote_code=trust_remote_code,
85 - dtype=dtype,
86 - max_model_len=max_model_len,
87 - enforce_eager=enforce_eager,
88 - kv_cache_dtype=kv_cache_dtype,
89 - device=device,
90 - )
79 + llm = LLM(model=model,
80 + tokenizer=tokenizer,
81 + quantization=quantization,
82 + tensor_parallel_size=tensor_parallel_size,
83 + seed=seed,
84 + trust_remote_code=trust_remote_code,
85 + dtype=dtype,
86 + max_model_len=max_model_len,
87 + enforce_eager=enforce_eager,
88 + kv_cache_dtype=kv_cache_dtype,
89 + device=device,
90 + enable_prefix_caching=enable_prefix_caching)
91
92 # Add the requests to the engine.
93 for prompt, _, output_len in requests:
@@ -211,7 +211,8 @@ def main(args: argparse.Namespace):
211 args.seed, args.n, args.use_beam_search,
212 args.trust_remote_code, args.dtype,
213 args.max_model_len, args.enforce_eager,
214 - args.kv_cache_dtype, args.device)
214 + args.kv_cache_dtype, args.device,
215 + args.enable_prefix_caching)
216 elif args.backend == "hf":
217 assert args.tensor_parallel_size == 1
218 elapsed_time = run_hf(requests, args.model, tokenizer, args.n,
@@ -302,6 +303,7 @@ if __name__ == "__main__":
303 default="cuda",
304 choices=["cuda"],
305 help='device type for vLLM execution, supporting CUDA only currently.')
306 + parser.add_argument("--enable_prefix_caching", action='store_true')
307 args = parser.parse_args()
308 if args.tokenizer is None:
309 args.tokenizer = args.model
docs/source/models/engine_args.rst
+4
@@ -81,6 +81,10 @@ Below, you can find an explanation of every engine argument for vLLM:
81
82 Token block size for contiguous chunks of tokens.
83
84 +.. option:: --enable-prefix-caching
85 +
86 + Enables automatic prefix caching
87 +
88 .. option:: --seed <seed>
89
90 Random seed for operations.
examples/offline_inference_with_prefix.py
+2 -9
@@ -37,20 +37,13 @@ for output in outputs:
37
38 print("-" * 80)
39
40 -# -1 since the last token can change when concatenating prompts.
41 -prefix_pos = len(llm.llm_engine.tokenizer.encode(prefix)) - 1
42 -
40 # The llm.generate call will batch all prompts and send the batch at once if resources allow.
41 # The prefix will only be cached after the first batch is processed, so we need to call generate once
42 # to calculate the prefix and cache it.
46 -outputs = llm.generate(generating_prompts[0],
47 - sampling_params,
48 - prefix_pos=[prefix_pos])
43 +outputs = llm.generate(generating_prompts[0], sampling_params)
44
45 # Subsequent batches can leverage the cached prefix
51 -outputs = llm.generate(generating_prompts,
52 - sampling_params,
53 - prefix_pos=[prefix_pos] * len(generating_prompts))
46 +outputs = llm.generate(generating_prompts, sampling_params)
47
48 # Print the outputs. You should see the same outputs as before
49 for output in outputs:
tests/prefix_caching/test_prefix_caching.py
+69 -34
@@ -4,38 +4,73 @@ Run `pytest tests/prefix_caching/test_prefix_caching.py`.
4 """
5 import pytest
6
7 -from vllm import LLM, SamplingParams
8 -
9 -prefix = (
10 - "You are an expert school principal, skilled in effectively managing "
11 - "faculty and staff. Draft 10-15 questions for a potential first grade "
12 - "Head Teacher for my K-12, all-girls', independent school that emphasizes "
13 - "community, joyful discovery, and life-long learning. The candidate is "
14 - "coming in for a first-round panel interview for a 8th grade Math "
15 - "teaching role. They have 5 years of previous teaching experience "
16 - "as an assistant teacher at a co-ed, public school with experience "
17 - "in middle school math teaching. Based on these information, fulfill "
18 - "the following paragraph: ")
19 -
20 -
21 -@pytest.mark.parametrize("model", ["facebook/opt-125m"])
22 -@pytest.mark.parametrize("max_tokens", [16])
23 -def test_prefix_caching(
24 - example_prompts,
25 - model: str,
26 - max_tokens: int,
7 +from vllm.core.block_manager import BlockAllocator
8 +from vllm.utils import Device
9 +
10 +
11 +@pytest.mark.parametrize("block_size", [16])
12 +@pytest.mark.parametrize("num_blocks", [16])
13 +def test_block_allocator(
14 + block_size: int,
15 + num_blocks: int,
16 ):
28 - llm = LLM(model=model)
29 - # -1 since the last token can change when concatenating prompts.
30 - prefix_pos = len(llm.llm_engine.tokenizer.encode(prefix)) - 1
31 - prompts = [prefix + prompt for prompt in example_prompts]
32 - sampling_params = SamplingParams(temperature=0.0, max_tokens=max_tokens)
33 - outputs_without_prefix = llm.generate(prompts, sampling_params)
34 - outputs_with_prefix = llm.generate(prompts,
35 - sampling_params,
36 - prefix_pos=[prefix_pos] * len(prompts))
37 - for output_without_prefix, output_with_prefix in zip(
38 - outputs_without_prefix, outputs_with_prefix):
39 - assert (output_without_prefix.outputs[0].token_ids ==
40 - output_with_prefix.outputs[0].token_ids)
41 - assert len(llm.llm_engine.scheduler.prefix_pool.prefixes) == 1
17 + block_hash = 1
18 + block_allocator = BlockAllocator(Device.CPU,
19 + block_size,
20 + num_blocks,
21 + enable_caching=True)
22 +
23 + # Allocate two PysicalTokenBlocks with the same hash and check that they are the same PhysicalTokenBlock
24 + first_block = block_allocator.allocate(block_hash, 0)
25 + second_block = block_allocator.allocate(block_hash, 0)
26 + assert (first_block == second_block)
27 + assert (second_block.ref_count == 2)
28 +
29 + # Free the first_block and confirm that the ref_count is correctly decremented on the second block
30 + block_allocator.free(first_block)
31 + assert (second_block.ref_count == 1)
32 +
33 + # Free the second block
34 + block_allocator.free(second_block)
35 +
36 + # Reallocate the first block and confirm that, even after the block had its ref_count go to 0, we still get the same block back
37 + first_block = block_allocator.allocate(block_hash, 0)
38 + assert (first_block == second_block)
39 + assert (first_block.block_hash == block_hash)
40 +
41 +
42 +@pytest.mark.parametrize("num_blocks", [16])
43 +def test_eviction(num_blocks: int, ):
44 + block_size = 16
45 + block_allocator = BlockAllocator(Device.CPU,
46 + block_size,
47 + num_blocks,
48 + enable_caching=True)
49 + blocks = []
50 +
51 + for i in range(num_blocks):
52 + # use i as the block_hash
53 + blocks.append(block_allocator.allocate(i, 0))
54 +
55 + #Free all blocks
56 + for block in blocks:
57 + block_allocator.free(block)
58 +
59 + # Allocate a new block and confirm that it's the first block freed. I.E The Least Recently Used block
60 + new_block_hash = block_size
61 + new_block = block_allocator.allocate(new_block_hash, 0)
62 + assert (new_block == blocks[0])
63 + assert (new_block.block_hash == new_block_hash)
64 +
65 + # Reallocate the second in blocks to remove it from the free list
66 + realloc_block_hash = 1
67 + realloc_block = block_allocator.allocate(realloc_block_hash, 0)
68 + assert (realloc_block == blocks[realloc_block_hash])
69 + assert (realloc_block.block_hash == realloc_block_hash)
70 +
71 + # Allocate a new block and confirm that it's not the realloc_block, since the realloc_block shouldn't be in the free list
72 + new_block_hash = block_size + 1
73 + new_block = block_allocator.allocate(new_block_hash, 0)
74 + assert (realloc_block != new_block)
75 + assert (new_block.block_hash == new_block_hash)
76 + assert (new_block.block_number == 2)
tests/test_cache_block_hashing.py new
+76
@@ -0,0 +1,76 @@
1 +"""Test hashing of cache blocks.
2 +
3 +Run `pytest tests/test_cache_block_hashing.py`.
4 +"""
5 +import pytest
6 +
7 +from vllm.transformers_utils.tokenizer import TokenizerGroup
8 +from vllm.sequence import Sequence
9 +
10 +# Make two prefixes with different first blocks.
11 +prefix_start = [("You are an expert"), ("You are a")]
12 +prefix_common = (
13 + " school principal, skilled in effectively managing "
14 + "faculty and staff. Draft 10-15 questions for a potential first grade "
15 + "Head Teacher for my K-12, all-girls', independent school that emphasizes "
16 + "community, joyful discovery, and life-long learning. The candidate is "
17 + "coming in for a first-round panel interview for a 8th grade Math "
18 + "teaching role. They have 5 years of previous teaching experience "
19 + "as an assistant teacher at a co-ed, public school with experience "
20 + "in middle school math teaching. Based on this, fulfill "
21 + "the following: ")
22 +prefixes = [start + prefix_common for start in prefix_start]
23 +
24 +# Sample prompts.
25 +sample_prompts = [
26 + "Hello, my name is", "The president of the United States is",
27 + "The capital of France is", "The future of AI is"
28 +]
29 +
30 +
31 +# Helper function.
32 +def flatten_2d(li):
33 + return [lss for ls in li for lss in ls]
34 +
35 +
36 +@pytest.mark.parametrize("model", ["facebook/opt-125m"])
37 +@pytest.mark.parametrize("block_size", [16])
38 +@pytest.mark.parametrize("max_num_seqs", [256])
39 +def test_auto_prefix_caching(model: str, block_size: int, max_num_seqs: int):
40 +
41 + tokenizer = TokenizerGroup(
42 + tokenizer_id="facebook/opt-125m",
43 + enable_lora=False,
44 + max_num_seqs=max_num_seqs,
45 + max_input_length=None,
46 + )
47 +
48 + hashes = []
49 +
50 + for prefix in prefixes:
51 + hashes.append([])
52 + prompts = [prefix + prompt for prompt in sample_prompts]
53 + seq_id = 0
54 + for prompt in prompts:
55 + hashes[-1].append([])
56 + prompt_token_ids = tokenizer.encode(prompt)
57 + seq = Sequence(seq_id, prompt, prompt_token_ids, block_size)
58 +
59 + num_blocks = len(prompt_token_ids) // block_size
60 + for idx in range(num_blocks):
61 + hashes[-1][-1].append(seq.hash_of_block(idx))
62 +
63 + seq_id += 1
64 +
65 + # Check that hashes made with two prefixes with different first blocks are
66 + # different everywhere.
67 + for hash0, hash1 in zip(flatten_2d(hashes[0]), flatten_2d(hashes[1])):
68 + assert (hash0 != hash1)
69 +
70 + # Check that hashes of different prompts made with the same prefix are the
71 + # same until the hashes that contain the prompt.
72 + for hash_pref in hashes:
73 + same_hashes = [tuple(h[:-1]) for h in hash_pref]
74 + different_hashes = [h[-1] for h in hash_pref]
75 + assert (len(set(same_hashes)) == 1)
76 + assert (len(set(different_hashes)) == len(different_hashes))
vllm/block.py
+13 -1
@@ -5,6 +5,8 @@ from vllm.utils import Device
5
6 _BLANK_TOKEN_ID = -1
7
8 +DEFAULT_LAST_ACCESSED_TIME = -1
9 +
10
11 class LogicalTokenBlock:
12 """A block that stores a contiguous chunk of tokens from left to right.
@@ -55,17 +57,27 @@ class PhysicalTokenBlock:
57 device: Device,
58 block_number: int,
59 block_size: int,
60 + block_hash: int,
61 + num_hashed_tokens: int,
62 ) -> None:
63 self.device = device
64 self.block_number = block_number
65 self.block_size = block_size
66 + self.block_hash = block_hash
67 + self.num_hashed_tokens = num_hashed_tokens
68
69 self.ref_count = 0
70 + self.last_accessed = DEFAULT_LAST_ACCESSED_TIME
71 +
72 + self.computed = False
73
74 def __repr__(self) -> str:
75 return (f'PhysicalTokenBlock(device={self.device}, '
76 f'block_number={self.block_number}, '
68 - f'ref_count={self.ref_count})')
77 + f'num_hashed_tokens={self.num_hashed_tokens}, '
78 + f'ref_count={self.ref_count}, '
79 + f'last_accessed={self.last_accessed}, '
80 + f'computed={self.computed})')
81
82
83 # Mapping: logical block number -> physical block.
vllm/config.py
+2
@@ -303,12 +303,14 @@ class CacheConfig:
303 swap_space: int,
304 cache_dtype: str,
305 sliding_window: Optional[int] = None,
306 + enable_prefix_caching: bool = False,
307 ) -> None:
308 self.block_size = block_size
309 self.gpu_memory_utilization = gpu_memory_utilization
310 self.swap_space_bytes = swap_space * _GB
311 self.cache_dtype = cache_dtype
312 self.sliding_window = sliding_window
313 + self.enable_prefix_caching = enable_prefix_caching
314 self._verify_args()
315 self._verify_cache_dtype()
316
vllm/core/block_manager.py
+213 -72
@@ -1,10 +1,13 @@
1 """A block manager that manages token blocks."""
2 import enum
3 +from itertools import count
4 +from os.path import commonprefix
5 from typing import Dict, List, Optional, Set, Tuple
6
7 from vllm.block import BlockTable, PhysicalTokenBlock
8 from vllm.sequence import Sequence, SequenceGroup, SequenceStatus
9 from vllm.utils import Device
10 +from vllm.core.evictor import Evictor, EvictionPolicy, make_evictor
11
12
13 class BlockAllocator:
@@ -15,29 +18,68 @@ class BlockAllocator:
18 the reference count becomes zero, the block is added back to the free list.
19 """
20
18 - def __init__(
19 - self,
20 - device: Device,
21 - block_size: int,
22 - num_blocks: int,
23 - ) -> None:
21 + def __init__(self,
22 + device: Device,
23 + block_size: int,
24 + num_blocks: int,
25 + eviction_policy: EvictionPolicy = EvictionPolicy.LRU,
26 + enable_caching: bool = False) -> None:
27 self.device = device
28 self.block_size = block_size
29 self.num_blocks = num_blocks
30 + self.enable_caching = enable_caching
31 +
32 + self.current_num_blocks = 0
33 + self.cached_blocks: Dict[int, PhysicalTokenBlock] = {}
34 +
35 + # Switch over to FIFO eviction when caching is disabled
36 + if not self.enable_caching:
37 + eviction_policy = EvictionPolicy.FIFO
38 + self.evictor: Evictor = make_evictor(eviction_policy)
39 +
40 + self.default_hash_ctr = count()
41 +
42 + def allocate_block(self, block_hash: int,
43 + num_hashed_tokens: int) -> PhysicalTokenBlock:
44 + if self.current_num_blocks == self.num_blocks:
45 + block = self.evictor.evict()
46 + block.block_hash = block_hash
47 + block.num_hashed_tokens = num_hashed_tokens
48 + return block
49 + block = PhysicalTokenBlock(device=self.device,
50 + block_number=self.current_num_blocks,
51 + block_size=self.block_size,
52 + block_hash=block_hash,
53 + num_hashed_tokens=num_hashed_tokens)
54 + self.current_num_blocks += 1
55 + return block
56
28 - # Initialize the free blocks.
29 - self.free_blocks: BlockTable = []
30 - for i in range(num_blocks):
31 - block = PhysicalTokenBlock(device=device,
32 - block_number=i,
33 - block_size=block_size)
34 - self.free_blocks.append(block)
35 -
36 - def allocate(self) -> PhysicalTokenBlock:
37 - if not self.free_blocks:
38 - raise ValueError("Out of memory! No free blocks are available.")
39 - block = self.free_blocks.pop()
40 - block.ref_count = 1
57 + def allocate(self,
58 + block_hash: Optional[int] = None,
59 + num_hashed_tokens: int = 0) -> PhysicalTokenBlock:
60 + # If caching is disabled, just allocate a new block and return it
61 + if not self.enable_caching:
62 + block = self.allocate_block(next(self.default_hash_ctr),
63 + num_hashed_tokens)
64 + block.ref_count += 1
65 + return block
66 +
67 + if block_hash is None:
68 + block_hash = next(self.default_hash_ctr)
69 + if block_hash in self.evictor:
70 + assert block_hash not in self.cached_blocks
71 + block = self.evictor.remove(block_hash)
72 + assert block.ref_count == 0
73 + self.cached_blocks[block_hash] = block
74 + block.ref_count += 1
75 + assert block.block_hash == block_hash
76 + return block
77 + if block_hash not in self.cached_blocks:
78 + self.cached_blocks[block_hash] = self.allocate_block(
79 + block_hash, num_hashed_tokens)
80 + block = self.cached_blocks[block_hash]
81 + assert block.block_hash == block_hash
82 + block.ref_count += 1
83 return block
84
85 def free(self, block: PhysicalTokenBlock) -> None:
@@ -45,10 +87,27 @@ class BlockAllocator:
87 raise ValueError(f"Double free! {block} is already freed.")
88 block.ref_count -= 1
89 if block.ref_count == 0:
48 - self.free_blocks.append(block)
90 + assert block.block_hash not in self.evictor
91 + self.evictor.add(block)
92 +
93 + # If caching is enabled, remove the block from the cached_blocks
94 + if self.enable_caching:
95 + del self.cached_blocks[block.block_hash]
96
97 def get_num_free_blocks(self) -> int:
51 - return len(self.free_blocks)
98 + return self.num_blocks - self.current_num_blocks + self.evictor.num_blocks
99 +
100 + def contains_block(self, block_hash: int) -> bool:
101 + return block_hash in self.cached_blocks or block_hash in self.evictor
102 +
103 + def update_hash(self, block_hash: int, block: PhysicalTokenBlock):
104 + # If caching is enabled, update the hash of block and the cached_blocks dictionary.
105 + if self.enable_caching:
106 + assert not self.contains_block(block_hash)
107 + old_hash = block.block_hash
108 + block.block_hash = block_hash
109 + del self.cached_blocks[old_hash]
110 + self.cached_blocks[block_hash] = block
111
112
113 class AllocStatus(enum.Enum):
@@ -75,6 +134,7 @@ class BlockSpaceManager:
134 num_cpu_blocks: int,
135 watermark: float = 0.01,
136 sliding_window: Optional[int] = None,
137 + enable_caching: bool = False,
138 ) -> None:
139 self.block_size = block_size
140 self.num_total_gpu_blocks = num_gpu_blocks
@@ -89,11 +149,17 @@ class BlockSpaceManager:
149 self.watermark = watermark
150 assert watermark >= 0.0
151
152 + self.enable_caching = enable_caching
153 +
154 self.watermark_blocks = int(watermark * num_gpu_blocks)
93 - self.gpu_allocator = BlockAllocator(Device.GPU, block_size,
94 - num_gpu_blocks)
95 - self.cpu_allocator = BlockAllocator(Device.CPU, block_size,
96 - num_cpu_blocks)
155 + self.gpu_allocator = BlockAllocator(Device.GPU,
156 + block_size,
157 + num_gpu_blocks,
158 + enable_caching=enable_caching)
159 + self.cpu_allocator = BlockAllocator(Device.CPU,
160 + block_size,
161 + num_cpu_blocks,
162 + enable_caching=enable_caching)
163 # Mapping: seq_id -> BlockTable.
164 self.block_tables: Dict[int, BlockTable] = {}
165
@@ -103,9 +169,6 @@ class BlockSpaceManager:
169 seq = seq_group.get_seqs(status=SequenceStatus.WAITING)[0]
170 num_required_blocks = len(seq.logical_token_blocks)
171
106 - if seq_group.prefix is not None and seq_group.prefix.allocated:
107 - num_required_blocks -= seq_group.prefix.get_num_blocks()
108 -
172 if self.block_sliding_window is not None:
173 num_required_blocks = min(num_required_blocks,
174 self.block_sliding_window)
@@ -129,36 +192,16 @@ class BlockSpaceManager:
192 num_prompt_blocks = len(seq.logical_token_blocks)
193
194 block_table: BlockTable = []
132 - prefix_block_table: BlockTable = []
133 - num_prefix_blocks = 0
134 -
135 - prefix = seq_group.prefix
136 - if prefix is not None and prefix.allocated:
137 - # Prefix has already been allocated. Use the existing block table.
138 - num_prompt_blocks -= prefix.get_num_blocks()
139 - for block in prefix.block_table:
140 - block.ref_count += seq_group.num_seqs()
141 - block_table.append(block)
142 -
195 for logical_idx in range(num_prompt_blocks):
196 if (self.block_sliding_window is not None
197 and logical_idx >= self.block_sliding_window):
198 block = block_table[logical_idx % self.block_sliding_window]
199 else:
148 - block = self.gpu_allocator.allocate()
149 - # Set the reference counts of the token blocks.
150 - block.ref_count = seq_group.num_seqs()
200 + block = self.gpu_allocator.allocate(
201 + seq.hash_of_block(logical_idx),
202 + seq.num_hashed_tokens_of_block(logical_idx))
203 block_table.append(block)
204
153 - if prefix is not None and not prefix.allocated:
154 - # Allocate blocks for the prefix, we will compute the prefix's
155 - # KV cache in this run.
156 - num_prefix_blocks = prefix.get_num_blocks()
157 - prefix_block_table = block_table[:num_prefix_blocks]
158 - for block in prefix_block_table:
159 - block.ref_count += 1
160 - prefix.set_block_table(prefix_block_table)
161 -
205 # Assign the block table for each sequence.
206 for seq in seq_group.get_seqs(status=SequenceStatus.WAITING):
207 self.block_tables[seq.seq_id] = block_table.copy()
@@ -170,12 +213,72 @@ class BlockSpaceManager:
213 num_seqs = seq_group.num_seqs(status=SequenceStatus.RUNNING)
214 return num_seqs <= num_free_gpu_blocks
215
173 - def append_slot(self, seq: Sequence) -> Optional[Tuple[int, int]]:
216 + def _promote_last_block(
217 + self,
218 + seq: Sequence,
219 + last_block: PhysicalTokenBlock,
220 + ) -> PhysicalTokenBlock:
221 + # Compute a new hash for the block so that it can be shared by other Sequences
222 + new_hash = seq.hash_of_block(len(seq.logical_token_blocks) - 1)
223 +
224 + # if new_hash is already in the cached table, then free last_block and return the cached version
225 + if self.gpu_allocator.contains_block(new_hash):
226 + self.gpu_allocator.free(last_block)
227 + return self.gpu_allocator.allocate(new_hash)
228 + else:
229 + self.gpu_allocator.update_hash(new_hash, last_block)
230 + return last_block
231 +
232 + def _is_last_block_full(
233 + self,
234 + seq: Sequence,
235 + ) -> bool:
236 + token_ids_len = len(seq.data.get_token_ids())
237 + return token_ids_len > 0 and token_ids_len % seq.block_size == 0
238 +
239 + def _is_last_block(
240 + self,
241 + seq: Sequence,
242 + index: int,
243 + ) -> bool:
244 + return index == len(seq.logical_token_blocks) - 1
245 +
246 + def _maybe_promote_last_block(
247 + self,
248 + seq: Sequence,
249 + last_block: PhysicalTokenBlock,
250 + ) -> PhysicalTokenBlock:
251 + if self._is_last_block_full(seq):
252 + return self._promote_last_block(seq, last_block)
253 + else:
254 + return last_block
255 +
256 + def _allocate_last_physical_block(
257 + self,
258 + seq: Sequence,
259 + ) -> PhysicalTokenBlock:
260 + block_hash: Optional[int] = None
261 + if (self._is_last_block_full(seq)):
262 + block_hash = seq.hash_of_block(len(seq.logical_token_blocks) - 1)
263 + num_hashed_tokens = seq.num_hashed_tokens_of_block(
264 + len(seq.logical_token_blocks) - 1)
265 + new_block = self.gpu_allocator.allocate(block_hash, num_hashed_tokens)
266 + if block_hash is None:
267 + assert new_block.ref_count == 1
268 + return new_block
269 +
270 + def append_slot(
271 + self,
272 + seq: Sequence,
273 + ) -> Optional[Tuple[int, int]]:
274 """Allocate a physical slot for a new token."""
275 logical_blocks = seq.logical_token_blocks
276 block_table = self.block_tables[seq.seq_id]
177 -
277 + # If we need to allocate a new physical block
278 if len(block_table) < len(logical_blocks):
279 + # Currently this code only supports adding one physical block
280 + assert len(block_table) == len(logical_blocks) - 1
281 +
282 if (self.block_sliding_window
283 and len(block_table) >= self.block_sliding_window):
284 # reuse a block
@@ -184,8 +287,8 @@ class BlockSpaceManager:
287 else:
288 # The sequence has a new logical block.
289 # Allocate a new physical block.
187 - block = self.gpu_allocator.allocate()
188 - block_table.append(block)
290 + new_block = self._allocate_last_physical_block(seq)
291 + block_table.append(new_block)
292 return None
293
294 # We want to append the token to the last physical block.
@@ -193,11 +296,15 @@ class BlockSpaceManager:
296 assert last_block.device == Device.GPU
297 if last_block.ref_count == 1:
298 # Not shared with other sequences. Appendable.
299 + # If the last block is now complete, promote it to a full block so that it can be shared
300 + new_block = self._maybe_promote_last_block(seq, last_block)
301 + block_table[-1] = new_block
302 return None
303 else:
304 # The last block is shared with other sequences.
305 # Copy on Write: Allocate a new block and copy the tokens.
200 - new_block = self.gpu_allocator.allocate()
306 + new_block = self._allocate_last_physical_block(seq)
307 +
308 block_table[-1] = new_block
309 self.gpu_allocator.free(last_block)
310 return last_block.block_number, new_block.block_number
@@ -233,25 +340,18 @@ class BlockSpaceManager:
340
341 def swap_in(self, seq_group: SequenceGroup) -> Dict[int, int]:
342 # CPU block -> GPU block.
236 - if seq_group.prefix is not None:
237 - # make sure to swap in the prefix first
238 - assert seq_group.prefix.allocated and seq_group.prefix.computed
239 -
343 mapping: Dict[PhysicalTokenBlock, PhysicalTokenBlock] = {}
344 for seq in seq_group.get_seqs(status=SequenceStatus.SWAPPED):
345 new_block_table: BlockTable = []
346 block_table = self.block_tables[seq.seq_id]
244 - if seq_group.prefix is not None:
245 - for block in seq_group.prefix.block_table:
246 - new_block_table.append(block)
247 - block.ref_count += 1
347
348 for cpu_block in block_table:
349 if cpu_block in mapping:
350 gpu_block = mapping[cpu_block]
351 gpu_block.ref_count += 1
352 else:
254 - gpu_block = self.gpu_allocator.allocate()
353 + gpu_block = self.gpu_allocator.allocate(
354 + cpu_block.block_hash, cpu_block.num_hashed_tokens)
355 mapping[cpu_block] = gpu_block
356 new_block_table.append(gpu_block)
357 # Free the CPU block swapped in to GPU.
@@ -276,17 +376,12 @@ class BlockSpaceManager:
376 block_table = self.block_tables[seq.seq_id]
377
378 for gpu_block in block_table:
279 - if (seq_group.prefix is not None
280 - and gpu_block in seq_group.prefix.block_table):
281 - # NOTE: We do not swap out the prefix blocks for now.
282 - self.gpu_allocator.free(gpu_block)
283 - continue
284 -
379 if gpu_block in mapping:
380 cpu_block = mapping[gpu_block]
381 cpu_block.ref_count += 1
382 else:
289 - cpu_block = self.cpu_allocator.allocate()
383 + cpu_block = self.cpu_allocator.allocate(
384 + gpu_block.block_hash, gpu_block.num_hashed_tokens)
385 mapping[gpu_block] = cpu_block
386 new_block_table.append(cpu_block)
387 # Free the GPU block swapped out to CPU.
@@ -328,3 +423,49 @@ class BlockSpaceManager:
423
424 def get_num_free_cpu_blocks(self) -> int:
425 return self.cpu_allocator.get_num_free_blocks()
426 +
427 + def access_all_blocks_in_seq(
428 + self,
429 + seq: Sequence,
430 + access_time: float,
431 + ) -> None:
432 + block_table = self.block_tables[seq.seq_id]
433 + for block in block_table:
434 + block.last_accessed = access_time
435 +
436 + def compute_last_full_block_in_seq(self, seq: Sequence):
437 + if seq.seq_id not in self.block_tables:
438 + return
439 + max_full_block = seq.get_len() // seq.block_size - 1
440 + block_table = self.block_tables[seq.seq_id]
441 + if max_full_block == -1:
442 + return
443 + block_table[max_full_block].computed = True
444 +
445 + def get_all_block_ids_till_computed(self, seq: Sequence) -> List[int]:
446 + if seq.seq_id not in self.block_tables:
447 + return []
448 + block_table = self.block_tables[seq.seq_id]
449 + for block_idx in reversed(range(len(block_table))):
450 + if block_table[block_idx].computed:
451 + return [b.block_number for b in block_table[:block_idx + 1]]
452 + return []
453 +
454 + # Can return non-empty result only with prefix caching enabled.
455 + def get_common_computed_block_ids(self,
456 + seq_group: SequenceGroup) -> List[int]:
457 + if not self.enable_caching:
458 + return []
459 +
460 + ids_list = [
461 + self.get_all_block_ids_till_computed(seq)
462 + for seq in iter(seq_group.seqs_dict.values())
463 + ]
464 + return commonprefix([ids for ids in ids_list if ids != []])
465 +
466 + # We only mark the last full block because with prefix caching,
467 + # all blocks until the marked one are guaranteed to be computed.
468 + def mark_blocks_as_computed(self, seq_group: SequenceGroup):
469 + if self.enable_caching:
470 + for seq in seq_group.seqs_dict.values():
471 + self.compute_last_full_block_in_seq(seq)
vllm/core/evictor.py new
+161
@@ -0,0 +1,161 @@
1 +import enum
2 +from typing import Dict, List, Optional
3 +from abc import ABC, abstractmethod, abstractproperty
4 +
5 +from vllm.block import PhysicalTokenBlock
6 +
7 +
8 +class EvictionPolicy(enum.Enum):
9 + """Enum for eviction policy used by make_evictor to instantiate the correct
10 + Evictor subclass.
11 + """
12 + LRU = enum.auto()
13 + FIFO = enum.auto()
14 +
15 +
16 +class Evictor(ABC):
17 + """The Evictor subclasses should be used by the BlockAllocator class to
18 + handle eviction of freed PhysicalTokenBlocks.
19 + """
20 +
21 + @abstractmethod
22 + def __init__(self):
23 + pass
24 +
25 + @abstractmethod
26 + def __contains__(self, block_hash: int) -> bool:
27 + pass
28 +
29 + @abstractmethod
30 + def evict(self) -> PhysicalTokenBlock:
31 + """Runs the eviction algorithm and returns the evicted block"""
32 + pass
33 +
34 + @abstractmethod
35 + def add(self, block: PhysicalTokenBlock):
36 + """Adds block to the evictor, making it a candidate for eviction"""
37 + pass
38 +
39 + @abstractmethod
40 + def remove(self, block_hash: int) -> PhysicalTokenBlock:
41 + """Simply removes the block with the hash value block_hash from the
42 + evictor. Caller is responsible for making sure that block_hash is contained
43 + in the evictor before calling remove. Should be used to "bring back" blocks
44 + that have been freed but not evicted yet.
45 + """
46 + pass
47 +
48 + @abstractproperty
49 + def num_blocks(self) -> int:
50 + pass
51 +
52 +
53 +class LRUEvictor(Evictor):
54 + """Evicts in a least-recently-used order using the last_accessed timestamp
55 + that's recorded in the PhysicalTokenBlock. If there are multiple blocks with
56 + the same last_accessed time, then the one with the largest num_hashed_tokens
57 + will be evicted. If two blocks each have the lowest last_accessed time and
58 + highest num_hashed_tokens value, then one will be chose arbitrarily
59 + """
60 +
61 + def __init__(self):
62 + self.free_table: Dict[int, PhysicalTokenBlock] = {}
63 +
64 + def __contains__(self, block_hash: int) -> bool:
65 + return block_hash in self.free_table
66 +
67 + # TODO: The performance of this evict function can be optimized further.
68 + def evict(self) -> PhysicalTokenBlock:
69 + free_blocks: List[PhysicalTokenBlock] = list(self.free_table.values())
70 + if len(free_blocks) == 0:
71 + raise ValueError("No usable cache memory left")
72 +
73 + # Find lowest timestamp
74 + lowest_timestamp = free_blocks[0].last_accessed
75 + for block in free_blocks:
76 + if block.last_accessed < lowest_timestamp:
77 + lowest_timestamp = block.last_accessed
78 +
79 + # Find all blocks with the lowest timestamp
80 + least_recent: List[PhysicalTokenBlock] = []
81 + for block in free_blocks:
82 + if block.last_accessed == lowest_timestamp:
83 + least_recent.append(block)
84 +
85 + # Find highest prefix count per block
86 + highest_num_hashed_tokens = 0
87 + for block in least_recent:
88 + if block.num_hashed_tokens > highest_num_hashed_tokens:
89 + highest_num_hashed_tokens = block.num_hashed_tokens
90 +
91 + evicted_block: Optional[PhysicalTokenBlock] = None
92 +
93 + # Find the first block with the lowest timestamp
94 + for block in least_recent:
95 + if block.num_hashed_tokens == highest_num_hashed_tokens:
96 + evicted_block = block
97 + break
98 +
99 + assert evicted_block is not None
100 +
101 + del self.free_table[evicted_block.block_hash]
102 +
103 + evicted_block.computed = False
104 + return evicted_block
105 +
106 + def add(self, block: PhysicalTokenBlock):
107 + self.free_table[block.block_hash] = block
108 +
109 + def remove(self, block_hash: int) -> PhysicalTokenBlock:
110 + if block_hash not in self.free_table:
111 + raise ValueError(
112 + "Attempting to remove block that's not in the evictor")
113 + block: PhysicalTokenBlock = self.free_table[block_hash]
114 + del self.free_table[block_hash]
115 + return block
116 +
117 + @property
118 + def num_blocks(self) -> int:
119 + return len(self.free_table)
120 +
121 +
122 +class RandomEvictor(Evictor):
123 + """Evicts in a first-in-first-out order"""
124 +
125 + def __init__(self):
126 + self.free_table: Dict[int, PhysicalTokenBlock] = {}
127 +
128 + def __contains__(self, block_hash: int) -> bool:
129 + return block_hash in self.free_table
130 +
131 + def evict(self) -> PhysicalTokenBlock:
132 + if len(self.free_table) == 0:
133 + raise ValueError("No usable cache memory left")
134 + evicted_block = next(iter(self.free_table.values()))
135 + evicted_block.computed = False
136 + del self.free_table[evicted_block.block_hash]
137 + return evicted_block
138 +
139 + def add(self, block: PhysicalTokenBlock):
140 + self.free_table[block.block_hash] = block
141 +
142 + def remove(self, block_hash: int) -> PhysicalTokenBlock:
143 + if block_hash not in self.free_table:
144 + raise ValueError(
145 + "Attempting to remove block that's not in the evictor")
146 + block: PhysicalTokenBlock = self.free_table[block_hash]
147 + del self.free_table[block_hash]
148 + return block
149 +
150 + @property
151 + def num_blocks(self) -> int:
152 + return len(self.free_table)
153 +
154 +
155 +def make_evictor(eviction_policy: EvictionPolicy) -> Evictor:
156 + if eviction_policy == EvictionPolicy.LRU:
157 + return LRUEvictor()
158 + elif eviction_policy == EvictionPolicy.FIFO:
159 + return RandomEvictor()
160 + else:
161 + raise ValueError(f"Unknown cache eviction policy: {eviction_policy}")
vllm/core/scheduler.py
+9 -6
@@ -10,7 +10,6 @@ from vllm.lora.request import LoRARequest
10 from vllm.logger import init_logger
11 from vllm.sequence import (Sequence, SequenceData, SequenceGroup,
12 SequenceGroupMetadata, SequenceStatus)
13 -from vllm.prefix import PrefixPool
13
14 logger = init_logger(__name__)
15
@@ -95,10 +94,8 @@ class Scheduler:
94 block_size=self.cache_config.block_size,
95 num_gpu_blocks=self.cache_config.num_gpu_blocks,
96 num_cpu_blocks=self.cache_config.num_cpu_blocks,
98 - sliding_window=self.cache_config.sliding_window)
99 -
100 - # Create the prefix pool to cache the prefixes.
101 - self.prefix_pool = PrefixPool(self.cache_config.block_size)
97 + sliding_window=self.cache_config.sliding_window,
98 + enable_caching=self.cache_config.enable_prefix_caching)
99
100 # Sequence groups in the WAITING state.
101 self.waiting: Deque[SequenceGroup] = deque()
@@ -374,10 +371,12 @@ class Scheduler:
371
372 seq_data: Dict[int, SequenceData] = {}
373 block_tables: Dict[int, List[int]] = {}
374 +
375 for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
376 seq_id = seq.seq_id
377 seq_data[seq_id] = seq.data
378 block_tables[seq_id] = self.block_manager.get_block_table(seq)
379 + self.block_manager.access_all_blocks_in_seq(seq, now)
380
381 seq_group_metadata = SequenceGroupMetadata(
382 request_id=seq_group.request_id,
@@ -386,7 +385,8 @@ class Scheduler:
385 sampling_params=seq_group.sampling_params,
386 block_tables=block_tables,
387 lora_request=seq_group.lora_request,
389 - prefix=seq_group.prefix,
388 + computed_block_nums=self.block_manager.
389 + get_common_computed_block_ids(seq_group),
390 state=seq_group.state,
391 )
392 seq_group_metadata_list.append(seq_group_metadata)
@@ -496,3 +496,6 @@ class Scheduler:
496 blocks_to_swap_out.update(mapping)
497 for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
498 seq.status = SequenceStatus.SWAPPED
499 +
500 + def mark_blocks_as_computed(self, seq_group: SequenceGroup):
501 + self.block_manager.mark_blocks_as_computed(seq_group)
vllm/engine/arg_utils.py
+8 -1
@@ -25,6 +25,7 @@ class EngineArgs:
25 tensor_parallel_size: int = 1
26 max_parallel_loading_workers: Optional[int] = None
27 block_size: int = 16
28 + enable_prefix_caching: bool = False
29 swap_space: int = 4 # GiB
30 gpu_memory_utilization: float = 0.90
31 max_num_batched_tokens: Optional[int] = None
@@ -173,6 +174,11 @@ class EngineArgs:
174 default=EngineArgs.block_size,
175 choices=[8, 16, 32, 128],
176 help='token block size')
177 +
178 + parser.add_argument('--enable-prefix-caching',
179 + action='store_true',
180 + help='Enables automatic prefix caching')
181 +
182 parser.add_argument('--seed',
183 type=int,
184 default=EngineArgs.seed,
@@ -293,7 +299,8 @@ class EngineArgs:
299 cache_config = CacheConfig(self.block_size,
300 self.gpu_memory_utilization,
301 self.swap_space, self.kv_cache_dtype,
296 - model_config.get_sliding_window())
302 + model_config.get_sliding_window(),
303 + self.enable_prefix_caching)
304 parallel_config = ParallelConfig(self.pipeline_parallel_size,
305 self.tensor_parallel_size,
306 self.worker_use_ray,
vllm/engine/async_llm_engine.py
+1 -13
@@ -225,7 +225,6 @@ class _AsyncLLMEngine(LLMEngine):
225 prompt_token_ids: Optional[List[int]] = None,
226 arrival_time: Optional[float] = None,
227 lora_request: Optional[LoRARequest] = None,
228 - prefix_pos: Optional[int] = None,
228 ) -> None:
229 if lora_request is not None and not self.lora_config:
230 raise ValueError(f"Got lora_request {lora_request} but LoRA is "
@@ -245,7 +244,6 @@ class _AsyncLLMEngine(LLMEngine):
244 sampling_params=sampling_params,
245 arrival_time=arrival_time,
246 lora_request=lora_request,
248 - prefix_pos=prefix_pos,
247 )
248
249 async def _run_workers_async(
@@ -422,7 +420,6 @@ class AsyncLLMEngine:
420 prompt_token_ids: Optional[List[int]] = None,
421 arrival_time: Optional[float] = None,
422 lora_request: Optional[LoRARequest] = None,
425 - prefix_pos: Optional[int] = None,
423 ) -> AsyncStream:
424 if self.log_requests:
425 shortened_prompt = prompt
@@ -435,7 +432,6 @@ class AsyncLLMEngine:
432 max_log_len]
433 logger.info(f"Received request {request_id}: "
434 f"prompt: {shortened_prompt!r}, "
438 - f"prefix_pos: {prefix_pos},"
435 f"sampling_params: {sampling_params}, "
436 f"prompt_token_ids: {shortened_token_ids}, "
437 f"lora_request: {lora_request}.")
@@ -472,8 +468,7 @@ class AsyncLLMEngine:
468 sampling_params=sampling_params,
469 prompt_token_ids=prompt_token_ids,
470 arrival_time=arrival_time,
475 - lora_request=lora_request,
476 - prefix_pos=prefix_pos)
471 + lora_request=lora_request)
472
473 return stream
474
@@ -484,7 +479,6 @@ class AsyncLLMEngine:
479 request_id: str,
480 prompt_token_ids: Optional[List[int]] = None,
481 lora_request: Optional[LoRARequest] = None,
487 - prefix_pos: Optional[int] = None,
482 ) -> AsyncIterator[RequestOutput]:
483 """Generate outputs for a request.
484
@@ -500,11 +494,6 @@ class AsyncLLMEngine:
494 prompt_token_ids: The token IDs of the prompt. If None, we
495 use the tokenizer to convert the prompts to token IDs.
496 lora_request: LoRA request to use for generation, if any.
503 - prefix_pos: If not None, we use the given position as the prefix
504 - position for each prompt. We will cache the prefix's KV
505 - cache and reuse it for the next request with the same prefix.
506 - This is an experimental feature, and may be replaced with
507 - automatic prefix caching in the future.
497
498 Yields:
499 The output `RequestOutput` objects from the LLMEngine for the
@@ -565,7 +554,6 @@ class AsyncLLMEngine:
554 prompt_token_ids=prompt_token_ids,
555 arrival_time=arrival_time,
556 lora_request=lora_request,
568 - prefix_pos=prefix_pos,
557 )
558
559 async for request_output in stream:
vllm/engine/llm_engine.py
+8 -18
@@ -415,7 +415,6 @@ class LLMEngine:
415 prompt_token_ids: Optional[List[int]] = None,
416 arrival_time: Optional[float] = None,
417 lora_request: Optional[LoRARequest] = None,
418 - prefix_pos: Optional[int] = None,
418 ) -> None:
419 """Add a request to the engine's request pool.
420
@@ -432,11 +431,6 @@ class LLMEngine:
431 use the tokenizer to convert the prompts to token IDs.
432 arrival_time: The arrival time of the request. If None, we use
433 the current monotonic time.
435 - prefix_pos: If not None, we use the given position as the prefix
436 - position for each prompt. We will cache the prefix's KV
437 - cache and reuse it for the next request with the same prefix.
438 - This is an experimental feature, and may be replaced with
439 - automatic prefix caching in the future.
434
435 Details:
436 - Set arrival_time to the current time if it is None.
@@ -479,18 +473,13 @@ class LLMEngine:
473 seq = Sequence(seq_id, prompt, prompt_token_ids, block_size,
474 lora_request)
475
482 - # Check whether the input specifies prefix
483 - prefix = self.scheduler.prefix_pool.add_or_get_prefix(
484 - prompt_token_ids[:prefix_pos], lora_request.lora_int_id
485 - if lora_request else 0) if prefix_pos is not None else None
486 -
476 # Defensive copy of SamplingParams, which are used by the sampler,
477 # this doesn't deep-copy LogitsProcessor objects
478 sampling_params = sampling_params.clone()
479
480 # Create the sequence group.
481 seq_group = SequenceGroup(request_id, [seq], sampling_params,
493 - arrival_time, lora_request, prefix)
482 + arrival_time, lora_request)
483
484 # Add the sequence group to the scheduler.
485 self.scheduler.add_seq_group(seq_group)
@@ -752,6 +741,13 @@ class LLMEngine:
741 now = time.time()
742 # Update the scheduled sequence groups with the model outputs.
743 scheduled_seq_groups = scheduler_outputs.scheduled_seq_groups
744 +
745 + # If prefix caching is enabled, mark all blocks in the sequence groups
746 + # as completed so that future requests don't attempt to recompute them
747 + if self.cache_config.enable_prefix_caching:
748 + for seq_group in scheduled_seq_groups:
749 + self.scheduler.mark_blocks_as_computed(seq_group)
750 +
751 for seq_group, outputs in zip(scheduled_seq_groups, output):
752 self._process_sequence_group_outputs(seq_group, outputs)
753
@@ -768,12 +764,6 @@ class LLMEngine:
764 request_output = RequestOutput.from_seq_group(seq_group)
765 request_outputs.append(request_output)
766
771 - # Update prefix state, now all the uncomputed prefixes are computed.
772 - for seq_group in scheduled_seq_groups:
773 - if (seq_group.prefix is not None and seq_group.prefix.allocated
774 - and not seq_group.prefix.computed):
775 - seq_group.prefix.computed = True
776 -
767 # Log stats.
768 if self.log_stats:
769 self.stat_logger.log(self._get_stats(scheduler_outputs))
vllm/entrypoints/api_server.py
+1 -5
@@ -39,15 +39,11 @@ async def generate(request: Request) -> Response:
39 """
40 request_dict = await request.json()
41 prompt = request_dict.pop("prompt")
42 - prefix_pos = request_dict.pop("prefix_pos", None)
42 stream = request_dict.pop("stream", False)
43 sampling_params = SamplingParams(**request_dict)
44 request_id = random_uuid()
45
47 - results_generator = engine.generate(prompt,
48 - sampling_params,
49 - request_id,
50 - prefix_pos=prefix_pos)
46 + results_generator = engine.generate(prompt, sampling_params, request_id)
47
48 # Streaming case
49 async def stream_results() -> AsyncGenerator[bytes, None]:
vllm/entrypoints/llm.py
+2 -12
@@ -124,7 +124,6 @@ class LLM:
124 prompts: Optional[Union[str, List[str]]] = None,
125 sampling_params: Optional[SamplingParams] = None,
126 prompt_token_ids: Optional[List[List[int]]] = None,
127 - prefix_pos: Optional[Union[int, List[int]]] = None,
127 use_tqdm: bool = True,
128 lora_request: Optional[LoRARequest] = None,
129 ) -> List[RequestOutput]:
@@ -140,11 +139,6 @@ class LLM:
139 None, we use the default sampling parameters.
140 prompt_token_ids: A list of token IDs for the prompts. If None, we
141 use the tokenizer to convert the prompts to token IDs.
143 - prefix_pos: If not None, we use the given position as the prefix
144 - position for each prompt. We will cache the prefix's KV
145 - cache and reuse it for the next request with the same prefix.
146 - This is an experimental feature, and may be replaced with
147 - automatic prefix caching in the future.
142 use_tqdm: Whether to use tqdm to display the progress bar.
143 lora_request: LoRA request to use for generation, if any.
144
@@ -171,14 +165,12 @@ class LLM:
165 prompt_token_ids)
166 for i in range(num_requests):
167 prompt = prompts[i] if prompts is not None else None
174 - prefix_pos_i = prefix_pos[i] if prefix_pos is not None else None
168 token_ids = None if prompt_token_ids is None else prompt_token_ids[
169 i]
170 self._add_request(prompt,
171 sampling_params,
172 token_ids,
180 - lora_request=lora_request,
181 - prefix_pos=prefix_pos_i)
173 + lora_request=lora_request)
174 return self._run_engine(use_tqdm)
175
176 def _add_request(
@@ -187,15 +179,13 @@ class LLM:
179 sampling_params: SamplingParams,
180 prompt_token_ids: Optional[List[int]],
181 lora_request: Optional[LoRARequest] = None,
190 - prefix_pos: Optional[int] = None,
182 ) -> None:
183 request_id = str(next(self.request_counter))
184 self.llm_engine.add_request(request_id,
185 prompt,
186 sampling_params,
187 prompt_token_ids,
197 - lora_request=lora_request,
198 - prefix_pos=prefix_pos)
188 + lora_request=lora_request)
189
190 def _run_engine(self, use_tqdm: bool) -> List[RequestOutput]:
191 # Initialize tqdm.
vllm/prefix.py deleted
-87
@@ -1,87 +0,0 @@
1 -from typing import Dict, List, Sequence, Tuple, Optional
2 -
3 -from vllm.block import BlockTable
4 -
5 -
6 -class Prefix:
7 - """Data and states associated with a prefix of prompt tokens for multiple
8 - sequence groups.
9 -
10 - NOTE: This feature is experimental and may be replaced with automatic
11 - prefix caching in the future.
12 -
13 - Args:
14 - token_ids: The token ids of the prefix.
15 - block_size: The block size of the executed model.
16 - """
17 -
18 - def __init__(
19 - self,
20 - token_ids: Sequence[int],
21 - block_size: int,
22 - ) -> None:
23 - self.token_ids = tuple(token_ids)
24 - self.block_size = block_size
25 - self.length = len(token_ids)
26 - self.hash = hash(token_ids)
27 - assert self.length % block_size == 0
28 - self.block_table: Optional[BlockTable] = None
29 - self.computed = False
30 -
31 - @property
32 - def allocated(self) -> bool:
33 - return self.block_table is not None
34 -
35 - def get_num_blocks(self) -> int:
36 - return self.length // self.block_size
37 -
38 - def get_block_numbers(self) -> List[int]:
39 - return [block.block_number for block in self.block_table]
40 -
41 - def get_length(self) -> int:
42 - return self.length
43 -
44 - def __hash__(self) -> int:
45 - return self.hash
46 -
47 - def set_block_table(self, block_table: BlockTable) -> None:
48 - self.block_table = block_table.copy()
49 -
50 -
51 -class PrefixPool:
52 - """Manages all the prompt prefixes.
53 -
54 - NOTE: This feature is experimental and may be replaced with automatic
55 - prefix caching in the future.
56 -
57 - Args:
58 - block_size: The block size of the executed model.
59 -
60 - Attributes:
61 - prefixes: A list of all the prefixes.
62 - block_size: The block size of the executed model.
63 - """
64 -
65 - def __init__(
66 - self,
67 - block_size: int,
68 - ) -> None:
69 - # TODO(zhuohan): Add a capacity limit to the prefix pool.
70 - self.prefixes: Dict[int, Prefix] = {}
71 - self.block_size = block_size
72 -
73 - def _truncate_token_ids(self, token_ids: Sequence[int]) -> Tuple[int]:
74 - new_length = len(token_ids) // self.block_size * self.block_size
75 - return tuple(token_ids[:new_length])
76 -
77 - def add_or_get_prefix(self, token_ids: Sequence[int],
78 - lora_int_id: int) -> Optional[Prefix]:
79 - token_ids = self._truncate_token_ids(token_ids)
80 - if len(token_ids) == 0:
81 - # Prefix is empty.
82 - return None
83 - prefix = Prefix(token_ids, self.block_size)
84 - prefix_hash = hash((prefix, lora_int_id))
85 - if prefix_hash not in self.prefixes:
86 - self.prefixes[prefix_hash] = prefix
87 - return self.prefixes[prefix_hash]
vllm/sequence.py
+16 -7
@@ -5,7 +5,6 @@ from dataclasses import dataclass
5 from typing import Dict, List, Optional, Union
6
7 from vllm.block import LogicalTokenBlock
8 -from vllm.prefix import Prefix
8 from vllm.sampling_params import SamplingParams
9 from vllm.lora.request import LoRARequest
10
@@ -161,6 +160,16 @@ class Sequence:
160 def lora_int_id(self) -> int:
161 return self.lora_request.lora_int_id if self.lora_request else 0
162
163 + # TODO The current hashing function is O(L^2). We should optimize this in
164 + # the future.
165 + def hash_of_block(self, logical_idx: int) -> int:
166 + # Compute the number of tokens in the sequence
167 + num_tokens = self.num_hashed_tokens_of_block(logical_idx)
168 + return hash(tuple(self.data.get_token_ids()[0:num_tokens]))
169 +
170 + def num_hashed_tokens_of_block(self, logical_idx: int):
171 + return logical_idx * self.block_size + self.block_size
172 +
173 def _append_logical_block(self) -> None:
174 block = LogicalTokenBlock(
175 block_number=len(self.logical_token_blocks),
@@ -265,7 +274,6 @@ class SequenceGroup:
274 sampling_params: The sampling parameters used to generate the outputs.
275 arrival_time: The arrival time of the request.
276 lora_request: LoRA request.
268 - prefix: The prefix of the prompt of the sequence group.
277 """
278
279 def __init__(
@@ -275,7 +283,6 @@ class SequenceGroup:
283 sampling_params: SamplingParams,
284 arrival_time: float,
285 lora_request: Optional[LoRARequest] = None,
278 - prefix: Optional[Prefix] = None,
286 ) -> None:
287 self.request_id = request_id
288 self.seqs_dict = {seq.seq_id: seq for seq in seqs}
@@ -286,7 +293,6 @@ class SequenceGroup:
293 first_token_time=None,
294 time_in_queue=None)
295 self.lora_request = lora_request
289 - self.prefix: Optional[Prefix] = prefix
296 self.prompt_logprobs: Optional[PromptLogprobs] = None
297 self.state = SequenceGroupState()
298
@@ -302,6 +308,10 @@ class SequenceGroup:
308 # We use the prompt of an arbitrary sequence.
309 return next(iter(self.seqs_dict.values())).data.prompt_token_ids
310
311 + @property
312 + def block_size(self) -> int:
313 + return next(iter(self.seqs_dict.values())).block_size
314 +
315 @property
316 def lora_int_id(self) -> int:
317 return self.lora_request.lora_int_id if self.lora_request else 0
@@ -408,7 +418,6 @@ class SequenceGroupMetadata:
418 numbers)
419 state: Internal state tied to this sequence group.
420 lora_request: LoRA request.
411 - prefix: The prefix of the prompt of the sequence group.
421 """
422
423 def __init__(
@@ -419,7 +428,7 @@ class SequenceGroupMetadata:
428 sampling_params: SamplingParams,
429 block_tables: Dict[int, List[int]],
430 lora_request: Optional[LoRARequest] = None,
422 - prefix: Optional[Prefix] = None,
431 + computed_block_nums: Optional[List[int]] = None,
432 state: Optional[SequenceGroupState] = None,
433 ) -> None:
434 self.request_id = request_id
@@ -428,7 +437,7 @@ class SequenceGroupMetadata:
437 self.sampling_params = sampling_params
438 self.block_tables = block_tables
439 self.lora_request = lora_request
431 - self.prefix = prefix
440 + self.computed_block_nums = computed_block_nums
441 self.state = SequenceGroupState() if state is None else state
442
443 @property
vllm/worker/model_runner.py
+17 -13
@@ -145,33 +145,37 @@ class ModelRunner:
145 prompt_tokens = seq_data.get_token_ids()
146 prompt_len = len(prompt_tokens)
147 prompt_lens.append(prompt_len)
148 - prefix_len = 0
149 - prefix = seq_group_metadata.prefix
150 - if prefix is not None and prefix.computed:
151 - prefix_len = prefix.get_length()
152 - prompt_tokens = prompt_tokens[prefix_len:]
153 - prefix_block_tables.append(prefix.get_block_numbers())
148 + computed_len = 0
149 +
150 + # NOTE: This only works for oooooooxxx style attention.
151 + computed_block_nums = seq_group_metadata.computed_block_nums
152 + if computed_block_nums is not None and len(
153 + computed_block_nums) > 0 and self.sliding_window is None:
154 + # Prefix is not supported with sliding_window
155 + computed_len = len(computed_block_nums) * self.block_size
156 + prompt_tokens = prompt_tokens[computed_len:]
157 + prefix_block_tables.append(computed_block_nums)
158 else:
159 prefix_block_tables.append([])
160 # actual prompt lens
157 - context_lens.append(prefix_len)
158 - subquery_lens.append(prompt_len - prefix_len)
161 + context_lens.append(computed_len)
162 + subquery_lens.append(prompt_len - computed_len)
163
164 input_tokens.append(prompt_tokens)
165 # NOTE(woosuk): Here we assume that the first token in the prompt
166 # is always the first token in the sequence.
167 input_positions.append(
164 - list(range(prefix_len, prefix_len + len(prompt_tokens))))
168 + list(range(computed_len, computed_len + len(prompt_tokens))))
169
170 lora_id = seq_group_metadata.lora_int_id
171
172 if lora_id > 0:
173 lora_requests.add(seq_group_metadata.lora_request)
174
171 - lora_index_mapping.append([lora_id] * (prompt_len - prefix_len))
175 + lora_index_mapping.append([lora_id] * (prompt_len - computed_len))
176 lora_prompt_mapping.extend(
177 [lora_id] *
174 - (prompt_len - prefix_len
178 + (prompt_len - computed_len
179 if seq_group_metadata.sampling_params.prompt_logprobs else 1))
180
181 if seq_group_metadata.block_tables is None:
@@ -190,11 +194,11 @@ class ModelRunner:
194 # mapping will be [-1, -1, 2, 3, 4, 5, 6, 7, 0, 1].
195 start_idx = 0
196 if self.sliding_window is not None:
193 - assert prefix_len == 0, (
197 + assert computed_len == 0, (
198 "Prefix caching is currently not supported with "
199 "sliding window attention")
200 start_idx = max(0, prompt_len - self.sliding_window)
197 - for i in range(prefix_len, prompt_len):
201 + for i in range(computed_len, prompt_len):
202 if i < start_idx:
203 slot_mapping[-1].append(_PAD_SLOT_ID)
204 continue