@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 14ccd94c89
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Core][Bugfix]Refactor block manager for better testability (#3492)

Cade Daniel committed Mar 27, 2024 at 23:59 UTC 14ccd94c89d0ffd9da283545d93ab1dfea5da340
30 files changed +3285 -77
tests/core/block/__init__.py
tests/core/block/e2e/conftest.py new
+56
@@ -0,0 +1,56 @@
1 +import contextlib
2 +import gc
3 +
4 +import pytest
5 +import ray
6 +import torch
7 +
8 +from vllm import LLM
9 +from vllm.model_executor.parallel_utils.parallel_state import (
10 + destroy_model_parallel)
11 +from vllm.model_executor.utils import set_random_seed
12 +
13 +
14 +def cleanup():
15 + destroy_model_parallel()
16 + with contextlib.suppress(AssertionError):
17 + torch.distributed.destroy_process_group()
18 + gc.collect()
19 + torch.cuda.empty_cache()
20 + ray.shutdown()
21 +
22 +
23 +@pytest.fixture
24 +def baseline_llm_generator(common_llm_kwargs, per_test_common_llm_kwargs,
25 + baseline_llm_kwargs, seed):
26 + return create_llm_generator(common_llm_kwargs, per_test_common_llm_kwargs,
27 + baseline_llm_kwargs, seed)
28 +
29 +
30 +@pytest.fixture
31 +def test_llm_generator(common_llm_kwargs, per_test_common_llm_kwargs,
32 + test_llm_kwargs, seed):
33 + return create_llm_generator(common_llm_kwargs, per_test_common_llm_kwargs,
34 + test_llm_kwargs, seed)
35 +
36 +
37 +def create_llm_generator(common_llm_kwargs, per_test_common_llm_kwargs,
38 + distinct_llm_kwargs, seed):
39 + kwargs = {
40 + **common_llm_kwargs,
41 + **per_test_common_llm_kwargs,
42 + **distinct_llm_kwargs,
43 + }
44 +
45 + def generator_inner():
46 + llm = LLM(**kwargs)
47 +
48 + set_random_seed(seed)
49 +
50 + yield llm
51 + del llm
52 + cleanup()
53 +
54 + for llm in generator_inner():
55 + yield llm
56 + del llm
tests/core/block/e2e/test_correctness.py new
+86
@@ -0,0 +1,86 @@
1 +from itertools import cycle
2 +
3 +import pytest
4 +
5 +from vllm import SamplingParams
6 +
7 +
8 +@pytest.mark.parametrize(
9 + "common_llm_kwargs",
10 + [{
11 + # Use a small model for a fast test.
12 + "model": "facebook/opt-125m",
13 +
14 + # skip cuda graph creation for fast test.
15 + "enforce_eager": True,
16 +
17 + # Allow only 5 sequences of ~1024 tokens in worst case.
18 + "block_size": 16,
19 + "forced_num_gpu_blocks": 5 * (64 + 1),
20 + }])
21 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
22 +@pytest.mark.parametrize("baseline_llm_kwargs", [{
23 + "use_v2_block_manager": False
24 +}])
25 +@pytest.mark.parametrize("test_llm_kwargs", [{"use_v2_block_manager": True}])
26 +@pytest.mark.parametrize("batch_size", [10])
27 +@pytest.mark.parametrize("seed", [1])
28 +def test_v1_v2_greedy_equality_with_preemption(baseline_llm_generator,
29 + test_llm_generator, batch_size):
30 + """Verify block manager v2 produces same outputs as block manager v1, even
31 + when there is preemption.
32 +
33 + This constructs two LLM, each with limited number of GPU blocks. The limit
34 + is decided such that as the sequences in the batch grow, sequences must be
35 + preempted and removed from cache.
36 +
37 + If the output token ids are equivalent, then we have confidence that the KV
38 + cache is not corrupted in the v2 block manager.
39 +
40 + NOTE: We want a significant number of generated tokens so that any incorrect
41 + KV mapping has time to build up error.
42 + """
43 + output_len = 1024
44 + temperature = 0.0
45 +
46 + # We want to ensure equality even with preemption.
47 + # We force the total block size to be 1 + cdiv(output_len, block_size)
48 + # so that only one sequence can fit at a time (once the sequences grow).
49 +
50 + prompts = [
51 + "Hello, my name is",
52 + "The president of the United States is",
53 + "The capital of France is",
54 + "The future of AI is",
55 + ]
56 +
57 + prompts = [prompt for prompt, _ in zip(cycle(prompts), range(batch_size))]
58 +
59 + sampling_params = SamplingParams(
60 + max_tokens=output_len,
61 + ignore_eos=True,
62 + temperature=temperature,
63 + )
64 +
65 + print('Getting token ids from block manager v1')
66 + baseline_token_ids = get_token_ids_from_llm_generator(
67 + baseline_llm_generator, prompts, sampling_params)
68 +
69 + print('Getting token ids from block manager v2')
70 + test_token_ids = get_token_ids_from_llm_generator(test_llm_generator,
71 + prompts, sampling_params)
72 +
73 + for expected_token_ids, actual_token_ids in zip(baseline_token_ids,
74 + test_token_ids):
75 + assert expected_token_ids == actual_token_ids
76 +
77 + assert baseline_token_ids == test_token_ids
78 +
79 +
80 +def get_token_ids_from_llm_generator(llm_generator, prompts, sampling_params):
81 + for llm in llm_generator:
82 + outputs = llm.generate(prompts, sampling_params, use_tqdm=True)
83 + token_ids = [output.outputs[0].token_ids for output in outputs]
84 + del llm
85 +
86 + return token_ids
tests/core/block/test_block_space_manager.py new
+50
@@ -0,0 +1,50 @@
1 +import pytest
2 +
3 +from vllm.core.block_manager_v2 import BlockSpaceManagerV2
4 +from vllm.core.interfaces import AllocStatus
5 +
6 +from ..utils import create_seq_group
7 +
8 +
9 +@pytest.mark.parametrize("block_size", [16])
10 +@pytest.mark.parametrize("num_gpu_blocks", [8, 40, 80])
11 +@pytest.mark.parametrize("num_seqs_per_group", [1, 4])
12 +@pytest.mark.parametrize("watermark", [0.0, 0.5])
13 +def test_can_allocate_seq_group(block_size: int, num_seqs_per_group: int,
14 + num_gpu_blocks: int, watermark: float):
15 + block_manager = BlockSpaceManagerV2(
16 + block_size=block_size,
17 + num_gpu_blocks=num_gpu_blocks,
18 + num_cpu_blocks=1024,
19 + watermark=watermark,
20 + )
21 + num_watermark_blocks = int(watermark * num_gpu_blocks)
22 +
23 + num_output_blocks_per_seq = 1
24 +
25 + # NOTE: This should be num_output_blocks_per_seq * num_seqs_per_group, but
26 + # the current implementation assumes all seqs are new prompts / don't have
27 + # different output lens.
28 + num_output_blocks = num_output_blocks_per_seq
29 +
30 + for num_prompt_blocks in range(1, num_gpu_blocks - num_output_blocks):
31 + seq_group = create_seq_group(
32 + seq_prompt_lens=block_size * num_prompt_blocks,
33 + seq_output_lens=[
34 + block_size * num_output_blocks_per_seq
35 + for _ in range(num_seqs_per_group)
36 + ],
37 + )
38 +
39 + assert num_prompt_blocks + num_output_blocks <= num_gpu_blocks
40 +
41 + can_allocate_result = block_manager.can_allocate(seq_group)
42 +
43 + num_required_blocks = num_prompt_blocks + num_output_blocks
44 +
45 + if num_gpu_blocks - num_required_blocks < num_watermark_blocks:
46 + assert can_allocate_result == AllocStatus.NEVER
47 + elif num_gpu_blocks >= num_required_blocks:
48 + assert can_allocate_result == AllocStatus.OK
49 + else:
50 + assert can_allocate_result == AllocStatus.LATER
tests/core/block/test_block_table.py new
+500
@@ -0,0 +1,500 @@
1 +import pytest
2 +
3 +from vllm.core.block.block_table import BlockTable
4 +from vllm.core.block.cpu_gpu_block_allocator import CpuGpuBlockAllocator
5 +from vllm.utils import Device, cdiv, chunk_list
6 +
7 +
8 +@pytest.mark.parametrize("block_size", [16])
9 +@pytest.mark.parametrize("sequence_len", [1, 16, 129])
10 +def test_allocate_naive(block_size: int, sequence_len: int):
11 + """Test the allocation of blocks using the naive allocator.
12 +
13 + This test creates a CpuGpuBlockAllocator with the specified block size and
14 + number of blocks. It then allocates multiple BlockTables with varying
15 + sequence lengths and verifies that the number of free blocks decreases as
16 + expected after each allocation.
17 + """
18 + assert block_size > 1
19 + num_gpu_blocks = 1024
20 +
21 + allocator = CpuGpuBlockAllocator.create(
22 + allocator_type="naive",
23 + num_gpu_blocks=num_gpu_blocks,
24 + num_cpu_blocks=1024,
25 + block_size=block_size,
26 + )
27 +
28 + token_ids = list(range(sequence_len))
29 + num_blocks_per_alloc = len(list(chunk_list(token_ids, block_size)))
30 +
31 + block_tables = []
32 + for i in range(5):
33 + assert allocator.get_num_free_blocks(
34 + device=Device.GPU) == num_gpu_blocks - i * num_blocks_per_alloc
35 +
36 + block_tables.append(
37 + BlockTable(
38 + block_size=block_size,
39 + block_allocator=allocator,
40 + ))
41 + block_tables[-1].allocate(token_ids=token_ids, device=Device.GPU)
42 +
43 +
44 +@pytest.mark.parametrize("block_size", [16])
45 +@pytest.mark.parametrize("sequence_len", [1, 16, 129])
46 +def test_allocate_prefix_caching(block_size: int, sequence_len: int):
47 + """Test the allocation of blocks using the prefix caching allocator.
48 +
49 + This test creates a CpuGpuBlockAllocator with the specified block size and
50 + number of blocks, using the prefix caching allocator. It then allocates
51 + multiple BlockTables with varying sequence lengths and verifies that the
52 + number of free blocks decreases as expected after each allocation.
53 +
54 + The test expects all sequences to share allocations, except for their last
55 + block, which may be mutable. It calculates the expected number of immutable
56 + and mutable blocks per allocation based on the sequence length and block
57 + size.
58 + """
59 + assert block_size > 1
60 + num_gpu_blocks = 1024
61 +
62 + allocator = CpuGpuBlockAllocator.create(
63 + allocator_type="prefix_caching",
64 + num_gpu_blocks=num_gpu_blocks,
65 + num_cpu_blocks=1024,
66 + block_size=block_size,
67 + )
68 +
69 + token_ids = list(range(sequence_len))
70 + chunked_tokens = list(chunk_list(token_ids, block_size))
71 + num_mutable_blocks_per_alloc = 0 if len(
72 + chunked_tokens[-1]) == block_size else 1
73 + num_immutable_blocks_per_alloc = len(
74 + chunked_tokens) - num_mutable_blocks_per_alloc
75 +
76 + block_tables = []
77 + for alloc_i in range(1, 6):
78 +
79 + block_tables.append(
80 + BlockTable(
81 + block_size=block_size,
82 + block_allocator=allocator,
83 + ))
84 + block_tables[-1].allocate(token_ids=token_ids, device=Device.GPU)
85 +
86 + # Expect all sequences to share allocations, except for their last block
87 + # (which may be mutable).
88 + assert allocator.get_num_free_blocks(
89 + device=Device.GPU) == num_gpu_blocks - (
90 + num_immutable_blocks_per_alloc + num_mutable_blocks_per_alloc *
91 + (alloc_i))
92 +
93 +
94 +@pytest.mark.parametrize("block_size", [16])
95 +@pytest.mark.parametrize("sequence_len", [1, 16, 129])
96 +@pytest.mark.parametrize("allocator_type", ["naive", "prefix_caching"])
97 +@pytest.mark.parametrize("device", ["cpu", "gpu"])
98 +def test_allocate_free(block_size: int, sequence_len: int, allocator_type: str,
99 + device: str):
100 + """Test the allocation and freeing of blocks using different allocators and
101 + devices.
102 +
103 + This test creates a CpuGpuBlockAllocator with the specified block size,
104 + number of blocks, allocator type, and device. It then allocates a BlockTable
105 + multiple times with the same sequence and verifies that the number of free
106 + blocks remains consistent after each allocation and freeing.
107 + """
108 + device = Device[device.upper()]
109 +
110 + num_device_blocks = 1024
111 + allocator = CpuGpuBlockAllocator.create(
112 + allocator_type=allocator_type,
113 + num_gpu_blocks=num_device_blocks,
114 + num_cpu_blocks=num_device_blocks,
115 + block_size=block_size,
116 + )
117 +
118 + token_ids = list(range(sequence_len))
119 + num_blocks_per_alloc = len(list(chunk_list(token_ids, block_size)))
120 +
121 + block_table = BlockTable(
122 + block_size=block_size,
123 + block_allocator=allocator,
124 + )
125 +
126 + for i in range(5):
127 + block_table.allocate(token_ids=token_ids, device=device)
128 + assert allocator.get_num_free_blocks(
129 + device) == num_device_blocks - num_blocks_per_alloc
130 + assert all(block_id is not None
131 + for block_id in block_table.physical_block_ids)
132 +
133 + block_table.free()
134 + assert allocator.get_num_free_blocks(device) == num_device_blocks
135 +
136 +
137 +@pytest.mark.parametrize("block_size", [1, 8])
138 +@pytest.mark.parametrize("sequence_len", [1, 16, 129])
139 +@pytest.mark.parametrize("append_len", [1, 16, 129])
140 +@pytest.mark.parametrize("allocator_type", ["naive", "prefix_caching"])
141 +def test_append_token_ids_allocation(block_size: int, sequence_len: int,
142 + append_len: int, allocator_type: str):
143 + """Test the allocation behavior when appending token IDs to a BlockTable.
144 +
145 + This test creates a CpuGpuBlockAllocator with the specified block size,
146 + number of blocks, and allocator type. It then allocates a BlockTable with an
147 + initial sequence and appends additional token IDs to it. The test verifies
148 + that the number of allocated blocks before and after appending matches the
149 + expected values.
150 + """
151 +
152 + num_gpu_blocks = 1024
153 +
154 + allocator = CpuGpuBlockAllocator.create(
155 + allocator_type=allocator_type,
156 + num_gpu_blocks=num_gpu_blocks,
157 + num_cpu_blocks=1024,
158 + block_size=block_size,
159 + )
160 +
161 + token_ids = list(range(sequence_len))
162 + token_ids_to_append = list(range(append_len))
163 +
164 + block_table = BlockTable(
165 + block_size=block_size,
166 + block_allocator=allocator,
167 + )
168 +
169 + num_expected_blocks_before_append = len(
170 + list(chunk_list(token_ids, block_size)))
171 + num_expected_appended_blocks = len(
172 + list(chunk_list(token_ids + token_ids_to_append,
173 + block_size))) - num_expected_blocks_before_append
174 +
175 + block_table.allocate(token_ids=token_ids, device=Device.GPU)
176 +
177 + assert len(
178 + block_table.physical_block_ids) == num_expected_blocks_before_append
179 + block_table.append_token_ids(token_ids_to_append)
180 + assert len(
181 + block_table.physical_block_ids
182 + ) == num_expected_blocks_before_append + num_expected_appended_blocks
183 +
184 +
185 +@pytest.mark.parametrize("block_size", [1, 8])
186 +@pytest.mark.parametrize("sequence_len", [1, 16, 129])
187 +@pytest.mark.parametrize("num_empty_slots", [1, 16, 129])
188 +@pytest.mark.parametrize("allocator_type", ["naive", "prefix_caching"])
189 +def test_ensure_num_empty_slots_allocation(block_size: int, sequence_len: int,
190 + num_empty_slots: int,
191 + allocator_type: str):
192 + """Test the allocation behavior when ensuring a certain number of empty
193 + slots in a BlockTable.
194 +
195 + This test creates a CpuGpuBlockAllocator with the specified block size,
196 + number of blocks, and allocator type. It then allocates a BlockTable with an
197 + initial sequence and ensures a certain number of empty slots. The test
198 + verifies that the number of allocated blocks before and after ensuring empty
199 + slots matches the expected values. It also checks that filling up the empty
200 + slots does not consume additional blocks.
201 + """
202 + num_gpu_blocks = 1024
203 +
204 + allocator = CpuGpuBlockAllocator.create(
205 + allocator_type=allocator_type,
206 + num_gpu_blocks=num_gpu_blocks,
207 + num_cpu_blocks=1024,
208 + block_size=block_size,
209 + )
210 +
211 + token_ids = list(range(sequence_len))
212 +
213 + block_table = BlockTable(
214 + block_size=block_size,
215 + block_allocator=allocator,
216 + )
217 +
218 + num_expected_blocks_before_append = len(
219 + list(chunk_list(token_ids, block_size)))
220 + num_expected_appended_blocks = len(
221 + list(chunk_list(token_ids + [-1] * num_empty_slots,
222 + block_size))) - num_expected_blocks_before_append
223 +
224 + block_table.allocate(token_ids=token_ids, device=Device.GPU)
225 +
226 + # Assert that the empty slots consume the expected number of additional
227 + # blocks.
228 + assert len(
229 + block_table.physical_block_ids) == num_expected_blocks_before_append
230 + block_table.ensure_num_empty_slots(num_empty_slots)
231 + assert len(
232 + block_table.physical_block_ids
233 + ) == num_expected_blocks_before_append + num_expected_appended_blocks
234 +
235 + # Now, ensure no additional blocks consumed as we fill up the empty slots.
236 + num_free_blocks = allocator.get_num_free_blocks(device=Device.GPU)
237 + block_table.append_token_ids(token_ids=list(range(num_empty_slots)))
238 + assert num_free_blocks == allocator.get_num_free_blocks(device=Device.GPU)
239 +
240 +
241 +@pytest.mark.parametrize("block_size", [1, 8])
242 +@pytest.mark.parametrize("sequence_len", [1, 9])
243 +@pytest.mark.parametrize("append_len", [1, 16, 129])
244 +@pytest.mark.parametrize("append_size", [1, 4, 129])
245 +@pytest.mark.parametrize("allocator_type", ["naive", "prefix_caching"])
246 +def test_append_token_ids_correct_content(block_size: int, sequence_len: int,
247 + append_len: int, allocator_type: str,
248 + append_size: int):
249 + """Verify token ids are correctly appended. Appends various amounts of
250 + token ids in various append sizes, and verifies the final sequence is
251 + correct.
252 + """
253 + num_gpu_blocks = 1024
254 +
255 + allocator = CpuGpuBlockAllocator.create(
256 + allocator_type=allocator_type,
257 + num_gpu_blocks=num_gpu_blocks,
258 + num_cpu_blocks=1024,
259 + block_size=block_size,
260 + )
261 +
262 + token_ids = list(range(sequence_len))
263 + token_ids_to_append = list(range(append_len))
264 +
265 + block_table = BlockTable(
266 + block_size=block_size,
267 + block_allocator=allocator,
268 + )
269 + block_table.allocate(token_ids=token_ids, device=Device.GPU)
270 +
271 + appended_so_far = []
272 + for append in chunk_list(token_ids_to_append, append_size):
273 + block_table.append_token_ids(append)
274 + appended_so_far.extend(append)
275 +
276 + assert block_table._get_all_token_ids() == token_ids + appended_so_far
277 +
278 + assert block_table._get_all_token_ids() == token_ids + token_ids_to_append
279 +
280 +
281 +@pytest.mark.parametrize("seq_len", [1, 9, 129])
282 +@pytest.mark.parametrize("block_size", [1, 8])
283 +@pytest.mark.parametrize("allocator_type", ["naive", "prefix_caching"])
284 +def test_fork(seq_len: int, block_size: int, allocator_type: str):
285 + """Create a sequence using the specified allocator.
286 + 1. Assert that after forking the sequence, the free block count is the
287 + same.
288 + 2. Assert that the forked sequence has the same physical mappings.
289 + 3. Then free the original sequence; verify that the free block count is
290 + the same.
291 + 4. Finally, free the forked sequence and verify that the free block
292 + count drops to zero.
293 + """
294 + num_gpu_blocks = 1024
295 +
296 + allocator = CpuGpuBlockAllocator.create(
297 + allocator_type=allocator_type,
298 + num_gpu_blocks=num_gpu_blocks,
299 + num_cpu_blocks=0,
300 + block_size=block_size,
301 + )
302 +
303 + token_ids = list(range(seq_len))
304 +
305 + block_table = BlockTable(
306 + block_size=block_size,
307 + block_allocator=allocator,
308 + )
309 +
310 + block_table.allocate(token_ids)
311 +
312 + num_free_blocks_before_fork = allocator.get_num_free_blocks(
313 + device=Device.GPU)
314 +
315 + forked_block_table = block_table.fork()
316 +
317 + # Expect physical_block_ids and token_ids to match.
318 + assert (block_table.physical_block_ids ==
319 + forked_block_table.physical_block_ids)
320 + assert block_table._get_all_token_ids(
321 + ) == forked_block_table._get_all_token_ids()
322 +
323 + # Do not expect any additional allocations.
324 + assert allocator.get_num_free_blocks(
325 + device=Device.GPU) == num_free_blocks_before_fork
326 +
327 + # Free the original blocks. Assert num free blocks does not change, since
328 + # refcount is nonzero.
329 + block_table.free()
330 + assert allocator.get_num_free_blocks(
331 + device=Device.GPU) == num_free_blocks_before_fork
332 +
333 + # Expect the forked block table to be unaffected by the free.
334 + assert all(block_id is not None
335 + for block_id in forked_block_table.physical_block_ids)
336 +
337 + # Free the forked blocks. Assert num free blocks does change, since
338 + # refcount is now zero.
339 + forked_block_table.free()
340 + assert allocator.get_num_free_blocks(device=Device.GPU) == num_gpu_blocks
341 +
342 +
343 +@pytest.mark.parametrize("block_size", [8])
344 +@pytest.mark.parametrize("sequence_len", [1, 16, 129])
345 +@pytest.mark.parametrize("append_len", [1, 16, 129])
346 +@pytest.mark.parametrize("appender", ["forked", "original"])
347 +@pytest.mark.parametrize("allocator_type", ["naive", "prefix_caching"])
348 +def test_cow(block_size: int, sequence_len: int, append_len: int,
349 + allocator_type: str, appender: str):
350 + """Fork a sequence; append to the forked sequence; verify there's a CoW.
351 + """
352 + num_gpu_blocks = 1024
353 +
354 + allocator = CpuGpuBlockAllocator.create(
355 + allocator_type=allocator_type,
356 + num_gpu_blocks=num_gpu_blocks,
357 + num_cpu_blocks=0,
358 + block_size=block_size,
359 + )
360 +
361 + token_ids = list(range(sequence_len))
362 + token_ids_to_append = list(range(append_len))
363 +
364 + original_block_table = BlockTable(
365 + block_size=block_size,
366 + block_allocator=allocator,
367 + )
368 +
369 + num_expected_non_cow_blocks = cdiv(sequence_len, block_size)
370 + num_expected_cow_blocks = cdiv(sequence_len + append_len,
371 + block_size) - (sequence_len // block_size)
372 +
373 + original_block_table.allocate(token_ids=token_ids, device=Device.GPU)
374 + original_block_ids = original_block_table.physical_block_ids
375 +
376 + forked_block_table = original_block_table.fork()
377 +
378 + # Expect no additional allocation (copy on _write_).
379 + assert allocator.get_num_free_blocks(
380 + Device.GPU) == (num_gpu_blocks - num_expected_non_cow_blocks)
381 +
382 + if appender == "forked":
383 + appender_block_table = forked_block_table
384 + static_block_table = original_block_table
385 + elif appender == "original":
386 + appender_block_table = original_block_table
387 + static_block_table = forked_block_table
388 + else:
389 + raise ValueError(f"unknown test config {appender=}")
390 +
391 + # Write tokens.
392 + appender_block_table.append_token_ids(token_ids_to_append)
393 +
394 + # Expect the non-appending block table to have no change.
395 + assert static_block_table.physical_block_ids == original_block_ids
396 + assert appender_block_table.physical_block_ids != original_block_ids
397 +
398 + # Expect the blocks changed during append to have a CoW.
399 + assert allocator.get_num_free_blocks(
400 + Device.GPU) == num_gpu_blocks - (num_expected_non_cow_blocks +
401 + num_expected_cow_blocks)
402 +
403 + cows = allocator.clear_copy_on_writes()
404 + if sequence_len % block_size > 0:
405 + # If the last block in the sequence is not full, then when appending we
406 + # expect a CoW.
407 + assert cows
408 +
409 + cow_block_id = sequence_len // block_size
410 + expected_src = static_block_table.physical_block_ids[cow_block_id]
411 + expected_dst = appender_block_table.physical_block_ids[cow_block_id]
412 +
413 + assert expected_src in cows
414 + assert expected_dst in cows[expected_src]
415 + else:
416 + # Otherwise, there should be no copy-on-write.
417 + assert not cows
418 +
419 + static_block_table.free()
420 + appender_block_table.free()
421 +
422 + # After free, expect all blocks to be freed.
423 + assert allocator.get_num_free_blocks(Device.GPU) == num_gpu_blocks
424 +
425 +
426 +@pytest.mark.parametrize("block_size", [8])
427 +@pytest.mark.parametrize("sequence_len", [1, 16, 129])
428 +@pytest.mark.parametrize("append_len", [1, 16, 129])
429 +@pytest.mark.parametrize("lookahead_slots", [1, 16, 129])
430 +@pytest.mark.parametrize("appender", ["forked", "original"])
431 +@pytest.mark.parametrize("allocator_type", ["naive", "prefix_caching"])
432 +def test_cow_lookahead_simple(block_size: int, sequence_len: int,
433 + append_len: int, lookahead_slots: int,
434 + allocator_type: str, appender: str):
435 + """Similar to test_cow, except with lookahead allocation. The assertions are
436 + less rigorous due to the complexity of the property under test.
437 + """
438 + num_gpu_blocks = 1024
439 +
440 + allocator = CpuGpuBlockAllocator.create(
441 + allocator_type=allocator_type,
442 + num_gpu_blocks=num_gpu_blocks,
443 + num_cpu_blocks=0,
444 + block_size=block_size,
445 + )
446 +
447 + token_ids = list(range(sequence_len))
448 + token_ids_to_append = list(range(append_len))
449 +
450 + original_block_table = BlockTable(
451 + block_size=block_size,
452 + block_allocator=allocator,
453 + )
454 +
455 + original_block_table.allocate(token_ids=token_ids, device=Device.GPU)
456 +
457 + # Allocate lookahead slots.
458 + original_block_table.ensure_num_empty_slots(lookahead_slots)
459 + original_block_ids = original_block_table.physical_block_ids
460 +
461 + forked_block_table = original_block_table.fork()
462 +
463 + if appender == "forked":
464 + appender_block_table = forked_block_table
465 + static_block_table = original_block_table
466 + elif appender == "original":
467 + appender_block_table = original_block_table
468 + static_block_table = forked_block_table
469 + else:
470 + raise ValueError(f"unknown test config {appender=}")
471 +
472 + # Write tokens.
473 + appender_block_table.append_token_ids(token_ids_to_append)
474 +
475 + # Expect the non-appending block table to have no change.
476 + assert static_block_table.physical_block_ids == original_block_ids
477 + assert appender_block_table.physical_block_ids != original_block_ids
478 +
479 + cows = allocator.clear_copy_on_writes()
480 +
481 + # Always expect copy-on-write
482 + assert cows
483 +
484 + if sequence_len % block_size > 0:
485 + # If the last block in the sequence is not full, then when appending we
486 + # expect a CoW.
487 + assert cows
488 +
489 + cow_block_id = sequence_len // block_size
490 + expected_src = static_block_table.physical_block_ids[cow_block_id]
491 + expected_dst = appender_block_table.physical_block_ids[cow_block_id]
492 +
493 + assert expected_src in cows
494 + assert expected_dst in cows[expected_src]
495 +
496 + static_block_table.free()
497 + appender_block_table.free()
498 +
499 + # After free, expect all blocks to be freed.
500 + assert allocator.get_num_free_blocks(Device.GPU) == num_gpu_blocks
tests/core/block/test_common.py new
+42
@@ -0,0 +1,42 @@
1 +import random
2 +
3 +import pytest
4 +
5 +from vllm.core.block.common import RefCounter
6 +
7 +
8 +@pytest.mark.parametrize("seed", list(range(20)))
9 +@pytest.mark.parametrize("num_incrs", [1, 100])
10 +@pytest.mark.parametrize("num_blocks", [1024])
11 +def test_incr(seed: int, num_incrs: int, num_blocks: int):
12 + random.seed(seed)
13 +
14 + all_block_indices = list(range(num_blocks))
15 + counter = RefCounter(all_block_indices=all_block_indices)
16 +
17 + block_id = random.randint(0, num_blocks - 1)
18 + for i in range(num_incrs):
19 + value = counter.incr(block_id)
20 + assert value == i + 1
21 +
22 +
23 +@pytest.mark.parametrize("seed", list(range(20)))
24 +@pytest.mark.parametrize("num_incrs", [1, 100])
25 +@pytest.mark.parametrize("num_blocks", [1024])
26 +def test_incr_decr(seed: int, num_incrs: int, num_blocks: int):
27 + random.seed(seed)
28 +
29 + all_block_indices = list(range(num_blocks))
30 + counter = RefCounter(all_block_indices=all_block_indices)
31 +
32 + block_id = random.randint(0, num_blocks - 1)
33 + for i in range(num_incrs):
34 + value = counter.incr(block_id)
35 + assert value == i + 1
36 +
37 + for i in range(num_incrs):
38 + value = counter.decr(block_id)
39 + assert value == num_incrs - (i + 1)
40 +
41 + with pytest.raises(AssertionError):
42 + counter.decr(block_id)
tests/core/block/test_cpu_gpu_block_allocator.py new
+93
@@ -0,0 +1,93 @@
1 +import pytest
2 +
3 +from vllm.core.block.cpu_gpu_block_allocator import CpuGpuBlockAllocator
4 +from vllm.utils import Device, chunk_list
5 +
6 +
7 +@pytest.mark.parametrize("num_cpu_blocks", [0, 512])
8 +@pytest.mark.parametrize("num_gpu_blocks", [1024])
9 +@pytest.mark.parametrize("block_size", [16])
10 +@pytest.mark.parametrize("allocator_type", ["naive", "prefix_caching"])
11 +def test_allocate_mutable(num_cpu_blocks: int, num_gpu_blocks: int,
12 + block_size: int, allocator_type: str):
13 + allocator = CpuGpuBlockAllocator.create(
14 + allocator_type=allocator_type,
15 + num_gpu_blocks=num_gpu_blocks,
16 + num_cpu_blocks=num_cpu_blocks,
17 + block_size=block_size,
18 + )
19 +
20 + assert allocator.get_num_free_blocks(Device.CPU) == num_cpu_blocks
21 + assert allocator.get_num_free_blocks(Device.GPU) == num_gpu_blocks
22 +
23 + cpu_blocks = [
24 + allocator.allocate_mutable(prev_block=None, device=Device.CPU)
25 + for _ in range(num_cpu_blocks)
26 + ]
27 + assert allocator.get_num_free_blocks(Device.CPU) == 0
28 + assert allocator.get_num_free_blocks(Device.GPU) == num_gpu_blocks
29 +
30 + gpu_blocks = [
31 + allocator.allocate_mutable(prev_block=None, device=Device.GPU)
32 + for _ in range(num_gpu_blocks)
33 + ]
34 + assert allocator.get_num_free_blocks(Device.CPU) == 0
35 + assert allocator.get_num_free_blocks(Device.GPU) == 0
36 +
37 + _ = [allocator.free(block) for block in cpu_blocks]
38 + assert allocator.get_num_free_blocks(Device.CPU) == num_cpu_blocks
39 + assert allocator.get_num_free_blocks(Device.GPU) == 0
40 +
41 + _ = [allocator.free(block) for block in gpu_blocks]
42 + assert allocator.get_num_free_blocks(Device.CPU) == num_cpu_blocks
43 + assert allocator.get_num_free_blocks(Device.GPU) == num_gpu_blocks
44 +
45 +
46 +@pytest.mark.parametrize("num_cpu_blocks", [0, 512])
47 +@pytest.mark.parametrize("num_gpu_blocks", [1024])
48 +@pytest.mark.parametrize("block_size", [2])
49 +@pytest.mark.parametrize("allocator_type", ["naive", "prefix_caching"])
50 +def test_allocate_immutable(num_cpu_blocks: int, num_gpu_blocks: int,
51 + block_size: int, allocator_type: str):
52 + allocator = CpuGpuBlockAllocator.create(
53 + allocator_type=allocator_type,
54 + num_gpu_blocks=num_gpu_blocks,
55 + num_cpu_blocks=num_cpu_blocks,
56 + block_size=block_size,
57 + )
58 +
59 + unique_token_ids = list(
60 + range((num_cpu_blocks + num_gpu_blocks) * block_size))
61 + gpu_token_ids = chunk_list(unique_token_ids[:num_gpu_blocks * block_size],
62 + block_size)
63 + cpu_token_ids = chunk_list(unique_token_ids[num_gpu_blocks * block_size:],
64 + block_size)
65 +
66 + assert allocator.get_num_free_blocks(Device.CPU) == num_cpu_blocks
67 + assert allocator.get_num_free_blocks(Device.GPU) == num_gpu_blocks
68 +
69 + cpu_blocks = [
70 + allocator.allocate_immutable(prev_block=None,
71 + token_ids=token_ids,
72 + device=Device.CPU)
73 + for token_ids in cpu_token_ids
74 + ]
75 + assert allocator.get_num_free_blocks(Device.CPU) == 0
76 + assert allocator.get_num_free_blocks(Device.GPU) == num_gpu_blocks
77 +
78 + gpu_blocks = [
79 + allocator.allocate_immutable(prev_block=None,
80 + token_ids=token_ids,
81 + device=Device.GPU)
82 + for token_ids in gpu_token_ids
83 + ]
84 + assert allocator.get_num_free_blocks(Device.CPU) == 0
85 + assert allocator.get_num_free_blocks(Device.GPU) == 0
86 +
87 + _ = [allocator.free(block) for block in cpu_blocks]
88 + assert allocator.get_num_free_blocks(Device.CPU) == num_cpu_blocks
89 + assert allocator.get_num_free_blocks(Device.GPU) == 0
90 +
91 + _ = [allocator.free(block) for block in gpu_blocks]
92 + assert allocator.get_num_free_blocks(Device.CPU) == num_cpu_blocks
93 + assert allocator.get_num_free_blocks(Device.GPU) == num_gpu_blocks
tests/core/block/test_naive_block.py new
+102
@@ -0,0 +1,102 @@
1 +from typing import List, Optional
2 +
3 +import pytest
4 +
5 +from vllm.core.block.interfaces import Block, BlockAllocator
6 +from vllm.core.block.naive_block import NaiveBlock, NaiveBlockAllocator
7 +
8 +
9 +class TestNaiveBlockAllocator:
10 +
11 + @staticmethod
12 + def create_allocate_lambda(allocate_type: str,
13 + allocator: NaiveBlockAllocator,
14 + prev_block: Optional[Block],
15 + token_ids: List[int]):
16 + if allocate_type == "immutable":
17 + allocate_block = lambda: allocator.allocate_immutable(
18 + prev_block=prev_block, token_ids=token_ids)
19 + elif allocate_type == "mutable":
20 + allocate_block = lambda: allocator.allocate_mutable(prev_block=
21 + prev_block)
22 + else:
23 + raise ValueError()
24 +
25 + return allocate_block
26 +
27 + @staticmethod
28 + @pytest.mark.parametrize("allocate_type", ["immutable", "mutable"])
29 + @pytest.mark.parametrize("num_blocks", [1, 1024])
30 + @pytest.mark.parametrize("block_size", [1, 16])
31 + def test_allocate_ooms(allocate_type: str, num_blocks: int,
32 + block_size: int):
33 + allocator = NaiveBlockAllocator(create_block=NaiveBlock,
34 + num_blocks=num_blocks,
35 + block_size=block_size)
36 + allocate_block = TestNaiveBlockAllocator.create_allocate_lambda(
37 + allocate_type,
38 + allocator,
39 + prev_block=None,
40 + token_ids=list(range(block_size)))
41 +
42 + [allocate_block() for _ in range(num_blocks)]
43 + with pytest.raises(BlockAllocator.NoFreeBlocksError):
44 + allocate_block()
45 +
46 + @staticmethod
47 + @pytest.mark.parametrize("allocate_type", ["immutable", "mutable"])
48 + @pytest.mark.parametrize("num_blocks", [1, 1024])
49 + @pytest.mark.parametrize("block_size", [1, 16])
50 + def test_free_prevents_oom(allocate_type: str, num_blocks: int,
51 + block_size: int):
52 + allocator = NaiveBlockAllocator(create_block=NaiveBlock,
53 + num_blocks=num_blocks,
54 + block_size=block_size)
55 + allocate_block = TestNaiveBlockAllocator.create_allocate_lambda(
56 + allocate_type,
57 + allocator,
58 + prev_block=None,
59 + token_ids=list(range(block_size)))
60 +
61 + blocks = [allocate_block() for _ in range(num_blocks)]
62 +
63 + with pytest.raises(BlockAllocator.NoFreeBlocksError):
64 + allocate_block()
65 +
66 + block_to_free = blocks.pop()
67 +
68 + for _ in range(100):
69 + block_id = block_to_free.block_id
70 + allocator.free(block_to_free)
71 + assert block_to_free.block_id is None
72 +
73 + new_block = allocate_block()
74 + assert new_block.block_id == block_id
75 +
76 + with pytest.raises(BlockAllocator.NoFreeBlocksError):
77 + allocate_block()
78 +
79 + block_to_free = new_block
80 +
81 + @staticmethod
82 + @pytest.mark.parametrize("allocate_type", ["immutable", "mutable"])
83 + @pytest.mark.parametrize("num_blocks", [1024])
84 + @pytest.mark.parametrize("block_size", [16])
85 + def test_get_num_free_blocks(allocate_type: str, num_blocks: int,
86 + block_size: int):
87 + allocator = NaiveBlockAllocator(create_block=NaiveBlock,
88 + num_blocks=num_blocks,
89 + block_size=block_size)
90 + allocate_block = TestNaiveBlockAllocator.create_allocate_lambda(
91 + allocate_type,
92 + allocator,
93 + prev_block=None,
94 + token_ids=list(range(block_size)))
95 +
96 + assert allocator.get_num_free_blocks() == num_blocks
97 +
98 + blocks = [allocate_block() for _ in range(num_blocks)]
99 +
100 + for i, block in enumerate(blocks):
101 + assert allocator.get_num_free_blocks() == i
102 + allocator.free(block)
tests/core/block/test_prefix_caching_block.py new
+384
@@ -0,0 +1,384 @@
1 +import math
2 +import random
3 +from typing import List, Optional
4 +from unittest.mock import MagicMock
5 +
6 +import pytest
7 +
8 +from vllm.core.block.interfaces import Block, BlockAllocator
9 +from vllm.core.block.prefix_caching_block import (PrefixCachingBlock,
10 + PrefixCachingBlockAllocator)
11 +
12 +
13 +class TestPrefixCachingBlock:
14 +
15 + @staticmethod
16 + @pytest.mark.parametrize("seed", list(range(10)))
17 + @pytest.mark.parametrize("block_size", [1, 16])
18 + @pytest.mark.parametrize("is_curr_block_full", [True, False])
19 + def test_first_block_has_correct_content_hash(seed: int, block_size: int,
20 + is_curr_block_full: bool):
21 + """Verify a block which is first in the sequence has the correct hash.
22 + """
23 + random.seed(seed)
24 + num_to_fill = block_size if is_curr_block_full else random.randint(
25 + 0, block_size - 1)
26 + token_ids = list(range(num_to_fill))
27 + mock_allocator = MagicMock(spec=PrefixCachingBlockAllocator)
28 +
29 + block_with_prev = PrefixCachingBlock(
30 + prev_block=None,
31 + token_ids=token_ids,
32 + block_size=block_size,
33 + prefix_caching_allocator=mock_allocator)
34 +
35 + if is_curr_block_full:
36 + # Expect hash since block is full.
37 + assert block_with_prev.content_hash == (
38 + PrefixCachingBlock.hash_block_tokens(
39 + is_first_block=True,
40 + prev_block_hash=None,
41 + cur_block_token_ids=token_ids))
42 + else:
43 + # Do not expect hash since block is not full.
44 + assert block_with_prev.content_hash is None
45 +
46 + @staticmethod
47 + @pytest.mark.parametrize("seed", list(range(10)))
48 + @pytest.mark.parametrize("block_size", [1, 16])
49 + @pytest.mark.parametrize("is_curr_block_full", [True, False])
50 + @pytest.mark.parametrize("prev_block_has_hash", [True, False])
51 + def test_nth_block_has_correct_content_hash(seed: int, block_size: int,
52 + is_curr_block_full: bool,
53 + prev_block_has_hash: bool):
54 + """Verify a block which is not first in the sequence has the correct
55 + hash.
56 + """
57 +
58 + random.seed(seed)
59 +
60 + previous_block = MagicMock(spec=PrefixCachingBlock)
61 + prev_block_hash = random.randint(0, 1000)
62 + previous_block.content_hash = (prev_block_hash
63 + if prev_block_has_hash else None)
64 +
65 + num_to_fill = block_size if is_curr_block_full else random.randint(
66 + 0, block_size - 1)
67 + token_ids = list(range(num_to_fill))
68 + mock_allocator = MagicMock(spec=PrefixCachingBlockAllocator)
69 +
70 + block_with_prev = PrefixCachingBlock(
71 + prev_block=previous_block,
72 + token_ids=token_ids,
73 + block_size=block_size,
74 + prefix_caching_allocator=mock_allocator,
75 + )
76 +
77 + if is_curr_block_full and prev_block_has_hash:
78 + # Expect hash since block is full and previous block has hash.
79 + assert (block_with_prev.content_hash ==
80 + PrefixCachingBlock.hash_block_tokens(
81 + is_first_block=False,
82 + prev_block_hash=prev_block_hash,
83 + cur_block_token_ids=token_ids))
84 + else:
85 + # Do not expect hash since block is not full or the previous block
86 + # does not have a hash.
87 + assert block_with_prev.content_hash is None
88 +
89 + @staticmethod
90 + @pytest.mark.parametrize("block_size", [1, 2, 16])
91 + @pytest.mark.parametrize("num_tokens", list(range(3)))
92 + @pytest.mark.parametrize("num_empty_trailing_blocks", [0, 1, 10])
93 + def test_blocks_have_correct_hash_in_chain(block_size: int,
94 + num_tokens: int,
95 + num_empty_trailing_blocks: int):
96 + """Create two chains of logical blocks with the same contents.
97 + Assert the hashes are equal.
98 + """
99 + random.seed(0)
100 +
101 + token_ids = [random.randint(0, 50_000) for _ in range(num_tokens)]
102 +
103 + first_chain, second_chain = [
104 + TestPrefixCachingBlock.create_chain(
105 + block_size=block_size,
106 + token_ids=token_ids,
107 + num_empty_trailing_blocks=num_empty_trailing_blocks)
108 + for _ in range(2)
109 + ]
110 +
111 + for first_chain_block, second_chain_block in zip(
112 + first_chain, second_chain):
113 + assert (first_chain_block.content_hash ==
114 + second_chain_block.content_hash)
115 +
116 + if not first_chain or not second_chain:
117 + assert first_chain == second_chain
118 + assert num_tokens == 0
119 +
120 + @staticmethod
121 + def create_chain(block_size: int,
122 + token_ids: List[int],
123 + num_empty_trailing_blocks=0) -> List[PrefixCachingBlock]:
124 + """Helper method which creates a chain of blocks.
125 + """
126 + blocks = []
127 + num_blocks = math.ceil(
128 + len(token_ids) / block_size) + num_empty_trailing_blocks
129 +
130 + if num_blocks == 0:
131 + return []
132 +
133 + allocator = MagicMock(spec=PrefixCachingBlockAllocator)
134 +
135 + prev_block = None
136 + for block_number in range(0, num_blocks):
137 + prev_block = PrefixCachingBlock(
138 + prev_block=prev_block,
139 + token_ids=[],
140 + block_size=block_size,
141 + prefix_caching_allocator=allocator,
142 + )
143 +
144 + tokens_to_append = token_ids[block_number *
145 + block_size:(block_number + 1) *
146 + block_size]
147 + if tokens_to_append:
148 + prev_block.append_token_ids(tokens_to_append)
149 +
150 + blocks.append(prev_block)
151 +
152 + return blocks
153 +
154 +
155 +class TestPrefixCachingBlockAllocator:
156 +
157 + @staticmethod
158 + def create_allocate_lambda(allocate_type: str, allocator: BlockAllocator,
159 + prev_block: Optional[Block],
160 + token_ids: List[int]):
161 + if allocate_type == "immutable":
162 + allocate_block = lambda: allocator.allocate_immutable(
163 + prev_block=prev_block, token_ids=token_ids)
164 + elif allocate_type == "mutable":
165 + allocate_block = lambda: allocator.allocate_mutable(prev_block=
166 + prev_block)
167 + else:
168 + raise ValueError()
169 +
170 + return allocate_block
171 +
172 + @staticmethod
173 + @pytest.mark.parametrize("num_blocks", [1, 1024])
174 + @pytest.mark.parametrize("block_size", [1, 16])
175 + def test_allocate_mutable_ooms(num_blocks: int, block_size: int):
176 + allocator = PrefixCachingBlockAllocator(num_blocks=num_blocks,
177 + block_size=block_size)
178 + allocate_block = TestPrefixCachingBlockAllocator.create_allocate_lambda(
179 + allocate_type="mutable",
180 + allocator=allocator,
181 + prev_block=None,
182 + token_ids=list(range(block_size)),
183 + )
184 +
185 + [allocate_block() for _ in range(num_blocks)]
186 + with pytest.raises(BlockAllocator.NoFreeBlocksError):
187 + allocate_block()
188 +
189 + @staticmethod
190 + @pytest.mark.parametrize("num_blocks", [1, 1024])
191 + @pytest.mark.parametrize("block_size", [1, 16])
192 + def test_allocate_immutable_does_not_oom_single_hash(
193 + num_blocks: int, block_size: int):
194 + allocator = PrefixCachingBlockAllocator(num_blocks=num_blocks,
195 + block_size=block_size)
196 + allocate_block = TestPrefixCachingBlockAllocator.create_allocate_lambda(
197 + allocate_type="immutable",
198 + allocator=allocator,
199 + prev_block=None,
200 + token_ids=list(range(block_size)),
201 + )
202 +
203 + blocks = [allocate_block() for _ in range(num_blocks)]
204 +
205 + # Expect no OOM. If these were mutable blocks, this would OOM.
206 + non_oom_block = allocate_block()
207 +
208 + # Expect all blocks to have same physical block index.
209 + for block in blocks:
210 + assert (block.block_id == non_oom_block.block_id)
211 +
212 + @staticmethod
213 + @pytest.mark.parametrize("num_blocks", [1, 1024])
214 + @pytest.mark.parametrize("block_size", [1, 16])
215 + def test_allocate_immutable_ooms_many_hash(num_blocks: int,
216 + block_size: int):
217 + """Consume all blocks using many different hashes/block content.
218 +
219 + Do this by creating a sequence that is very long.
220 + Expect next block to OOM.
221 + """
222 + allocator = PrefixCachingBlockAllocator(num_blocks=num_blocks,
223 + block_size=block_size)
224 +
225 + # Create token ids that will exhaust all blocks.
226 + token_ids = list(range(num_blocks * block_size))
227 +
228 + chain = TestPrefixCachingBlockAllocator.create_immutable_chain(
229 + block_size=block_size,
230 + token_ids=token_ids,
231 + allocator=allocator,
232 + )
233 +
234 + # Expect allocation with unseen hash to fail.
235 + with pytest.raises(BlockAllocator.NoFreeBlocksError):
236 + allocator.allocate_immutable(prev_block=chain[-1],
237 + token_ids=list(range(block_size)))
238 +
239 + # Expect mutable allocation to fail.
240 + with pytest.raises(BlockAllocator.NoFreeBlocksError):
241 + allocator.allocate_mutable(prev_block=chain[-1])
242 +
243 + # Expect allocation of exact same chain to pass.
244 + second_chain = TestPrefixCachingBlockAllocator.create_immutable_chain(
245 + block_size=block_size,
246 + token_ids=token_ids,
247 + allocator=allocator,
248 + )
249 +
250 + # Expect physical block indices to be the same in both chains.
251 + assert chain and second_chain
252 + for first_chain_block, second_chain_block in zip(chain, second_chain):
253 + assert (first_chain_block.block_id == second_chain_block.block_id)
254 +
255 + @staticmethod
256 + @pytest.mark.parametrize("num_blocks", [1, 1024])
257 + @pytest.mark.parametrize("block_size", [1, 16])
258 + def test_free_prevents_oom(num_blocks: int, block_size: int):
259 + allocator = PrefixCachingBlockAllocator(num_blocks=num_blocks,
260 + block_size=block_size)
261 +
262 + # Create token ids that will exhaust all blocks.
263 + token_ids = list(range(num_blocks * block_size))
264 +
265 + chain = TestPrefixCachingBlockAllocator.create_immutable_chain(
266 + block_size=block_size,
267 + token_ids=token_ids,
268 + allocator=allocator,
269 + )
270 +
271 + # Expect mutable allocation to fail.
272 + with pytest.raises(BlockAllocator.NoFreeBlocksError):
273 + allocator.allocate_mutable(prev_block=None)
274 +
275 + block_to_free = chain[-1]
276 +
277 + # Expect free/allocate loop to succeed many times.
278 + for i in range(100):
279 + block_id = block_to_free.block_id
280 + allocator.free(block_to_free)
281 + assert block_to_free.block_id is None, i
282 +
283 + new_block = allocator.allocate_mutable(prev_block=None)
284 + assert new_block.block_id == block_id, i
285 +
286 + with pytest.raises(BlockAllocator.NoFreeBlocksError):
287 + allocator.allocate_mutable(prev_block=None)
288 +
289 + block_to_free = new_block
290 +
291 + @staticmethod
292 + @pytest.mark.parametrize("num_blocks", [1024])
293 + @pytest.mark.parametrize("block_size", [16])
294 + @pytest.mark.parametrize("seed", list(range(20)))
295 + def test_get_num_free_blocks(num_blocks: int, block_size: int, seed: int):
296 + random.seed(seed)
297 + allocator = PrefixCachingBlockAllocator(num_blocks=num_blocks,
298 + block_size=block_size)
299 + num_blocks_to_consume = random.randint(1, num_blocks - 1)
300 +
301 + # Create token ids that will exhaust all blocks.
302 + token_ids = list(range(num_blocks_to_consume * block_size))
303 +
304 + chain = TestPrefixCachingBlockAllocator.create_immutable_chain(
305 + block_size=block_size,
306 + token_ids=token_ids,
307 + allocator=allocator,
308 + )
309 +
310 + # Free each block in chain, assert num free blocks includes new free
311 + # block.
312 + for i, block in enumerate(chain):
313 + assert allocator.get_num_free_blocks() == (num_blocks -
314 + num_blocks_to_consume +
315 + i)
316 + allocator.free(block)
317 +
318 + @staticmethod
319 + @pytest.mark.parametrize("num_blocks", [1024])
320 + @pytest.mark.parametrize("block_size", [16])
321 + @pytest.mark.parametrize("seed", list(range(20)))
322 + def test_get_num_free_blocks_shared(num_blocks: int, block_size: int,
323 + seed: int):
324 + """Verify sharing occurs by allocating two sequences that share prefixes
325 + and incrementally freeing blocks.
326 + """
327 + random.seed(seed)
328 + allocator = PrefixCachingBlockAllocator(num_blocks=num_blocks,
329 + block_size=block_size)
330 + num_blocks_to_consume = random.randint(1, num_blocks - 1)
331 +
332 + # Create token ids that will exhaust all blocks.
333 + token_ids = list(range(num_blocks_to_consume * block_size))
334 +
335 + first_chain = TestPrefixCachingBlockAllocator.create_immutable_chain(
336 + block_size=block_size,
337 + token_ids=token_ids,
338 + allocator=allocator,
339 + )
340 + second_chain = TestPrefixCachingBlockAllocator.create_immutable_chain(
341 + block_size=block_size,
342 + token_ids=token_ids,
343 + allocator=allocator,
344 + )
345 +
346 + # Free each block in the first chain. Since all blocks are shared, the
347 + # free count should stay constant.
348 + for i, block in enumerate(first_chain):
349 + assert allocator.get_num_free_blocks() == (num_blocks -
350 + num_blocks_to_consume)
351 + allocator.free(block)
352 +
353 + # Free each block in the second chain. Since the refcount is now zero,
354 + # the free count should increment with each free.
355 + for i, block in enumerate(second_chain):
356 + assert allocator.get_num_free_blocks() == (num_blocks -
357 + num_blocks_to_consume +
358 + i)
359 + allocator.free(block)
360 +
361 + @staticmethod
362 + def create_immutable_chain(
363 + block_size: int,
364 + token_ids: List[int],
365 + allocator: PrefixCachingBlockAllocator,
366 + ) -> List[PrefixCachingBlock]:
367 + """Helper method which creates a chain of blocks.
368 + """
369 + blocks = []
370 + num_blocks = math.ceil(len(token_ids) / block_size)
371 +
372 + if num_blocks == 0:
373 + return []
374 +
375 + prev_block = None
376 + for block_number in range(0, num_blocks):
377 + block_token_ids = token_ids[block_number *
378 + block_size:(block_number + 1) *
379 + block_size]
380 + prev_block = allocator.allocate_immutable(
381 + prev_block=prev_block, token_ids=block_token_ids)
382 + blocks.append(prev_block)
383 +
384 + return blocks
tests/core/test_block_manager.py
+40 -39
@@ -5,8 +5,9 @@ import pytest
5
6 from vllm import SamplingParams
7 from vllm.block import PhysicalTokenBlock
8 -from vllm.core.block_manager import (AllocStatus, BlockSpaceManager,
9 - UncachedBlockAllocator)
8 +from vllm.core.block_manager_v1 import (BlockSpaceManagerV1,
9 + UncachedBlockAllocator)
10 +from vllm.core.interfaces import AllocStatus
11 from vllm.sequence import Logprob, Sequence, SequenceGroup, SequenceStatus
12 from vllm.utils import Device
13
@@ -63,10 +64,10 @@ def test_allocate():
64 block_size = 4
65 num_cpu_blocks = 4
66 num_gpu_blocks = 4
66 - block_manager = BlockSpaceManager(block_size,
67 - num_cpu_blocks,
68 - num_gpu_blocks,
69 - watermark=0)
67 + block_manager = BlockSpaceManagerV1(block_size,
68 + num_cpu_blocks,
69 + num_gpu_blocks,
70 + watermark=0)
71
72 # Allocate same sequence group to all available gpu blocks.
73 for i in range(num_gpu_blocks):
@@ -77,10 +78,10 @@ def test_allocate():
78
79 # Allocate same sequence group to all available gpu blocks.
80 # Use watermark to reserve one gpu block.
80 - block_manager = BlockSpaceManager(block_size,
81 - num_cpu_blocks,
82 - num_gpu_blocks,
83 - watermark=1 / num_gpu_blocks)
81 + block_manager = BlockSpaceManagerV1(block_size,
82 + num_cpu_blocks,
83 + num_gpu_blocks,
84 + watermark=1 / num_gpu_blocks)
85 for i in range(num_gpu_blocks - 1):
86 _, seq_group = create_dummy_prompt(str(i), block_size)
87 assert block_manager.can_allocate(seq_group)
@@ -92,10 +93,10 @@ def test_append_slot_single_seq():
93 block_size = 4
94 num_cpu_blocks = 4
95 num_gpu_blocks = 4
95 - block_manager = BlockSpaceManager(block_size,
96 - num_cpu_blocks,
97 - num_gpu_blocks,
98 - watermark=0)
96 + block_manager = BlockSpaceManagerV1(block_size,
97 + num_cpu_blocks,
98 + num_gpu_blocks,
99 + watermark=0)
100
101 # Allocate single seq to gpu block.
102 prompt, seq_group = create_dummy_prompt("1", block_size)
@@ -124,10 +125,10 @@ def test_append_slot_cow():
125 block_size = 4
126 num_cpu_blocks = 4
127 num_gpu_blocks = 4
127 - block_manager = BlockSpaceManager(block_size=block_size,
128 - num_cpu_blocks=num_cpu_blocks,
129 - num_gpu_blocks=num_gpu_blocks,
130 - watermark=0)
128 + block_manager = BlockSpaceManagerV1(block_size=block_size,
129 + num_cpu_blocks=num_cpu_blocks,
130 + num_gpu_blocks=num_gpu_blocks,
131 + watermark=0)
132
133 # Allocate prompt to gpu block. There is one slot left in the block.
134 prompt = Sequence(seq_id=1,
@@ -165,10 +166,10 @@ def test_fork():
166 block_size = 4
167 num_cpu_blocks = 4
168 num_gpu_blocks = 4
168 - block_manager = BlockSpaceManager(block_size,
169 - num_cpu_blocks,
170 - num_gpu_blocks,
171 - watermark=0)
169 + block_manager = BlockSpaceManagerV1(block_size,
170 + num_cpu_blocks,
171 + num_gpu_blocks,
172 + watermark=0)
173
174 prompt, seq_group = create_dummy_prompt("1",
175 block_size - 1,
@@ -192,10 +193,10 @@ def test_swap():
193 block_size = 4
194 num_cpu_blocks = 4
195 num_gpu_blocks = 4
195 - block_manager = BlockSpaceManager(block_size,
196 - num_cpu_blocks,
197 - num_gpu_blocks,
198 - watermark=0)
196 + block_manager = BlockSpaceManagerV1(block_size,
197 + num_cpu_blocks,
198 + num_gpu_blocks,
199 + watermark=0)
200
201 prompt, seq_group = create_dummy_prompt("1", prompt_length=block_size - 1)
202 prompt.status = SequenceStatus.WAITING
@@ -238,10 +239,10 @@ def test_free():
239 block_size = 4
240 num_cpu_blocks = 4
241 num_gpu_blocks = 4
241 - block_manager = BlockSpaceManager(block_size,
242 - num_cpu_blocks,
243 - num_gpu_blocks,
244 - watermark=0)
242 + block_manager = BlockSpaceManagerV1(block_size,
243 + num_cpu_blocks,
244 + num_gpu_blocks,
245 + watermark=0)
246
247 prompt, seq_group = create_dummy_prompt("1", block_size)
248 block_manager.allocate(seq_group)
@@ -262,10 +263,10 @@ def test_reset():
263 block_size = 4
264 num_cpu_blocks = 4
265 num_gpu_blocks = 4
265 - block_manager = BlockSpaceManager(block_size,
266 - num_cpu_blocks,
267 - num_gpu_blocks,
268 - watermark=0)
266 + block_manager = BlockSpaceManagerV1(block_size,
267 + num_cpu_blocks,
268 + num_gpu_blocks,
269 + watermark=0)
270
271 # Allocate same seq group on all available gpu blocks.
272 original_blocks = block_manager.get_num_free_gpu_blocks()
@@ -289,11 +290,11 @@ def test_sliding_window_multi_seq():
290 num_cpu_blocks = 8
291 num_gpu_blocks = 8
292 sliding_window = 2
292 - block_manager = BlockSpaceManager(block_size,
293 - num_cpu_blocks,
294 - num_gpu_blocks,
295 - sliding_window=sliding_window,
296 - watermark=0)
293 + block_manager = BlockSpaceManagerV1(block_size,
294 + num_cpu_blocks,
295 + num_gpu_blocks,
296 + sliding_window=sliding_window,
297 + watermark=0)
298
299 assert block_manager.get_num_free_gpu_blocks() == num_gpu_blocks
300
tests/core/test_scheduler.py
+1 -1
@@ -13,7 +13,7 @@ from .utils import create_dummy_prompt
13 def test_scheduler_add_seq_group():
14 block_size = 4
15 scheduler_config = SchedulerConfig(100, 64, 1)
16 - cache_config = CacheConfig(block_size, 1.0, 1, "auto")
16 + cache_config = CacheConfig(block_size, 1.0, 1, cache_dtype="auto")
17 cache_config.num_cpu_blocks = 4
18 cache_config.num_gpu_blocks = 4
19 scheduler = Scheduler(scheduler_config, cache_config, None)
tests/core/utils.py
+38 -1
@@ -2,7 +2,7 @@ import time
2 from typing import Tuple
3
4 from vllm import SamplingParams
5 -from vllm.sequence import Sequence, SequenceGroup
5 +from vllm.sequence import Logprob, Sequence, SequenceGroup
6
7
8 def create_dummy_prompt(
@@ -23,5 +23,42 @@ def create_dummy_prompt(
23 return prompt, seq_group
24
25
26 +def create_seq_group(
27 + seq_prompt_lens=1024,
28 + seq_output_lens=(128, ),
29 + request_id='0',
30 + seq_id_start=0,
31 +) -> SequenceGroup:
32 +
33 + assert len(seq_output_lens) > 0
34 +
35 + prompt_token_ids = [0] * seq_prompt_lens
36 +
37 + seqs = []
38 + for seq_id_offset, output_len in enumerate(seq_output_lens):
39 + seq = Sequence(
40 + seq_id=seq_id_start + seq_id_offset,
41 + prompt="",
42 + prompt_token_ids=prompt_token_ids,
43 + block_size=16,
44 + )
45 +
46 + for i in range(output_len):
47 + seq.append_token_id(
48 + token_id=i,
49 + logprobs={i: Logprob(0.0)},
50 + )
51 + seqs.append(seq)
52 +
53 + seq_group = SequenceGroup(
54 + request_id=request_id,
55 + seqs=seqs,
56 + sampling_params=SamplingParams(),
57 + arrival_time=time.time(),
58 + )
59 +
60 + return seq_group
61 +
62 +
63 def round_up_to_next_block(seq_len: int, block_size: int) -> int:
64 return (seq_len + block_size - 1) // block_size
tests/prefix_caching/test_prefix_caching.py
+1 -1
@@ -4,7 +4,7 @@ Run `pytest tests/prefix_caching/test_prefix_caching.py`.
4 """
5 import pytest
6
7 -from vllm.core.block_manager import CachedBlockAllocator
7 +from vllm.core.block_manager_v1 import CachedBlockAllocator
8 from vllm.utils import Device
9
10
vllm/config.py
+7
@@ -324,6 +324,8 @@ class CacheConfig:
324 vLLM execution.
325 swap_space: Size of the CPU swap space per GPU (in GiB).
326 cache_dtype: Data type for kv cache storage.
327 + forced_num_gpu_blocks: Number of GPU blocks to use. This overrides the
328 + profiled num_gpu_blocks if specified. Does nothing if None.
329 """
330
331 def __init__(
@@ -332,12 +334,14 @@ class CacheConfig:
334 gpu_memory_utilization: float,
335 swap_space: int,
336 cache_dtype: str,
337 + forced_num_gpu_blocks: Optional[int] = None,
338 sliding_window: Optional[int] = None,
339 enable_prefix_caching: bool = False,
340 ) -> None:
341 self.block_size = block_size
342 self.gpu_memory_utilization = gpu_memory_utilization
343 self.swap_space_bytes = swap_space * _GB
344 + self.forced_num_gpu_blocks = forced_num_gpu_blocks
345 self.cache_dtype = cache_dtype
346 self.sliding_window = sliding_window
347 self.enable_prefix_caching = enable_prefix_caching
@@ -528,6 +532,7 @@ class SchedulerConfig:
532 and generated text).
533 delay_factor: Apply a delay (of delay factor multiplied by previous
534 prompt latency) before scheduling next prompt.
535 + use_v2_block_manager: Whether to use the BlockSpaceManagerV2 or not.
536 """
537
538 def __init__(
@@ -535,6 +540,7 @@ class SchedulerConfig:
540 max_num_batched_tokens: Optional[int],
541 max_num_seqs: int,
542 max_model_len: int,
543 + use_v2_block_manager: bool = False,
544 delay_factor: float = 0.0,
545 ) -> None:
546 if max_num_batched_tokens is not None:
@@ -546,6 +552,7 @@ class SchedulerConfig:
552 self.max_num_seqs = max_num_seqs
553 self.max_model_len = max_model_len
554 self.delay_factor = delay_factor
555 + self.use_v2_block_manager = use_v2_block_manager
556 self._verify_args()
557
558 def _verify_args(self) -> None:
vllm/core/block/block_table.py new
+245
@@ -0,0 +1,245 @@
1 +from typing import List, Optional
2 +
3 +from vllm.core.block.interfaces import Block, DeviceAwareBlockAllocator
4 +from vllm.utils import Device, cdiv, chunk_list
5 +
6 +
7 +class BlockTable:
8 + """A class to manage blocks for a specific sequence.
9 +
10 + The BlockTable maps a sequence of tokens to a list of blocks, where each
11 + block represents a contiguous memory allocation for a portion of the
12 + sequence. The blocks are managed by a DeviceAwareBlockAllocator, which is
13 + responsible for allocating and freeing memory for the blocks.
14 +
15 + Args:
16 + block_size (int): The maximum number of tokens that can be stored in a
17 + single block.
18 + block_allocator (DeviceAwareBlockAllocator): The block allocator used to
19 + manage memory for the blocks.
20 + _blocks (Optional[List[Block]], optional): An optional list of existing
21 + blocks to initialize the BlockTable with. If not provided, an empty
22 + BlockTable is created.
23 +
24 + Attributes:
25 + _block_size (int): The maximum number of tokens that can be stored in a
26 + single block.
27 + _allocator (DeviceAwareBlockAllocator): The block allocator used to
28 + manage memory for the blocks.
29 + _blocks (Optional[List[Block]]): The list of blocks managed by this
30 + BlockTable.
31 + _num_full_slots (int): The number of tokens currently stored in the
32 + blocks.
33 + """
34 +
35 + def __init__(
36 + self,
37 + block_size: int,
38 + block_allocator: DeviceAwareBlockAllocator,
39 + _blocks: Optional[List[Block]] = None,
40 + ):
41 + self._block_size = block_size
42 + self._allocator = block_allocator
43 + self._blocks: Optional[List[Block]] = _blocks
44 +
45 + # Use helper method instead of directly calculating, as blocks
46 + # may not be allocated.
47 + self._num_full_slots = len(self._get_all_token_ids())
48 +
49 + @staticmethod
50 + def get_num_required_blocks(token_ids: List[int], block_size: int) -> int:
51 + """Calculates the minimum number of blocks required to store a given
52 + sequence of token IDs.
53 +
54 + This assumes worst-case scenario, where every block requires a new
55 + allocation (e.g. ignoring prefix caching).
56 +
57 + Args:
58 + token_ids (List[int]): The sequence of token IDs to be stored.
59 + block_size (int): The maximum number of tokens that can be stored in
60 + a single block.
61 +
62 + Returns:
63 + int: The minimum number of blocks required to store the given
64 + sequence of token IDs.
65 + """
66 + return cdiv(len(token_ids), block_size)
67 +
68 + def allocate(self,
69 + token_ids: List[int],
70 + device: Device = Device.GPU) -> None:
71 + """Allocates memory blocks for storing the given sequence of token IDs.
72 +
73 + This method allocates the required number of blocks to store the given
74 + sequence of token IDs.
75 +
76 + Args:
77 + token_ids (List[int]): The sequence of token IDs to be stored.
78 + device (Device, optional): The device on which the blocks should be
79 + allocated. Defaults to Device.GPU.
80 + """
81 + assert not self._is_allocated
82 + assert token_ids
83 + self._blocks = self._allocate_blocks_for_token_ids(prev_block=None,
84 + token_ids=token_ids,
85 + device=device)
86 + self._num_full_slots = len(token_ids)
87 +
88 + def append_token_ids(self, token_ids: List[int]) -> None:
89 + """Appends a sequence of token IDs to the existing blocks in the
90 + BlockTable.
91 +
92 + This method appends the given sequence of token IDs to the existing
93 + blocks in the BlockTable. If there is not enough space in the existing
94 + blocks, new blocks are allocated using the `ensure_num_empty_slots`
95 + method to accommodate the additional tokens.
96 +
97 + The token IDs are divided into chunks of size `block_size` (except for
98 + the first chunk, which may be smaller), and each chunk is appended to a
99 + separate block.
100 +
101 + Args:
102 + token_ids (List[int]): The sequence of token IDs to be appended.
103 + """
104 + assert self._is_allocated
105 +
106 + self.ensure_num_empty_slots(num_empty_slots=len(token_ids))
107 +
108 + blocks = self._blocks[self._num_full_slots // self._block_size:]
109 + first_chunk_size = self._block_size - (self._num_full_slots %
110 + self._block_size)
111 + token_blocks = [token_ids[:first_chunk_size]] + chunk_list(
112 + token_ids[first_chunk_size:], self._block_size)
113 +
114 + for block, token_block in zip(blocks, token_blocks):
115 + block.append_token_ids(token_block)
116 +
117 + self._num_full_slots += len(token_ids)
118 +
119 + def ensure_num_empty_slots(self, num_empty_slots: int) -> None:
120 + """Ensures that the BlockTable has at least the specified number of
121 + empty slots available.
122 +
123 + This method checks if the BlockTable has enough empty slots (i.e.,
124 + available space) to accommodate the requested number of tokens. If not,
125 + it allocates additional blocks on the GPU to ensure that the required
126 + number of empty slots is available.
127 +
128 + Args:
129 + num_empty_slots (int): The minimum number of empty slots required.
130 + """
131 + # Currently the block table only supports
132 + # appending tokens to GPU blocks.
133 + device = Device.GPU
134 + assert self._is_allocated
135 +
136 + if self._num_empty_slots >= num_empty_slots:
137 + return
138 +
139 + slots_to_allocate = num_empty_slots - self._num_empty_slots
140 + blocks_to_allocate = cdiv(slots_to_allocate, self._block_size)
141 +
142 + for _ in range(blocks_to_allocate):
143 + self._blocks.append(
144 + self._allocator.allocate_mutable(prev_block=self._blocks[-1],
145 + device=device))
146 +
147 + def fork(self) -> "BlockTable":
148 + """Creates a new BlockTable instance with a copy of the blocks from the
149 + current instance.
150 +
151 + This method creates a new BlockTable instance with the same block size,
152 + block allocator, and a copy of the blocks from the current instance. The
153 + new BlockTable has its own independent set of blocks, but shares the
154 + same underlying memory allocation with the original BlockTable.
155 +
156 + Returns:
157 + BlockTable: A new BlockTable instance with a copy of the blocks from
158 + the current instance.
159 + """
160 + assert self._is_allocated
161 + forked_blocks = self._allocator.fork(self._blocks[-1])
162 + return BlockTable(
163 + block_size=self._block_size,
164 + block_allocator=self._allocator,
165 + _blocks=forked_blocks,
166 + )
167 +
168 + def free(self) -> None:
169 + """Frees the memory occupied by the blocks in the BlockTable.
170 +
171 + This method iterates over all the blocks in the `_blocks` list and calls
172 + the `free` method of the `_allocator` object to release the memory
173 + occupied by each block. After freeing all the blocks, the `_blocks` list
174 + is set to `None`.
175 + """
176 + assert self._is_allocated
177 + for block in self._blocks:
178 + self._allocator.free(block)
179 + self._blocks = None
180 +
181 + @property
182 + def physical_block_ids(self) -> List[int]:
183 + """Returns a list of physical block indices for the blocks in the
184 + BlockTable.
185 +
186 + This property returns a list of integers, where each integer represents
187 + the physical block index of a corresponding block in the `_blocks` list.
188 + The physical block index is a unique identifier for the memory location
189 + occupied by the block.
190 +
191 + Returns:
192 + List[int]: A list of physical block indices for the blocks in the
193 + BlockTable.
194 + """
195 + assert self._is_allocated
196 + return [block.block_id for block in self._blocks]
197 +
198 + def _allocate_blocks_for_token_ids(self, prev_block: Optional[Block],
199 + token_ids: List[int],
200 + device: Device) -> List[Block]:
201 + blocks = []
202 + for block_token_ids in chunk_list(token_ids, self._block_size):
203 + if len(block_token_ids) == self._block_size:
204 + # If the block is full, create an immutable block.
205 + prev_block = self._allocator.allocate_immutable(
206 + prev_block, token_ids=block_token_ids, device=device)
207 + else:
208 + # Else, partially fill a mutable block with token ids.
209 + prev_block = self._allocator.allocate_mutable(
210 + prev_block=prev_block, device=device)
211 + prev_block.append_token_ids(block_token_ids)
212 + blocks.append(prev_block)
213 +
214 + return blocks
215 +
216 + def _get_all_token_ids(self) -> List[int]:
217 + # NOTE: This function is O(seq_len); use sparingly.
218 + token_ids = []
219 +
220 + if not self._is_allocated:
221 + return token_ids
222 +
223 + for block in self._blocks:
224 + token_ids.extend(block.token_ids)
225 +
226 + return token_ids
227 +
228 + @property
229 + def _is_allocated(self) -> bool:
230 + return self._blocks is not None
231 +
232 + @property
233 + def _num_empty_slots(self) -> int:
234 + assert self._is_allocated
235 + return len(self._blocks) * self._block_size - self._num_full_slots
236 +
237 + @property
238 + def num_full_slots(self) -> int:
239 + """Returns the total number of tokens currently stored in the
240 + BlockTable.
241 +
242 + Returns:
243 + int: The total number of tokens currently stored in the BlockTable.
244 + """
245 + return self._num_full_slots
vllm/core/block/common.py new
+185
@@ -0,0 +1,185 @@
1 +from collections import defaultdict
2 +from typing import Dict, Iterable, List, Optional
3 +
4 +from vllm.core.block.interfaces import Block, BlockAllocator
5 +
6 +BlockId = int
7 +RefCount = int
8 +
9 +
10 +class RefCounter:
11 + """A class for managing reference counts for a set of block indices.
12 +
13 + The RefCounter class maintains a dictionary that maps block indices to their
14 + corresponding reference counts. It provides methods to increment, decrement,
15 + and retrieve the reference count for a given block index.
16 +
17 + Args:
18 + all_block_indices (Iterable[BlockId]): An iterable of block indices
19 + to initialize the reference counter with.
20 + """
21 +
22 + def __init__(self, all_block_indices: Iterable[BlockId]):
23 + deduped = set(all_block_indices)
24 + self._refcounts: Dict[BlockId,
25 + RefCount] = {index: 0
26 + for index in deduped}
27 +
28 + def incr(self, block_id: BlockId) -> RefCount:
29 + assert block_id in self._refcounts
30 + pre_incr_refcount = self._refcounts[block_id]
31 +
32 + assert pre_incr_refcount >= 0
33 +
34 + post_incr_refcount = pre_incr_refcount + 1
35 + self._refcounts[block_id] = post_incr_refcount
36 + return post_incr_refcount
37 +
38 + def decr(self, block_id: BlockId) -> RefCount:
39 + assert block_id in self._refcounts
40 + refcount = self._refcounts[block_id]
41 +
42 + assert refcount > 0
43 + refcount -= 1
44 +
45 + self._refcounts[block_id] = refcount
46 +
47 + return refcount
48 +
49 + def get(self, block_id: BlockId) -> RefCount:
50 + assert block_id in self._refcounts
51 + return self._refcounts[block_id]
52 +
53 + def as_readonly(self) -> "ReadOnlyRefCounter":
54 + return ReadOnlyRefCounter(self)
55 +
56 +
57 +class ReadOnlyRefCounter:
58 + """A read-only view of the RefCounter class.
59 +
60 + The ReadOnlyRefCounter class provides a read-only interface to access the
61 + reference counts maintained by a RefCounter instance. It does not allow
62 + modifications to the reference counts.
63 +
64 + Args:
65 + refcounter (RefCounter): The RefCounter instance to create a read-only
66 + view for.
67 + """
68 +
69 + def __init__(self, refcounter: RefCounter):
70 + self._refcounter = refcounter
71 +
72 + def incr(self, block_id: BlockId) -> RefCount:
73 + raise ValueError("Incr not allowed")
74 +
75 + def decr(self, block_id: BlockId) -> RefCount:
76 + raise ValueError("Decr not allowed")
77 +
78 + def get(self, block_id: BlockId) -> RefCount:
79 + return self._refcounter.get(block_id)
80 +
81 +
82 +class CopyOnWriteTracker:
83 + """A class for tracking and managing copy-on-write operations for blocks.
84 +
85 + The CopyOnWriteTracker class maintains a mapping of source block indices to
86 + their corresponding copy-on-write destination block indices. It works in
87 + conjunction with a RefCounter and a BlockAllocator to handle reference
88 + counting and block allocation.
89 +
90 + Args:
91 + refcounter (RefCounter): The reference counter used to track block
92 + reference counts.
93 + allocator (BlockAllocator): The block allocator used to allocate and
94 + free blocks.
95 + """
96 +
97 + def __init__(
98 + self,
99 + refcounter: RefCounter,
100 + allocator: BlockAllocator,
101 + ):
102 + self._copy_on_writes = defaultdict(list)
103 + self._refcounter = refcounter
104 + self._allocator = allocator
105 +
106 + def cow_block_if_not_appendable(self, block: Block) -> Optional[BlockId]:
107 + """Performs a copy-on-write operation on the given block if it is not
108 + appendable.
109 +
110 + This method checks the reference count of the given block. If the
111 + reference count is greater than 1, indicating that the block is shared,
112 + a copy-on-write operation is performed. The original block is freed,
113 + and a new block is allocated with the same content. The new block index
114 + is returned.
115 +
116 + Args:
117 + block (Block): The block to check for copy-on-write.
118 +
119 + Returns:
120 + Optional[BlockId]: The block index of the new block if a copy-on
121 + -write operation was performed, or the original block index if
122 + no copy-on-write was necessary.
123 + """
124 + block_id = block.block_id
125 + if block_id is None:
126 + return block_id
127 +
128 + refcount = self._refcounter.get(block_id)
129 + assert refcount != 0
130 + if refcount > 1:
131 + src_block_id = block_id
132 +
133 + # Decrement refcount of the old block.
134 + self._allocator.free(block)
135 +
136 + # Allocate a fresh new block.
137 + block_id = self._allocator.allocate_mutable(
138 + prev_block=block.prev_block).block_id
139 +
140 + # Track src/dst copy.
141 + self._copy_on_writes[src_block_id].append(block_id)
142 +
143 + return block_id
144 +
145 + def clear_cows(self) -> Dict[BlockId, List[BlockId]]:
146 + """Clears the copy-on-write tracking information and returns the current
147 + state.
148 +
149 + This method returns a dictionary mapping source block indices to lists
150 + of destination block indices for the current copy-on-write operations.
151 + It then clears the internal tracking information.
152 +
153 + Returns:
154 + Dict[BlockId, List[BlockId]]: A dictionary mapping source
155 + block indices to lists of destination block indices for the
156 + current copy-on-write operations.
157 + """
158 + cows = dict(self._copy_on_writes)
159 + self._copy_on_writes.clear()
160 + return cows
161 +
162 +
163 +def get_all_blocks_recursively(last_block: Block) -> List[Block]:
164 + """Retrieves all the blocks in a sequence starting from the last block.
165 +
166 + This function recursively traverses the sequence of blocks in reverse order,
167 + starting from the given last block, and returns a list of all the blocks in
168 + the sequence.
169 +
170 + Args:
171 + last_block (Block): The last block in the sequence.
172 +
173 + Returns:
174 + List[Block]: A list of all the blocks in the sequence, in the order they
175 + appear.
176 + """
177 +
178 + def recurse(block: Block, lst: List[Block]) -> None:
179 + if block.prev_block is not None:
180 + recurse(block.prev_block, lst)
181 + lst.append(block)
182 +
183 + all_blocks = []
184 + recurse(last_block, all_blocks)
185 + return all_blocks
vllm/core/block/cpu_gpu_block_allocator.py new
+206
@@ -0,0 +1,206 @@
1 +from typing import Dict, List, Optional
2 +
3 +from vllm.core.block.interfaces import (Block, BlockAllocator,
4 + DeviceAwareBlockAllocator)
5 +from vllm.core.block.naive_block import NaiveBlock, NaiveBlockAllocator
6 +from vllm.core.block.prefix_caching_block import PrefixCachingBlockAllocator
7 +from vllm.utils import Device
8 +
9 +
10 +class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
11 + """A block allocator that can allocate blocks on both CPU and GPU memory.
12 +
13 + This class implements the `DeviceAwareBlockAllocator` interface and provides
14 + functionality for allocating and managing blocks of memory on both CPU and
15 + GPU devices.
16 +
17 + The `CpuGpuBlockAllocator` maintains separate memory pools for CPU and GPU
18 + blocks, and allows for allocation, deallocation, forking, and swapping of
19 + blocks across these memory pools.
20 + """
21 +
22 + @staticmethod
23 + def create(
24 + allocator_type: str,
25 + num_gpu_blocks: int,
26 + num_cpu_blocks: int,
27 + block_size: int,
28 + ) -> DeviceAwareBlockAllocator:
29 + """Creates a CpuGpuBlockAllocator instance with the specified
30 + configuration.
31 +
32 + This static method creates and returns a CpuGpuBlockAllocator instance
33 + based on the provided parameters. It initializes the CPU and GPU block
34 + allocators with the specified number of blocks, block size, and
35 + allocator type.
36 +
37 + Args:
38 + allocator_type (str): The type of block allocator to use for CPU
39 + and GPU blocks. Currently supported values are "naive" and
40 + "prefix_caching".
41 + num_gpu_blocks (int): The number of blocks to allocate for GPU
42 + memory.
43 + num_cpu_blocks (int): The number of blocks to allocate for CPU
44 + memory.
45 + block_size (int): The size of each block in number of tokens.
46 +
47 + Returns:
48 + DeviceAwareBlockAllocator: A CpuGpuBlockAllocator instance with the
49 + specified configuration.
50 +
51 + Notes:
52 + - The block IDs are assigned contiguously, with GPU block IDs coming
53 + before CPU block IDs.
54 + """
55 + block_ids = list(range(num_gpu_blocks + num_cpu_blocks))
56 + gpu_block_ids = block_ids[:num_gpu_blocks]
57 + cpu_block_ids = block_ids[num_gpu_blocks:]
58 +
59 + if allocator_type == "naive":
60 + gpu_allocator = NaiveBlockAllocator(
61 + create_block=NaiveBlock,
62 + num_blocks=num_gpu_blocks,
63 + block_size=block_size,
64 + block_ids=gpu_block_ids,
65 + )
66 +
67 + cpu_allocator = NaiveBlockAllocator(
68 + create_block=NaiveBlock,
69 + num_blocks=num_cpu_blocks,
70 + block_size=block_size,
71 + block_ids=cpu_block_ids,
72 + )
73 + elif allocator_type == "prefix_caching":
74 + gpu_allocator = PrefixCachingBlockAllocator(
75 + num_blocks=num_gpu_blocks,
76 + block_size=block_size,
77 + block_ids=gpu_block_ids,
78 + )
79 +
80 + cpu_allocator = PrefixCachingBlockAllocator(
81 + num_blocks=num_cpu_blocks,
82 + block_size=block_size,
83 + block_ids=cpu_block_ids,
84 + )
85 + else:
86 + raise ValueError(f"Unknown allocator type {allocator_type=}")
87 +
88 + return CpuGpuBlockAllocator(
89 + cpu_block_allocator=cpu_allocator,
90 + gpu_block_allocator=gpu_allocator,
91 + )
92 +
93 + def __init__(
94 + self,
95 + cpu_block_allocator: BlockAllocator,
96 + gpu_block_allocator: BlockAllocator,
97 + ):
98 + assert not (
99 + cpu_block_allocator.all_block_ids
100 + & gpu_block_allocator.all_block_ids
101 + ), "cpu and gpu block allocators can't have intersection of block ids"
102 +
103 + self._allocators = {
104 + Device.CPU: cpu_block_allocator,
105 + Device.GPU: gpu_block_allocator,
106 + }
107 +
108 + self._block_ids_to_allocator = {}
109 + for _, allocator in self._allocators.items():
110 + for block_id in allocator.all_block_ids:
111 + self._block_ids_to_allocator[block_id] = allocator
112 +
113 + def allocate_mutable(self, prev_block: Optional[Block],
114 + device: Device) -> Block:
115 + """Allocates a new mutable block on the specified device.
116 +
117 + Args:
118 + prev_block (Optional[Block]): The previous block to in the sequence.
119 + Used for prefix hashing.
120 + device (Device): The device on which to allocate the new block.
121 +
122 + Returns:
123 + Block: The newly allocated mutable block.
124 + """
125 + return self._allocators[device].allocate_mutable(prev_block)
126 +
127 + def allocate_immutable(self, prev_block: Optional[Block],
128 + token_ids: List[int], device: Device) -> Block:
129 + """Allocates a new immutable block with the provided token IDs on the
130 + specified device.
131 +
132 + Args:
133 + prev_block (Optional[Block]): The previous block in the sequence.
134 + Used for prefix hashing.
135 + token_ids (List[int]): The list of token IDs to be stored in the new
136 + block.
137 + device (Device): The device on which to allocate the new block.
138 +
139 + Returns:
140 + Block: The newly allocated immutable block containing the provided
141 + token IDs.
142 + """
143 + return self._allocators[device].allocate_immutable(
144 + prev_block, token_ids)
145 +
146 + def free(self, block: Block) -> None:
147 + """Frees the memory occupied by the given block.
148 +
149 + Args:
150 + block (Block): The block to be freed.
151 + """
152 + allocator = self._block_ids_to_allocator[block.block_id]
153 + return allocator.free(block)
154 +
155 + def fork(self, last_block: Block) -> List[Block]:
156 + """Creates a new sequence of blocks that shares the same underlying
157 + memory as the original sequence.
158 +
159 + Args:
160 + last_block (Block): The last block in the original sequence.
161 +
162 + Returns:
163 + List[Block]: A new list of blocks that shares the same memory as the
164 + original sequence.
165 + """
166 + allocator = self._block_ids_to_allocator[last_block.block_id]
167 + return allocator.fork(last_block)
168 +
169 + def get_num_free_blocks(self, device: Device) -> int:
170 + """Returns the number of free blocks available on the specified device.
171 +
172 + Args:
173 + device (Device): The device for which to query the number of free
174 + blocks.
175 +
176 + Returns:
177 + int: The number of free blocks available on the specified device.
178 + """
179 + return self._allocators[device].get_num_free_blocks()
180 +
181 + def clear_copy_on_writes(self) -> Dict[int, List[int]]:
182 + """Clears the copy-on-write (CoW) state and returns the mapping of
183 + source to destination block IDs.
184 +
185 + Returns:
186 + Dict[int, List[int]]: A dictionary mapping source block IDs to lists
187 + of destination block IDs.
188 + """
189 + # CoW only supported on GPU
190 + device = Device.GPU
191 + return self._allocators[device].clear_copy_on_writes()
192 +
193 + def mark_blocks_as_computed(self) -> None:
194 + # Prefix caching only supported on GPU.
195 + device = Device.GPU
196 + return self._allocators[device].mark_blocks_as_computed()
197 +
198 + def get_common_computed_block_ids(
199 + self, seq_block_ids: List[List[int]]) -> List[int]:
200 + # Prefix caching only supported on GPU.
201 + device = Device.GPU
202 + return self._allocators[device].get_common_computed_block_ids(
203 + seq_block_ids)
204 +
205 + def all_block_ids(self) -> frozenset[int]:
206 + return frozenset(self._block_ids_to_allocator.keys())
vllm/core/block/interfaces.py new
+105
@@ -0,0 +1,105 @@
1 +from abc import ABC, abstractmethod, abstractproperty
2 +from typing import Dict, List, Optional, Protocol
3 +
4 +from vllm.utils import Device
5 +
6 +
7 +class Block(ABC):
8 +
9 + @abstractmethod
10 + def append_token_ids(self, token_ids: List[int]) -> None:
11 + pass
12 +
13 + @abstractproperty
14 + def block_id(self) -> Optional[int]:
15 + pass
16 +
17 + @abstractproperty
18 + def token_ids(self) -> List[int]:
19 + pass
20 +
21 + @abstractproperty
22 + def num_empty_slots(self) -> int:
23 + pass
24 +
25 + @abstractproperty
26 + def is_full(self) -> bool:
27 + pass
28 +
29 + @abstractproperty
30 + def prev_block(self) -> Optional["Block"]:
31 + pass
32 +
33 + class Factory(Protocol):
34 +
35 + @abstractmethod
36 + def __call__(
37 + self,
38 + prev_block: Optional["Block"],
39 + token_ids: List[int],
40 + block_size: int,
41 + allocator: "BlockAllocator",
42 + block_id: Optional[int] = None,
43 + ) -> "Block":
44 + pass
45 +
46 +
47 +class BlockAllocator(ABC):
48 +
49 + @abstractmethod
50 + def allocate_mutable(self, prev_block: Optional[Block]) -> Block:
51 + pass
52 +
53 + @abstractmethod
54 + def allocate_immutable(self, prev_block: Optional[Block],
55 + token_ids: List[int]) -> Block:
56 + pass
57 +
58 + @abstractmethod
59 + def free(self, block: Block) -> None:
60 + pass
61 +
62 + @abstractmethod
63 + def fork(self, last_block: Block) -> List[Block]:
64 + pass
65 +
66 + @abstractmethod
67 + def get_num_free_blocks(self) -> int:
68 + pass
69 +
70 + @abstractproperty
71 + def all_block_ids(self) -> frozenset[int]:
72 + pass
73 +
74 + @abstractmethod
75 + def clear_copy_on_writes(self) -> Dict[int, List[int]]:
76 + pass
77 +
78 + @abstractmethod
79 + def mark_blocks_as_computed(self) -> None:
80 + pass
81 +
82 + @abstractmethod
83 + def get_common_computed_block_ids(
84 + self, seq_block_ids: List[List[int]]) -> List[int]:
85 + pass
86 +
87 + class NoFreeBlocksError(ValueError):
88 + pass
89 +
90 +
91 +class DeviceAwareBlockAllocator(BlockAllocator):
92 +
93 + @abstractmethod
94 + def allocate_mutable(self, prev_block: Optional[Block],
95 + device: Device) -> Block:
96 + pass
97 +
98 + @abstractmethod
99 + def allocate_immutable(self, prev_block: Optional[Block],
100 + token_ids: List[int], device: Device) -> Block:
101 + pass
102 +
103 + @abstractmethod
104 + def get_num_free_blocks(self, device: Device) -> int:
105 + pass
vllm/core/block/naive_block.py new
+275
@@ -0,0 +1,275 @@
1 +from typing import Dict, Iterable, List, Optional, Set
2 +
3 +from vllm.core.block.common import (CopyOnWriteTracker, RefCounter,
4 + get_all_blocks_recursively)
5 +from vllm.core.block.interfaces import Block, BlockAllocator
6 +
7 +BlockId = int
8 +Refcount = int
9 +
10 +
11 +class NaiveBlockAllocator(BlockAllocator):
12 + """A simple block allocator that manages blocks of memory without prefix
13 + caching.
14 +
15 + Args:
16 + create_block (Block.Factory): A factory function for creating new
17 + blocks. This is used when a NaiveBlockAllocator is composed within
18 + a prefix caching allocator -- the naive block allocator must
19 + construct prefix caching blocks (but shouldn't know anything else
20 + about them).
21 + num_blocks (int): The total number of blocks to manage.
22 + block_size (int): The size of each block in tokens.
23 + block_ids (Optional[Iterable[int]], optional): An optional iterable of
24 + block IDs. If not provided, block IDs will be assigned sequentially
25 + from 0 to num_blocks - 1.
26 + """
27 +
28 + def __init__(
29 + self,
30 + create_block: Block.Factory,
31 + num_blocks: int,
32 + block_size: int,
33 + block_ids: Optional[Iterable[int]] = None,
34 + ):
35 + if block_ids is None:
36 + block_ids = range(num_blocks)
37 +
38 + self._free_block_indices: Set[BlockId] = set(block_ids)
39 + self._all_block_indices = frozenset(block_ids)
40 + assert len(self._all_block_indices) == num_blocks
41 +
42 + self._refcounter = RefCounter(
43 + all_block_indices=self._free_block_indices)
44 + self._create_block = create_block
45 + self._block_size = block_size
46 +
47 + self._cow_tracker = CopyOnWriteTracker(
48 + refcounter=self._refcounter.as_readonly(),
49 + allocator=self,
50 + )
51 +
52 + def allocate_immutable(self, prev_block: Optional[Block],
53 + token_ids: List[int]) -> Block:
54 + """Allocates a new immutable block with the given token IDs, linked to
55 + the previous block.
56 +
57 + Args:
58 + prev_block (Optional[Block]): The previous block in the sequence. If
59 + None, then the block to be allocated is the first block in the
60 + sequence.
61 + token_ids (List[int]): The token IDs to be stored in the new block.
62 +
63 + Returns:
64 + Block: The newly allocated immutable block.
65 + """
66 + block = self.allocate_mutable(prev_block=prev_block)
67 + block.append_token_ids(token_ids)
68 + return block
69 +
70 + def allocate_mutable(self, prev_block: Optional[Block]) -> Block:
71 + """Allocates a new mutable block, linked to the previous block.
72 +
73 + Args:
74 + prev_block (Optional[Block]): The previous block in the sequence. If
75 + None, then the block to be allocated is the first block in the
76 + sequence.
77 +
78 + Returns:
79 + Block: The newly allocated mutable block.
80 + """
81 + block_id = self._allocate_new_block_id()
82 + return self._create_block(
83 + prev_block=prev_block,
84 + token_ids=[],
85 + block_id=block_id,
86 + block_size=self._block_size,
87 + allocator=self,
88 + )
89 +
90 + def free(self, block: Block) -> None:
91 + self._free_block_id(block.block_id)
92 +
93 + # Mark the block as having no allocation.
94 + block.block_id = None
95 +
96 + def fork(self, last_block: Block) -> List[Block]:
97 + """Creates a new sequence of blocks that shares the same underlying
98 + memory as the original sequence.
99 +
100 + Args:
101 + last_block (Block): The last block in the original sequence.
102 +
103 + Returns:
104 + List[Block]: The new sequence of blocks that shares the same memory
105 + as the original sequence.
106 + """
107 + source_blocks = get_all_blocks_recursively(last_block)
108 +
109 + forked_blocks = []
110 + prev_block = None
111 + for block in source_blocks:
112 +
113 + # Increment refcount for each block.
114 + refcount = self._refcounter.incr(block.block_id)
115 + assert refcount != 1, "can't fork free'd block"
116 +
117 + forked_blocks.append(
118 + self._create_block(
119 + prev_block=prev_block,
120 + token_ids=block.token_ids,
121 + block_id=block.block_id,
122 + block_size=self._block_size,
123 + allocator=self,
124 + ))
125 + prev_block = forked_blocks[-1]
126 +
127 + return forked_blocks
128 +
129 + def get_num_free_blocks(self) -> int:
130 + return len(self._free_block_indices)
131 +
132 + def _allocate_new_block_id(self) -> BlockId:
133 + if not self._free_block_indices:
134 + raise BlockAllocator.NoFreeBlocksError()
135 +
136 + block_id = next(iter(self._free_block_indices))
137 + self._refcounter.incr(block_id)
138 + self._free_block_indices.remove(block_id)
139 + return block_id
140 +
141 + def _free_block_id(self, block_id: BlockId) -> None:
142 + refcount = self._refcounter.decr(block_id)
143 + if refcount == 0:
144 + self._free_block_indices.add(block_id)
145 +
146 + @property
147 + def refcounter(self):
148 + return self._refcounter
149 +
150 + @property
151 + def all_block_ids(self):
152 + return self._all_block_indices
153 +
154 + def cow_block_if_not_appendable(self, block: Block) -> Optional[BlockId]:
155 + """Performs a copy-on-write operation on the given block if it is not
156 + appendable.
157 +
158 + Args:
159 + block (Block): The block to check for copy-on-write.
160 +
161 + Returns:
162 + Optional[BlockId]: The block index of the new block if a copy-on
163 + -write operation was performed, or the original block index if
164 + no copy-on-write was necessary.
165 + """
166 + return self._cow_tracker.cow_block_if_not_appendable(block)
167 +
168 + def clear_copy_on_writes(self) -> Dict[BlockId, List[BlockId]]:
169 + """Returns the copy-on-write source->destination mapping and clears it.
170 +
171 + Returns:
172 + Dict[BlockId, List[BlockId]]: A dictionary mapping source
173 + block indices to lists of destination block indices.
174 + """
175 + return self._cow_tracker.clear_cows()
176 +
177 + def mark_blocks_as_computed(self) -> None:
178 + """Mark blocks as computed, used in prefix caching.
179 +
180 + Since the naive allocator does not implement prefix caching, we do
181 + nothing.
182 + """
183 + pass
184 +
185 + def get_common_computed_block_ids(
186 + self, seq_block_ids: List[List[int]]) -> List[int]:
187 + """Determine blocks that can be skipped in prefill.
188 +
189 + Since the naive allocator does not support prefix caching, always return
190 + an empty list.
191 + """
192 + return []
193 +
194 +
195 +class NaiveBlock(Block):
196 + """An implementation of the Block class that does not support prefix
197 + caching.
198 +
199 + The NaiveBlock class represents a block of token IDs with a fixed size. It
200 + provides methods for appending token IDs to the block and manages copy-on
201 + -write operations when necessary.
202 +
203 + Args:
204 + prev_block (Block): The previous block in the sequence.
205 + token_ids (List[int]): The initial token IDs to be stored in the block.
206 + block_size (int): The maximum number of token IDs that can be stored in
207 + the block.
208 + allocator (BlockAllocator): The block allocator associated with this
209 + block.
210 + block_id (Optional[int], optional): The physical block index
211 + of this block. Defaults to None, which means no allocation has been
212 + made.
213 + _cow_target (Optional[Block], optional): The copy-on-write target block.
214 + If not provided, it defaults to self.
215 + """
216 +
217 + def __init__(self,
218 + prev_block: Block,
219 + token_ids: List[int],
220 + block_size: int,
221 + allocator: BlockAllocator,
222 + block_id: Optional[int] = None,
223 + _cow_target: Optional[Block] = None):
224 + self._token_ids = []
225 + self._block_size = block_size
226 + self._prev_block = prev_block
227 + self._block_id = block_id
228 + self._allocator = allocator
229 + self._cow_target = _cow_target if _cow_target is not None else self
230 +
231 + self._append_token_ids_no_cow(token_ids)
232 +
233 + def append_token_ids(self, token_ids: List[int]) -> None:
234 + """Appends the given token IDs to the block, instructing the allocator
235 + to perform a copy-on-write if necessary.
236 +
237 + Args:
238 + token_ids (List[int]): The token IDs to be appended to the block.
239 + """
240 + self._append_token_ids_no_cow(token_ids)
241 +
242 + if self._block_id is not None:
243 + self._block_id = (self._allocator.cow_block_if_not_appendable(
244 + self._cow_target))
245 +
246 + def _append_token_ids_no_cow(self, token_ids: List[int]) -> None:
247 + assert self.num_empty_slots >= len(token_ids)
248 + self._token_ids.extend(token_ids)
249 +
250 + @property
251 + def block_id(self) -> Optional[int]:
252 + return self._block_id
253 +
254 + @block_id.setter
255 + def block_id(self, value: Optional[int]) -> None:
256 + self._block_id = value
257 +
258 + @property
259 + def is_full(self) -> bool:
260 + return self.num_empty_slots == 0
261 +
262 + @property
263 + def num_empty_slots(self) -> int:
264 + return self._block_size - len(self._token_ids)
265 +
266 + @property
267 + def token_ids(self) -> List[int]:
268 + return self._token_ids
269 +
270 + def block_size(self) -> int:
271 + return self._block_size
272 +
273 + @property
274 + def prev_block(self) -> Optional["Block"]:
275 + return self._prev_block
vllm/core/block/prefix_caching_block.py new
+472
@@ -0,0 +1,472 @@
1 +"""Token blocks."""
2 +from itertools import takewhile
3 +from os.path import commonprefix
4 +from typing import Dict, Iterable, List, Optional
5 +
6 +from vllm.core.block.common import (CopyOnWriteTracker,
7 + get_all_blocks_recursively)
8 +from vllm.core.block.interfaces import Block, BlockAllocator
9 +from vllm.core.block.naive_block import NaiveBlock, NaiveBlockAllocator
10 +
11 +PrefixHash = int
12 +BlockId = int
13 +
14 +
15 +class PrefixCachingBlockAllocator(BlockAllocator):
16 + """A block allocator that implements prefix caching.
17 +
18 + The PrefixCachingBlockAllocator maintains a cache of blocks based on their
19 + content hash. It reuses blocks with the same content hash to avoid redundant
20 + memory allocation. The allocator also supports copy-on-write operations.
21 +
22 + Args:
23 + num_blocks (int): The total number of blocks to manage.
24 + block_size (int): The size of each block in tokens.
25 + block_ids(Optional[Iterable[int]], optional): An optional iterable of
26 + block IDs. If not provided, block IDs will be assigned sequentially
27 + from 0 to num_blocks - 1.
28 + """
29 +
30 + # TODO last access time / evictor integration
31 +
32 + def __init__(
33 + self,
34 + num_blocks: int,
35 + block_size: int,
36 + block_ids: Optional[Iterable[int]] = None,
37 + ):
38 + # A mapping of prefix hash to block index. All blocks which have a
39 + # prefix hash will be in this dict, even if they have refcount 0.
40 + self._cached_blocks: Dict[PrefixHash, BlockId] = {}
41 +
42 + # A mapping of prefix hash to block index. All blocks which have a
43 + # prefix hash AND refcount 0 will be in this dict. Thus, it is a subset
44 + # of self._cached_blocks.
45 + self._unused_cached_blocks: Dict[PrefixHash, BlockId] = {}
46 +
47 + # An allocator for blocks that do not have prefix hashes.
48 + self._hashless_allocator = NaiveBlockAllocator(
49 + create_block=self._create_block,
50 + num_blocks=num_blocks,
51 + block_size=block_size,
52 + block_ids=block_ids,
53 + )
54 +
55 + self._block_size = block_size
56 +
57 + # We share the refcounter between allocators. This allows us to promote
58 + # blocks originally allocated in the hashless allocator to immutable
59 + # blocks.
60 + self._refcounter = self._hashless_allocator.refcounter
61 +
62 + self._cow_tracker = CopyOnWriteTracker(
63 + refcounter=self._refcounter.as_readonly(),
64 + allocator=self,
65 + )
66 +
67 + # Implements Block.Factory.
68 + def _create_block(
69 + self,
70 + prev_block: Optional[Block],
71 + token_ids: List[int],
72 + block_size: int,
73 + allocator: BlockAllocator,
74 + block_id: Optional[int] = None,
75 + ) -> Block:
76 + # Bind block to self.
77 + allocator = self
78 +
79 + return PrefixCachingBlock(
80 + prev_block=prev_block,
81 + token_ids=token_ids,
82 + block_size=block_size,
83 + block_id=block_id,
84 + prefix_caching_allocator=allocator,
85 + )
86 +
87 + def allocate_immutable(self, prev_block: Optional[Block],
88 + token_ids: List[int]) -> Block:
89 + """Allocates an immutable block with the given token IDs, reusing cached
90 + blocks if possible.
91 +
92 + Args:
93 + prev_block (Optional[Block]): The previous block in the sequence.
94 + token_ids (List[int]): The token IDs to be stored in the block.
95 +
96 + Returns:
97 + Block: The allocated immutable block.
98 + """
99 + assert_prefix_caching_block_or_none(prev_block)
100 +
101 + block = self._create_block(
102 + prev_block=prev_block,
103 + token_ids=token_ids,
104 + block_size=self._block_size,
105 + allocator=self,
106 + )
107 + assert block.content_hash is not None
108 +
109 + cached_block_id = self._cached_blocks.get(block.content_hash, None)
110 + if cached_block_id is not None:
111 + block.block_id = cached_block_id
112 + self._incr_refcount_cached_block(block.content_hash,
113 + block.block_id)
114 + return block
115 +
116 + block = self.allocate_mutable(prev_block)
117 + block.append_token_ids(token_ids)
118 + assert block.content_hash is not None
119 + # TODO computed bit
120 +
121 + return block
122 +
123 + def allocate_mutable(self, prev_block: Block) -> Block:
124 + """Allocates a mutable block. If there are no free blocks, this will
125 + evict unused cached blocks.
126 +
127 + Args:
128 + prev_block (Block): The previous block in the sequence.
129 +
130 + Returns:
131 + Block: The allocated mutable block.
132 + """
133 + assert_prefix_caching_block_or_none(prev_block)
134 +
135 + try:
136 + return self._hashless_allocator.allocate_mutable(
137 + prev_block=prev_block)
138 + except BlockAllocator.NoFreeBlocksError:
139 + # We must check the unused cached blocks before raising OOM.
140 + pass
141 +
142 + if self._unused_cached_blocks:
143 + # TODO policy for selecting block to remove
144 + content_hash_to_evict = next(iter(self._unused_cached_blocks))
145 +
146 + # Clear content hash mapping; the block will be overwritten.
147 + del self._cached_blocks[content_hash_to_evict]
148 +
149 + block_id = self._unused_cached_blocks.pop(content_hash_to_evict)
150 + refcount = self._refcounter.incr(block_id)
151 + assert refcount == 1
152 + block = self._create_block(
153 + prev_block=prev_block,
154 + token_ids=[],
155 + block_size=self._block_size,
156 + allocator=self,
157 + block_id=block_id,
158 + )
159 + assert block.content_hash is None
160 + return block
161 +
162 + # No block available in hashless allocator, nor in unused cache blocks.
163 + raise BlockAllocator.NoFreeBlocksError()
164 +
165 + def _incr_refcount_cached_block(self, content_hash: int,
166 + block_id: BlockId) -> None:
167 + refcount = self._refcounter.incr(block_id)
168 + if refcount == 1:
169 + assert content_hash in self._unused_cached_blocks
170 + del self._unused_cached_blocks[content_hash]
171 +
172 + def free(self, block: Block) -> None:
173 + """Decrement the refcount of the block. If the decremented refcount is
174 + zero, store the block in the freelist.
175 +
176 + If the block has a content hash (meaning it is immutable), then we will
177 + keep the block around in case future allocations require it.
178 + """
179 + assert (block.block_id
180 + is not None), "freeing unallocated block is undefined"
181 +
182 + self._free_block_id_for_block(block.block_id, block)
183 + block.block_id = None
184 +
185 + def _free_block_id_for_block(self, block_id: BlockId,
186 + block: Block) -> None:
187 + assert isinstance(block, PrefixCachingBlock)
188 +
189 + if block.content_hash is None:
190 + return self._hashless_allocator.free(block)
191 +
192 + refcount = self._refcounter.decr(block_id)
193 +
194 + # If no longer used, add the block to the unused cached blocks.
195 + if refcount == 0:
196 + assert block.content_hash not in self._unused_cached_blocks
197 + assert block.content_hash in self._cached_blocks
198 + self._unused_cached_blocks[block.content_hash] = block_id
199 +
200 + def fork(self, last_block: Block) -> List[Block]:
201 + """Creates a new sequence of blocks that shares the same underlying
202 + memory as the original sequence.
203 +
204 + Args:
205 + last_block (Block): The last block in the original sequence.
206 +
207 + Returns:
208 + List[Block]: The new sequence of blocks that shares the same memory
209 + as the original sequence.
210 + """
211 + source_blocks = get_all_blocks_recursively(last_block)
212 +
213 + forked_blocks = []
214 + prev_block = None
215 + for block in source_blocks:
216 + refcount = self._refcounter.incr(block.block_id)
217 + assert refcount != 1, "can't fork free'd block"
218 +
219 + forked_blocks.append(
220 + self._create_block(
221 + prev_block=prev_block,
222 + token_ids=block.token_ids,
223 + block_id=block.block_id,
224 + block_size=self._block_size,
225 + allocator=self,
226 + ))
227 + prev_block = forked_blocks[-1]
228 +
229 + return forked_blocks
230 +
231 + def get_num_free_blocks(self) -> int:
232 + # The number of free blocks is the number of hashless free blocks
233 + # plus the number of hashful blocks that are unused.
234 + return self._hashless_allocator.get_num_free_blocks() + len(
235 + self._unused_cached_blocks)
236 +
237 + @property
238 + def all_block_ids(self) -> frozenset[int]:
239 + return self._hashless_allocator.all_block_ids
240 +
241 + def promote_to_immutable_block(self,
242 + block: "PrefixCachingBlock") -> BlockId:
243 + """Once a mutable block is full, it can be promoted to an immutable
244 + block. This means that its content can be referenced by future blocks
245 + having the same prefix.
246 +
247 + Note that if we already have a cached block with the same content, we
248 + will replace the newly-promoted block's mapping with the existing cached
249 + block.
250 +
251 + Args:
252 + block (PrefixCachingBlock): The mutable block to be promoted.
253 +
254 + Returns:
255 + BlockId: Either the original block index, or the block index of
256 + the previously cached block matching the same content.
257 + """
258 + assert block.content_hash is not None
259 + assert block.block_id is not None
260 + assert self._refcounter.get(block.block_id) > 0
261 +
262 + # If the content hash does not have a corresponding cached block,
263 + # set this block as the cached block.
264 + if block.content_hash not in self._cached_blocks:
265 + self._cached_blocks[block.content_hash] = block.block_id
266 + else:
267 + self._free_block_id_for_block(block.block_id, block)
268 + self._incr_refcount_cached_block(
269 + block.content_hash, self._cached_blocks[block.content_hash])
270 +
271 + return self._cached_blocks[block.content_hash]
272 +
273 + def cow_block_if_not_appendable(self, block: Block) -> Optional[BlockId]:
274 + """Performs a copy-on-write operation on the given block if it is not
275 + appendable.
276 +
277 + Args:
278 + block (Block): The block to check for copy-on-write.
279 +
280 + Returns:
281 + Optional[BlockId]: The block index of the new block if a copy-on
282 + -write operation was performed, or the original block index if
283 + no copy-on-write was necessary.
284 + """
285 + return self._cow_tracker.cow_block_if_not_appendable(block)
286 +
287 + def clear_copy_on_writes(self) -> Dict[BlockId, List[BlockId]]:
288 + """Returns the copy-on-write source->destination mapping and clears it.
289 +
290 + Returns:
291 + Dict[BlockId, List[BlockId]]: A dictionary mapping source
292 + block indices to lists of destination block indices.
293 + """
294 + return self._cow_tracker.clear_cows()
295 +
296 + def mark_blocks_as_computed(self) -> None:
297 + """Mark blocks as computed, used in prefix caching."""
298 + # TODO Track computed blocks.
299 + pass
300 +
301 + def get_common_computed_block_ids(
302 + self, seq_block_ids: List[List[int]]) -> List[int]:
303 + """Return the block ids that are common for a given sequence group.
304 +
305 + Used in prefill (can skip prefill of some blocks).
306 + """
307 +
308 + # TODO: Track computed blocks.
309 + computed = lambda block_id: False
310 +
311 + # NOTE We exclude the last block to avoid the case where the entire
312 + # prompt is cached. This would cause erroneous behavior in model
313 + # runner.
314 + ids_list = [
315 + takewhile(lambda block_id: computed(block_id), seq[:-1])
316 + for seq in seq_block_ids
317 + ]
318 + return commonprefix([ids for ids in ids_list if ids != []])
319 +
320 +
321 +class PrefixCachingBlock(Block):
322 + """A block implementation that supports prefix caching.
323 +
324 + The PrefixCachingBlock class represents a block of token IDs with prefix
325 + caching capabilities. It wraps a NaiveBlock internally and provides
326 + additional functionality for content hashing and promoting immutable blocks
327 + with the prefix caching allocator.
328 +
329 + Args:
330 + prev_block (Optional[PrefixCachingBlock]): The previous block in the
331 + sequence.
332 + token_ids (List[int]): The initial token IDs to be stored in the block.
333 + block_size (int): The maximum number of token IDs that can be stored in
334 + the block.
335 + prefix_caching_allocator (PrefixCachingBlockAllocator): The prefix
336 + caching block allocator associated with this block.
337 + block_id (Optional[int], optional): The physical block index
338 + of this block. Defaults to None.
339 + """
340 +
341 + def __init__(
342 + self,
343 + prev_block: Optional["PrefixCachingBlock"],
344 + token_ids: List[int],
345 + block_size: int,
346 + prefix_caching_allocator: PrefixCachingBlockAllocator,
347 + block_id: Optional[int] = None,
348 + ):
349 + assert_prefix_caching_block_or_none(prev_block)
350 +
351 + self._prev_block = prev_block
352 + self._cached_content_hash: Optional[int] = None
353 + self._prefix_caching_allocator = prefix_caching_allocator
354 +
355 + self._block = NaiveBlock(
356 + prev_block=prev_block,
357 + token_ids=token_ids,
358 + block_size=block_size,
359 + block_id=block_id,
360 + allocator=prefix_caching_allocator,
361 + _cow_target=self,
362 + )
363 +
364 + def append_token_ids(self, token_ids: List[int]) -> None:
365 + """Appends the given token IDs to the block and registers the block as
366 + immutable if the block becomes full.
367 +
368 + Internally, the naive block handles CoW.
369 +
370 + Args:
371 + token_ids (List[int]): The token IDs to be appended to the block.
372 + """
373 + assert token_ids
374 +
375 + # naive block handles CoW.
376 + self._block.append_token_ids(token_ids)
377 +
378 + # If the content hash is present, then the block can be made immutable.
379 + # Register ourselves with the allocator, potentially replacing the
380 + # physical block index.
381 + if self.content_hash is not None:
382 + self.block_id = (self._prefix_caching_allocator.
383 + promote_to_immutable_block(self))
384 +
385 + @property
386 + def block_id(self) -> Optional[int]:
387 + return self._block.block_id
388 +
389 + @block_id.setter
390 + def block_id(self, value) -> None:
391 + self._block.block_id = value
392 +
393 + @property
394 + def is_full(self) -> bool:
395 + return self._block.is_full
396 +
397 + @property
398 + def num_empty_slots(self) -> int:
399 + return self._block.num_empty_slots
400 +
401 + @property
402 + def block_size(self) -> int:
403 + return self._block.block_size
404 +
405 + @property
406 + def token_ids(self) -> List[int]:
407 + return self._block.token_ids
408 +
409 + @property
410 + def prev_block(self) -> Optional[Block]:
411 + return self._prev_block
412 +
413 + @property
414 + def content_hash(self) -> Optional[int]:
415 + """Return the content-based hash of the current block, or None if it is
416 + not yet defined.
417 +
418 + For the content-based hash to be defined, the current block must be
419 + full.
420 + """
421 +
422 + # If the hash is already computed, return it.
423 + if self._cached_content_hash is not None:
424 + return self._cached_content_hash
425 +
426 + # We cannot compute a hash for the current block because it is not full.
427 + if not self.is_full:
428 + return None
429 +
430 + is_first_block = self._prev_block is None
431 + prev_block_hash = (None if is_first_block else
432 + self._prev_block.content_hash)
433 +
434 + # Previous block exists but does not yet have a hash.
435 + # Return no hash in this case.
436 + if prev_block_hash is None and not is_first_block:
437 + return None
438 +
439 + self._cached_content_hash = PrefixCachingBlock.hash_block_tokens(
440 + is_first_block,
441 + prev_block_hash,
442 + cur_block_token_ids=self.token_ids)
443 + return self._cached_content_hash
444 +
445 + @staticmethod
446 + def hash_block_tokens(is_first_block: bool, prev_block_hash: Optional[int],
447 + cur_block_token_ids: List[int]) -> int:
448 + """Computes a hash value corresponding to the contents of a block and
449 + the contents of the preceding block(s). The hash value is used for
450 + prefix caching.
451 +
452 + NOTE: Content-based hashing does not yet support LoRA.
453 +
454 + Parameters:
455 + - is_first_block (bool): A flag indicating if the block is the first in
456 + the sequence.
457 + - prev_block_hash (Optional[int]): The hash of the previous block. None
458 + if this is the first block.
459 + - cur_block_token_ids (List[int]): A list of token ids in the current
460 + block. The current block is assumed to be full.
461 +
462 + Returns:
463 + - int: The computed hash value for the block.
464 + """
465 + assert (prev_block_hash is None) == is_first_block
466 + return hash((is_first_block, prev_block_hash, *cur_block_token_ids))
467 +
468 +
469 +def assert_prefix_caching_block_or_none(block: Optional[Block]):
470 + if block is None:
471 + return
472 + assert isinstance(block, PrefixCachingBlock)
vllm/core/block_manager_v1.py renamed
+20 -22
@@ -1,5 +1,4 @@
1 """A block manager that manages token blocks."""
2 -import enum
2 from abc import ABC, abstractmethod
3 from itertools import count, takewhile
4 from os.path import commonprefix
@@ -7,6 +6,7 @@ from typing import Dict, List, Optional, Set, Tuple
6
7 from vllm.block import BlockTable, PhysicalTokenBlock
8 from vllm.core.evictor import EvictionPolicy, Evictor, make_evictor
9 +from vllm.core.interfaces import AllocStatus, BlockSpaceManager
10 from vllm.logger import init_logger
11 from vllm.sequence import Sequence, SequenceGroup, SequenceStatus
12 from vllm.utils import Device
@@ -196,21 +196,7 @@ class UncachedBlockAllocator(BlockAllocatorBase):
196 "Invalid codepath for uncached block allocator.")
197
198
199 -class AllocStatus(enum.Enum):
200 - """Result for BlockSpaceManager.can_allocate
201 -
202 - 1. Ok: seq_group can be allocated now.
203 - 2. Later: seq_group cannot be allocated.
204 - The capacity of allocator is larger than seq_group required.
205 - 3. Never: seq_group can never be allocated.
206 - The seq_group is too large to allocated in GPU.
207 - """
208 - OK = enum.auto()
209 - LATER = enum.auto()
210 - NEVER = enum.auto()
211 -
212 -
213 -class BlockSpaceManager:
199 +class BlockSpaceManagerV1(BlockSpaceManager):
200 """Manages the mapping between logical and physical token blocks."""
201
202 def __init__(
@@ -355,6 +341,11 @@ class BlockSpaceManager:
341 self,
342 seq: Sequence,
343 ) -> PhysicalTokenBlock:
344 + # Called before a new block is appended.
345 + # This is in charge of allocating a new physical block (to be appended).
346 +
347 + # None if the last block is not full. Otherwise, we set it to the
348 + # content hash.
349 if not self.enable_caching:
350 return self.gpu_allocator.allocate()
351 block_hash: Optional[int] = None
@@ -362,7 +353,14 @@ class BlockSpaceManager:
353 block_hash = seq.hash_of_block(len(seq.logical_token_blocks) - 1)
354 num_hashed_tokens = seq.num_hashed_tokens_of_block(
355 len(seq.logical_token_blocks) - 1)
356 +
357 + # num_hashed_tokens is used to compute future hashes
358 + # (e.g. in the hashing function, it is used to ask the sequence for
359 + # prefix tokens)
360 new_block = self.gpu_allocator.allocate(block_hash, num_hashed_tokens)
361 +
362 + # If the block has is None, then the block is not full.
363 + # If the block is not full, then we expect it to have a refcount of 1.
364 if block_hash is None:
365 assert new_block.ref_count == 1
366 return new_block
@@ -576,16 +574,16 @@ class BlockSpaceManager:
574 for b in takewhile(lambda b: b.computed, block_table[:-1])
575 ]
576
579 - def get_common_computed_block_ids(self,
580 - seq_group: SequenceGroup) -> List[int]:
577 + def get_common_computed_block_ids(self, seqs: List[Sequence]) -> List[int]:
578 + """Return the block ids that are common for a given sequence group.
579 +
580 + Used in prefill (can skip prefill of some blocks).
581 + """
582 # Can return non-empty result only with prefix caching enabled.
583 if not self.enable_caching:
584 return []
585
585 - ids_list = [
586 - self.get_all_computed_blocks(seq)
587 - for seq in iter(seq_group.seqs_dict.values())
588 - ]
586 + ids_list = [self.get_all_computed_blocks(seq) for seq in seqs]
587 return commonprefix([ids for ids in ids_list if ids != []])
588
589 def mark_blocks_as_computed(self, seq_group: SequenceGroup):
vllm/core/block_manager_v2.py new
+210
@@ -0,0 +1,210 @@
1 +"""A block manager that manages token blocks."""
2 +from typing import Dict, List, Optional, Tuple
3 +
4 +from vllm.core.block.block_table import BlockTable
5 +from vllm.core.block.cpu_gpu_block_allocator import CpuGpuBlockAllocator
6 +from vllm.core.interfaces import AllocStatus, BlockSpaceManager
7 +from vllm.sequence import Sequence, SequenceGroup, SequenceStatus
8 +from vllm.utils import Device
9 +
10 +SeqId = int
11 +
12 +
13 +class BlockSpaceManagerV2(BlockSpaceManager):
14 + """BlockSpaceManager which manages the allocation of KV cache.
15 +
16 + It owns responsibility for allocation, swapping, allocating memory for
17 + autoregressively-generated tokens, and other advanced features such as
18 + prefix caching, forking/copy-on-write, and sliding-window memory allocation.
19 +
20 + The current implementation is partial; in particular prefix caching and
21 + sliding-window are not feature complete. This class implements the design
22 + described in https://github.com/vllm-project/vllm/pull/3492.
23 +
24 + Args:
25 + block_size (int): The size of each memory block.
26 + num_gpu_blocks (int): The number of memory blocks allocated on GPU.
27 + num_cpu_blocks (int): The number of memory blocks allocated on CPU.
28 + watermark (float, optional): The threshold used for memory swapping.
29 + Defaults to 0.01.
30 + sliding_window (Optional[int], optional): The size of the sliding
31 + window. Defaults to None.
32 + enable_caching (bool, optional): Flag indicating whether caching is
33 + enabled. Defaults to False.
34 + """
35 +
36 + def __init__(
37 + self,
38 + block_size: int,
39 + num_gpu_blocks: int,
40 + num_cpu_blocks: int,
41 + watermark: float = 0.01,
42 + sliding_window: Optional[int] = None,
43 + enable_caching: bool = False,
44 + ) -> None:
45 + self.block_size = block_size
46 + self.num_total_gpu_blocks = num_gpu_blocks
47 + self.num_total_cpu_blocks = num_cpu_blocks
48 +
49 + assert sliding_window is None, "Sliding window not yet supported"
50 +
51 + self.block_sliding_window = None
52 +
53 + self.watermark = watermark
54 + assert watermark >= 0.0
55 +
56 + assert not enable_caching, "Prefix caching not yet supported"
57 + self.enable_caching = enable_caching
58 +
59 + self.watermark_blocks = int(watermark * num_gpu_blocks)
60 +
61 + self.block_allocator = CpuGpuBlockAllocator.create(
62 + # Currently, only naive blocks are supported (no prefix caching).
63 + allocator_type="naive",
64 + num_gpu_blocks=num_gpu_blocks,
65 + num_cpu_blocks=num_cpu_blocks,
66 + block_size=block_size,
67 + )
68 +
69 + self.block_tables: Dict[SeqId, BlockTable] = {}
70 +
71 + def can_allocate(self, seq_group: SequenceGroup) -> AllocStatus:
72 + # FIXME(woosuk): Here we assume that all sequences in the group share
73 + # the same prompt. This may not be true for preempted sequences.
74 + seq = seq_group.get_seqs(status=SequenceStatus.WAITING)[0]
75 +
76 + num_required_blocks = BlockTable.get_num_required_blocks(
77 + seq.get_token_ids(),
78 + block_size=self.block_size,
79 + )
80 +
81 + assert self.block_sliding_window is None
82 + if self.block_sliding_window is not None:
83 + num_required_blocks = min(num_required_blocks,
84 + self.block_sliding_window)
85 +
86 + num_free_gpu_blocks = self.block_allocator.get_num_free_blocks(
87 + device=Device.GPU)
88 +
89 + # Use watermark to avoid frequent cache eviction.
90 + if (self.num_total_gpu_blocks - num_required_blocks <
91 + self.watermark_blocks):
92 + return AllocStatus.NEVER
93 + if num_free_gpu_blocks - num_required_blocks >= self.watermark_blocks:
94 + return AllocStatus.OK
95 + else:
96 + return AllocStatus.LATER
97 +
98 + def allocate(self, seq_group: SequenceGroup) -> None:
99 + waiting_seqs = seq_group.get_seqs(status=SequenceStatus.WAITING)
100 + assert not (set(seq.seq_id for seq in waiting_seqs)
101 + & self.block_tables.keys()), "block table already exists"
102 +
103 + # NOTE: Here we assume that all sequences in the group have the same
104 + # prompt.
105 + seq = waiting_seqs[0]
106 +
107 + block_table = BlockTable(
108 + block_size=self.block_size,
109 + block_allocator=self.block_allocator,
110 + )
111 + assert self.block_sliding_window is None
112 + block_table.allocate(seq.get_token_ids())
113 + self.block_tables[seq.seq_id] = block_table
114 +
115 + # Assign the block table for each sequence.
116 + for seq in waiting_seqs[1:]:
117 + self.block_tables[seq.seq_id] = block_table.fork()
118 +
119 + def can_append_slot(self, seq_group: SequenceGroup) -> bool:
120 + # Simple heuristic: If there is at least one free block
121 + # for each sequence, we can append.
122 + num_free_gpu_blocks = self.block_allocator.get_num_free_blocks(
123 + Device.GPU)
124 + num_seqs = seq_group.num_seqs(status=SequenceStatus.RUNNING)
125 + return num_seqs <= num_free_gpu_blocks
126 +
127 + def append_slot(
128 + self,
129 + seq: Sequence,
130 + ) -> Optional[Tuple[int, int]]:
131 +
132 + block_table = self.block_tables[seq.seq_id]
133 +
134 + # Get unseen token ids.
135 + num_full_slots = block_table.num_full_slots
136 + unseen_token_ids = seq.get_token_ids()[num_full_slots:]
137 + assert unseen_token_ids
138 +
139 + block_table.append_token_ids(unseen_token_ids)
140 +
141 + # Return any copy-on-writes.
142 + _ = self.block_allocator.clear_copy_on_writes()
143 +
144 + # TODO extend append_slot interface to append_slots
145 + # @cadedaniel will do in https://github.com/vllm-project/vllm/pull/3250
146 +
147 + return None
148 +
149 + def free(self, seq: Sequence) -> None:
150 + if seq.seq_id not in self.block_tables:
151 + # Already freed or haven't been scheduled yet.
152 + return
153 + self.block_tables[seq.seq_id].free()
154 + del self.block_tables[seq.seq_id]
155 +
156 + def get_block_table(self, seq: Sequence) -> List[int]:
157 + assert seq.seq_id in self.block_tables
158 + block_ids = self.block_tables[seq.seq_id].physical_block_ids
159 + assert all(b is not None for b in block_ids)
160 + return block_ids
161 +
162 + def access_all_blocks_in_seq(self, seq, now):
163 + # TODO add prefix caching support.
164 + # Tracked here https://github.com/vllm-project/vllm/issues/3667
165 + pass
166 +
167 + def mark_blocks_as_computed(self, seq_group: SequenceGroup):
168 + # We ignore the sequence group as its not necessary. After the batch is
169 + # formed by the scheduler, we do not need to mark blocks from individual
170 + # sequence groups as computed -- all blocks in the batch can be marked
171 + # as computed.
172 + self.block_allocator.mark_blocks_as_computed()
173 +
174 + def get_common_computed_block_ids(self, seqs: List[Sequence]) -> List[int]:
175 + """Determine which blocks for which we skip prefill.
176 +
177 + With prefix caching we can skip prefill for previously-generated blocks.
178 + Currently, the attention implementation only supports skipping cached
179 + blocks if they are a contiguous prefix of cached blocks.
180 +
181 + This method determines which blocks can be safely skipped for all
182 + sequences in the sequence group.
183 + """
184 + seq_block_ids = [
185 + self.block_tables[seq.seq_id].physical_block_ids for seq in seqs
186 + ]
187 + return self.block_allocator.get_common_computed_block_ids(
188 + seq_block_ids)
189 +
190 + def fork(self, parent_seq: Sequence, child_seq: Sequence) -> None:
191 + src_block_table = self.block_tables[parent_seq.seq_id]
192 + self.block_tables[child_seq.seq_id] = src_block_table.fork()
193 +
194 + def can_swap_in(self, seq_group: SequenceGroup) -> bool:
195 + return False
196 +
197 + def swap_in(self, seq_group: SequenceGroup) -> Dict[int, int]:
198 + raise NotImplementedError
199 +
200 + def can_swap_out(self, seq_group: SequenceGroup) -> bool:
201 + return False
202 +
203 + def swap_out(self, seq_group: SequenceGroup) -> Dict[int, int]:
204 + raise NotImplementedError
205 +
206 + def get_num_free_gpu_blocks(self) -> int:
207 + return self.block_allocator.get_num_free_blocks(Device.GPU)
208 +
209 + def get_num_free_cpu_blocks(self) -> int:
210 + return self.block_allocator.get_num_free_blocks(Device.CPU)
vllm/core/interfaces.py new
+107
@@ -0,0 +1,107 @@
1 +import enum
2 +from abc import ABC, abstractmethod
3 +from typing import Dict, List, Optional, Tuple
4 +
5 +from vllm.sequence import Sequence, SequenceGroup
6 +
7 +
8 +class AllocStatus(enum.Enum):
9 + """Result for BlockSpaceManager.can_allocate
10 +
11 + 1. Ok: seq_group can be allocated now.
12 + 2. Later: seq_group cannot be allocated.
13 + The capacity of allocator is larger than seq_group required.
14 + 3. Never: seq_group can never be allocated.
15 + The seq_group is too large to allocated in GPU.
16 + """
17 + OK = enum.auto()
18 + LATER = enum.auto()
19 + NEVER = enum.auto()
20 +
21 +
22 +class BlockSpaceManager(ABC):
23 +
24 + @staticmethod
25 + def get_block_space_manager_class(version: str):
26 + version = version.lower()
27 +
28 + if version == "v1":
29 + from vllm.core.block_manager_v1 import BlockSpaceManagerV1
30 + return BlockSpaceManagerV1
31 +
32 + if version == "v2":
33 + from vllm.core.block_manager_v2 import BlockSpaceManagerV2
34 + return BlockSpaceManagerV2
35 +
36 + raise ValueError(f"Unknown version {version=}")
37 +
38 + @abstractmethod
39 + def can_allocate(self, seq_group: SequenceGroup) -> AllocStatus:
40 + pass
41 +
42 + @abstractmethod
43 + def allocate(self, seq_group: SequenceGroup) -> None:
44 + pass
45 +
46 + @abstractmethod
47 + def can_append_slot(self, seq_group: SequenceGroup) -> bool:
48 + pass
49 +
50 + @abstractmethod
51 + def append_slot(
52 + self,
53 + seq: Sequence,
54 + ) -> Optional[Tuple[int, int]]:
55 + pass
56 +
57 + @abstractmethod
58 + def fork(self, parent_seq: Sequence, child_seq: Sequence) -> None:
59 + pass
60 +
61 + @abstractmethod
62 + def can_swap_in(self, seq_group: SequenceGroup) -> bool:
63 + pass
64 +
65 + @abstractmethod
66 + def swap_in(self, seq_group: SequenceGroup) -> Dict[int, int]:
67 + pass
68 +
69 + @abstractmethod
70 + def can_swap_out(self, seq_group: SequenceGroup) -> bool:
71 + pass
72 +
73 + @abstractmethod
74 + def swap_out(self, seq_group: SequenceGroup) -> Dict[int, int]:
75 + pass
76 +
77 + @abstractmethod
78 + def free(self, seq: Sequence) -> None:
79 + pass
80 +
81 + @abstractmethod
82 + def get_block_table(self, seq: Sequence) -> List[int]:
83 + pass
84 +
85 + @abstractmethod
86 + def get_num_free_gpu_blocks(self) -> int:
87 + pass
88 +
89 + @abstractmethod
90 + def get_num_free_cpu_blocks(self) -> int:
91 + pass
92 +
93 + @abstractmethod
94 + def access_all_blocks_in_seq(
95 + self,
96 + seq: Sequence,
97 + access_time: float,
98 + ) -> None:
99 + pass
100 +
101 + @abstractmethod
102 + def get_common_computed_block_ids(self, seqs: List[Sequence]) -> List[int]:
103 + pass
104 +
105 + @abstractmethod
106 + def mark_blocks_as_computed(self, seq_group: SequenceGroup):
107 + pass
vllm/core/scheduler.py
+20 -7
@@ -4,7 +4,7 @@ from collections import deque
4 from typing import Deque, Dict, Iterable, List, Optional, Set, Tuple, Union
5
6 from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig
7 -from vllm.core.block_manager import AllocStatus, BlockSpaceManager
7 +from vllm.core.interfaces import AllocStatus, BlockSpaceManager
8 from vllm.core.policy import PolicyFactory
9 from vllm.logger import init_logger
10 from vllm.lora.request import LoRARequest
@@ -88,8 +88,13 @@ class Scheduler:
88
89 # Instantiate the scheduling policy.
90 self.policy = PolicyFactory.get_policy(policy_name="fcfs")
91 +
92 + BlockSpaceManagerImpl = BlockSpaceManager.get_block_space_manager_class(
93 + version="v2" if self.scheduler_config.
94 + use_v2_block_manager else "v1")
95 +
96 # Create the block space manager.
92 - self.block_manager = BlockSpaceManager(
97 + self.block_manager = BlockSpaceManagerImpl(
98 block_size=self.cache_config.block_size,
99 num_gpu_blocks=self.cache_config.num_gpu_blocks,
100 num_cpu_blocks=self.cache_config.num_cpu_blocks,
@@ -378,6 +383,10 @@ class Scheduler:
383 block_tables[seq_id] = self.block_manager.get_block_table(seq)
384 self.block_manager.access_all_blocks_in_seq(seq, now)
385
386 + common_computed_block_nums = (
387 + self.block_manager.get_common_computed_block_ids(
388 + seq_group.get_seqs(status=SequenceStatus.RUNNING)))
389 +
390 seq_group_metadata = SequenceGroupMetadata(
391 request_id=seq_group.request_id,
392 is_prompt=scheduler_outputs.prompt_run,
@@ -385,8 +394,7 @@ class Scheduler:
394 sampling_params=seq_group.sampling_params,
395 block_tables=block_tables,
396 lora_request=seq_group.lora_request,
388 - computed_block_nums=self.block_manager.
389 - get_common_computed_block_ids(seq_group),
397 + computed_block_nums=common_computed_block_nums,
398 state=seq_group.state,
399 # `multi_modal_data` will only be present for the 1st comm
400 # between engine and worker.
@@ -396,6 +404,14 @@ class Scheduler:
404 if scheduler_outputs.prompt_run else None,
405 )
406 seq_group_metadata_list.append(seq_group_metadata)
407 +
408 + # Now that the batch has been created, we can assume all blocks in the
409 + # batch will have been computed before the next scheduling invocation.
410 + # This is because the engine assumes that a failure in model execution
411 + # will crash the vLLM instance / will not retry.
412 + for seq_group in scheduler_outputs.scheduled_seq_groups:
413 + self.block_manager.mark_blocks_as_computed(seq_group)
414 +
415 return seq_group_metadata_list, scheduler_outputs
416
417 def fork_seq(self, parent_seq: Sequence, child_seq: Sequence) -> None:
@@ -503,9 +519,6 @@ class Scheduler:
519 for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
520 seq.status = SequenceStatus.SWAPPED
521
506 - def mark_blocks_as_computed(self, seq_group: SequenceGroup):
507 - self.block_manager.mark_blocks_as_computed(seq_group)
508 -
522 def _passed_delay(self, now: float) -> bool:
523 if self.prev_prompt:
524 self.last_prompt_latency = now - self.prev_time
vllm/engine/arg_utils.py
+15
@@ -28,6 +28,7 @@ class EngineArgs:
28 max_parallel_loading_workers: Optional[int] = None
29 block_size: int = 16
30 enable_prefix_caching: bool = False
31 + use_v2_block_manager: bool = False
32 swap_space: int = 4 # GiB
33 gpu_memory_utilization: float = 0.90
34 max_num_batched_tokens: Optional[int] = None
@@ -52,6 +53,9 @@ class EngineArgs:
53 max_cpu_loras: Optional[int] = None
54 device: str = 'auto'
55 ray_workers_use_nsight: bool = False
56 +
57 + forced_num_gpu_blocks: Optional[int] = None
58 +
59 # Related to Vision-language models such as llava
60 image_input_type: Optional[str] = None
61 image_token_id: Optional[int] = None
@@ -194,6 +198,9 @@ class EngineArgs:
198 parser.add_argument('--enable-prefix-caching',
199 action='store_true',
200 help='Enables automatic prefix caching')
201 + parser.add_argument('--use-v2-block-manager',
202 + action='store_true',
203 + help='Use BlockSpaceMangerV2')
204
205 parser.add_argument('--seed',
206 type=int,
@@ -210,6 +217,12 @@ class EngineArgs:
217 help='the fraction of GPU memory to be used for '
218 'the model executor, which can range from 0 to 1.'
219 'If unspecified, will use the default value of 0.9.')
220 + parser.add_argument(
221 + '--forced-num-gpu-blocks',
222 + type=int,
223 + default=None,
224 + help='If specified, ignore GPU profiling result and use this number'
225 + 'of GPU blocks. Used for testing preemption.')
226 parser.add_argument('--max-num-batched-tokens',
227 type=int,
228 default=EngineArgs.max_num_batched_tokens,
@@ -369,6 +382,7 @@ class EngineArgs:
382 cache_config = CacheConfig(self.block_size,
383 self.gpu_memory_utilization,
384 self.swap_space, self.kv_cache_dtype,
385 + self.forced_num_gpu_blocks,
386 model_config.get_sliding_window(),
387 self.enable_prefix_caching)
388 parallel_config = ParallelConfig(
@@ -383,6 +397,7 @@ class EngineArgs:
397 scheduler_config = SchedulerConfig(self.max_num_batched_tokens,
398 self.max_num_seqs,
399 model_config.max_model_len,
400 + self.use_v2_block_manager,
401 self.scheduler_delay_factor)
402 lora_config = LoRAConfig(
403 max_lora_rank=self.max_lora_rank,
vllm/engine/llm_engine.py
-6
@@ -553,12 +553,6 @@ class LLMEngine:
553 # Update the scheduled sequence groups with the model outputs.
554 scheduled_seq_groups = scheduler_outputs.scheduled_seq_groups
555
556 - # If prefix caching is enabled, mark all blocks in the sequence groups
557 - # as completed so that future requests don't attempt to recompute them
558 - if self.cache_config.enable_prefix_caching:
559 - for seq_group in scheduled_seq_groups:
560 - self.scheduler.mark_blocks_as_computed(seq_group)
561 -
556 for seq_group, outputs in zip(scheduled_seq_groups, output):
557 self._process_sequence_group_outputs(seq_group, outputs)
558
vllm/executor/gpu_executor.py
+6
@@ -85,6 +85,12 @@ class GPUExecutor(ExecutorBase):
85 cache_dtype=self.cache_config.cache_dtype,
86 ))
87
88 + if self.cache_config.forced_num_gpu_blocks is not None:
89 + forced_num_gpu_blocks = self.cache_config.forced_num_gpu_blocks
90 + logger.info(f"Replacing profiled {num_gpu_blocks=} with "
91 + f"{forced_num_gpu_blocks=}")
92 + num_gpu_blocks = forced_num_gpu_blocks
93 +
94 logger.info(f"# GPU blocks: {num_gpu_blocks}, "
95 f"# CPU blocks: {num_cpu_blocks}")
96
vllm/executor/ray_gpu_executor.py
+7
@@ -232,6 +232,13 @@ class RayGPUExecutor(ExecutorBase):
232 # operators can be applied to all workers.
233 num_gpu_blocks = min(b[0] for b in num_blocks)
234 num_cpu_blocks = min(b[1] for b in num_blocks)
235 +
236 + if self.cache_config.forced_num_gpu_blocks is not None:
237 + forced_num_gpu_blocks = self.cache_config.forced_num_gpu_blocks
238 + logger.info(f"Replacing profiled {num_gpu_blocks=} with "
239 + f"{forced_num_gpu_blocks=}")
240 + num_gpu_blocks = forced_num_gpu_blocks
241 +
242 logger.info(f"# GPU blocks: {num_gpu_blocks}, "
243 f"# CPU blocks: {num_cpu_blocks}")
244
vllm/sequence.py
+2
@@ -196,6 +196,8 @@ class Sequence:
196 return self.lora_request.lora_int_id if self.lora_request else 0
197
198 def hash_of_block(self, logical_idx: int) -> int:
199 + # TODO This can produce incorrect hash when block size > prompt size
200 +
201 # Compute the number of tokens in the sequence
202 # TODO: The current hashing function is O(L^2). We should optimize
203 # this in the future.
vllm/utils.py
+10
@@ -227,6 +227,16 @@ def set_cuda_visible_devices(device_ids: List[int]) -> None:
227 os.environ["CUDA_VISIBLE_DEVICES"] = ",".join(map(str, device_ids))
228
229
230 +def chunk_list(lst, chunk_size):
231 + """Yield successive chunk_size chunks from lst."""
232 + return [lst[i:i + chunk_size] for i in range(0, len(lst), chunk_size)]
233 +
234 +
235 +def cdiv(a: int, b: int) -> int:
236 + """Ceiling division."""
237 + return -(a // -b)
238 +
239 +
240 @lru_cache(maxsize=None)
241 def get_nvcc_cuda_version() -> Optional[Version]:
242 cuda_home = os.environ.get('CUDA_HOME')