A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Core][Bugfix]Refactor block manager for better testability (#3492)
Cade Daniel committed
Mar 27, 2024 at 23:59 UTC
14ccd94c89d0ffd9da283545d93ab1dfea5da340
30 files changed
+3285
-77
tests/core/block/__init__.py
tests/core/block/e2e/conftest.py
new
+56
@@ -0,0 +1,56 @@
1
+import contextlib
2
+import gc
3
+
4
+import pytest
5
+import ray
6
+import torch
7
+
8
+from vllm import LLM
9
+from vllm.model_executor.parallel_utils.parallel_state import (
10
+ destroy_model_parallel)
11
+from vllm.model_executor.utils import set_random_seed
12
+
13
+
14
+def cleanup():
15
+ destroy_model_parallel()
16
+ with contextlib.suppress(AssertionError):
17
+ torch.distributed.destroy_process_group()
18
+ gc.collect()
19
+ torch.cuda.empty_cache()
20
+ ray.shutdown()
21
+
22
+
23
+@pytest.fixture
24
+def baseline_llm_generator(common_llm_kwargs, per_test_common_llm_kwargs,
25
+ baseline_llm_kwargs, seed):
26
+ return create_llm_generator(common_llm_kwargs, per_test_common_llm_kwargs,
27
+ baseline_llm_kwargs, seed)
28
+
29
+
30
+@pytest.fixture
31
+def test_llm_generator(common_llm_kwargs, per_test_common_llm_kwargs,
32
+ test_llm_kwargs, seed):
33
+ return create_llm_generator(common_llm_kwargs, per_test_common_llm_kwargs,
34
+ test_llm_kwargs, seed)
35
+
36
+
37
+def create_llm_generator(common_llm_kwargs, per_test_common_llm_kwargs,
38
+ distinct_llm_kwargs, seed):
39
+ kwargs = {
40
+ **common_llm_kwargs,
41
+ **per_test_common_llm_kwargs,
42
+ **distinct_llm_kwargs,
43
+ }
44
+
45
+ def generator_inner():
46
+ llm = LLM(**kwargs)
47
+
48
+ set_random_seed(seed)
49
+
50
+ yield llm
51
+ del llm
52
+ cleanup()
53
+
54
+ for llm in generator_inner():
55
+ yield llm
56
+ del llm
tests/core/block/e2e/test_correctness.py
new
+86
@@ -0,0 +1,86 @@
1
+from itertools import cycle
2
+
3
+import pytest
4
+
5
+from vllm import SamplingParams
6
+
7
+
8
+@pytest.mark.parametrize(
9
+ "common_llm_kwargs",
10
+ [{
11
+ # Use a small model for a fast test.
12
+ "model": "facebook/opt-125m",
13
+
14
+ # skip cuda graph creation for fast test.
15
+ "enforce_eager": True,
16
+
17
+ # Allow only 5 sequences of ~1024 tokens in worst case.
18
+ "block_size": 16,
19
+ "forced_num_gpu_blocks": 5 * (64 + 1),
20
+ }])
21
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
22
+@pytest.mark.parametrize("baseline_llm_kwargs", [{
23
+ "use_v2_block_manager": False
24
+}])
25
+@pytest.mark.parametrize("test_llm_kwargs", [{"use_v2_block_manager": True}])
26
+@pytest.mark.parametrize("batch_size", [10])
27
+@pytest.mark.parametrize("seed", [1])
28
+def test_v1_v2_greedy_equality_with_preemption(baseline_llm_generator,
29
+ test_llm_generator, batch_size):
30
+ """Verify block manager v2 produces same outputs as block manager v1, even
31
+ when there is preemption.
32
+
33
+ This constructs two LLM, each with limited number of GPU blocks. The limit
34
+ is decided such that as the sequences in the batch grow, sequences must be
35
+ preempted and removed from cache.
36
+
37
+ If the output token ids are equivalent, then we have confidence that the KV
38
+ cache is not corrupted in the v2 block manager.
39
+
40
+ NOTE: We want a significant number of generated tokens so that any incorrect
41
+ KV mapping has time to build up error.
42
+ """
43
+ output_len = 1024
44
+ temperature = 0.0
45
+
46
+ # We want to ensure equality even with preemption.
47
+ # We force the total block size to be 1 + cdiv(output_len, block_size)
48
+ # so that only one sequence can fit at a time (once the sequences grow).
49
+
50
+ prompts = [
51
+ "Hello, my name is",
52
+ "The president of the United States is",
53
+ "The capital of France is",
54
+ "The future of AI is",
55
+ ]
56
+
57
+ prompts = [prompt for prompt, _ in zip(cycle(prompts), range(batch_size))]
58
+
59
+ sampling_params = SamplingParams(
60
+ max_tokens=output_len,
61
+ ignore_eos=True,
62
+ temperature=temperature,
63
+ )
64
+
65
+ print('Getting token ids from block manager v1')
66
+ baseline_token_ids = get_token_ids_from_llm_generator(
67
+ baseline_llm_generator, prompts, sampling_params)
68
+
69
+ print('Getting token ids from block manager v2')
70
+ test_token_ids = get_token_ids_from_llm_generator(test_llm_generator,
71
+ prompts, sampling_params)
72
+
73
+ for expected_token_ids, actual_token_ids in zip(baseline_token_ids,
74
+ test_token_ids):
75
+ assert expected_token_ids == actual_token_ids
76
+
77
+ assert baseline_token_ids == test_token_ids
78
+
79
+
80
+def get_token_ids_from_llm_generator(llm_generator, prompts, sampling_params):
81
+ for llm in llm_generator:
82
+ outputs = llm.generate(prompts, sampling_params, use_tqdm=True)
83
+ token_ids = [output.outputs[0].token_ids for output in outputs]
84
+ del llm
85
+
86
+ return token_ids
tests/core/block/test_block_space_manager.py
new
+50
@@ -0,0 +1,50 @@
1
+import pytest
2
+
3
+from vllm.core.block_manager_v2 import BlockSpaceManagerV2
4
+from vllm.core.interfaces import AllocStatus
5
+
6
+from ..utils import create_seq_group
7
+
8
+
9
+@pytest.mark.parametrize("block_size", [16])
10
+@pytest.mark.parametrize("num_gpu_blocks", [8, 40, 80])
11
+@pytest.mark.parametrize("num_seqs_per_group", [1, 4])
12
+@pytest.mark.parametrize("watermark", [0.0, 0.5])
13
+def test_can_allocate_seq_group(block_size: int, num_seqs_per_group: int,
14
+ num_gpu_blocks: int, watermark: float):
15
+ block_manager = BlockSpaceManagerV2(
16
+ block_size=block_size,
17
+ num_gpu_blocks=num_gpu_blocks,
18
+ num_cpu_blocks=1024,
19
+ watermark=watermark,
20
+ )
21
+ num_watermark_blocks = int(watermark * num_gpu_blocks)
22
+
23
+ num_output_blocks_per_seq = 1
24
+
25
+ # NOTE: This should be num_output_blocks_per_seq * num_seqs_per_group, but
26
+ # the current implementation assumes all seqs are new prompts / don't have
27
+ # different output lens.
28
+ num_output_blocks = num_output_blocks_per_seq
29
+
30
+ for num_prompt_blocks in range(1, num_gpu_blocks - num_output_blocks):
31
+ seq_group = create_seq_group(
32
+ seq_prompt_lens=block_size * num_prompt_blocks,
33
+ seq_output_lens=[
34
+ block_size * num_output_blocks_per_seq
35
+ for _ in range(num_seqs_per_group)
36
+ ],
37
+ )
38
+
39
+ assert num_prompt_blocks + num_output_blocks <= num_gpu_blocks
40
+
41
+ can_allocate_result = block_manager.can_allocate(seq_group)
42
+
43
+ num_required_blocks = num_prompt_blocks + num_output_blocks
44
+
45
+ if num_gpu_blocks - num_required_blocks < num_watermark_blocks:
46
+ assert can_allocate_result == AllocStatus.NEVER
47
+ elif num_gpu_blocks >= num_required_blocks:
48
+ assert can_allocate_result == AllocStatus.OK
49
+ else:
50
+ assert can_allocate_result == AllocStatus.LATER
tests/core/block/test_block_table.py
new
+500
@@ -0,0 +1,500 @@
1
+import pytest
2
+
3
+from vllm.core.block.block_table import BlockTable
4
+from vllm.core.block.cpu_gpu_block_allocator import CpuGpuBlockAllocator
5
+from vllm.utils import Device, cdiv, chunk_list
6
+
7
+
8
+@pytest.mark.parametrize("block_size", [16])
9
+@pytest.mark.parametrize("sequence_len", [1, 16, 129])
10
+def test_allocate_naive(block_size: int, sequence_len: int):
11
+ """Test the allocation of blocks using the naive allocator.
12
+
13
+ This test creates a CpuGpuBlockAllocator with the specified block size and
14
+ number of blocks. It then allocates multiple BlockTables with varying
15
+ sequence lengths and verifies that the number of free blocks decreases as
16
+ expected after each allocation.
17
+ """
18
+ assert block_size > 1
19
+ num_gpu_blocks = 1024
20
+
21
+ allocator = CpuGpuBlockAllocator.create(
22
+ allocator_type="naive",
23
+ num_gpu_blocks=num_gpu_blocks,
24
+ num_cpu_blocks=1024,
25
+ block_size=block_size,
26
+ )
27
+
28
+ token_ids = list(range(sequence_len))
29
+ num_blocks_per_alloc = len(list(chunk_list(token_ids, block_size)))
30
+
31
+ block_tables = []
32
+ for i in range(5):
33
+ assert allocator.get_num_free_blocks(
34
+ device=Device.GPU) == num_gpu_blocks - i * num_blocks_per_alloc
35
+
36
+ block_tables.append(
37
+ BlockTable(
38
+ block_size=block_size,
39
+ block_allocator=allocator,
40
+ ))
41
+ block_tables[-1].allocate(token_ids=token_ids, device=Device.GPU)
42
+
43
+
44
+@pytest.mark.parametrize("block_size", [16])
45
+@pytest.mark.parametrize("sequence_len", [1, 16, 129])
46
+def test_allocate_prefix_caching(block_size: int, sequence_len: int):
47
+ """Test the allocation of blocks using the prefix caching allocator.
48
+
49
+ This test creates a CpuGpuBlockAllocator with the specified block size and
50
+ number of blocks, using the prefix caching allocator. It then allocates
51
+ multiple BlockTables with varying sequence lengths and verifies that the
52
+ number of free blocks decreases as expected after each allocation.
53
+
54
+ The test expects all sequences to share allocations, except for their last
55
+ block, which may be mutable. It calculates the expected number of immutable
56
+ and mutable blocks per allocation based on the sequence length and block
57
+ size.
58
+ """
59
+ assert block_size > 1
60
+ num_gpu_blocks = 1024
61
+
62
+ allocator = CpuGpuBlockAllocator.create(
63
+ allocator_type="prefix_caching",
64
+ num_gpu_blocks=num_gpu_blocks,
65
+ num_cpu_blocks=1024,
66
+ block_size=block_size,
67
+ )
68
+
69
+ token_ids = list(range(sequence_len))
70
+ chunked_tokens = list(chunk_list(token_ids, block_size))
71
+ num_mutable_blocks_per_alloc = 0 if len(
72
+ chunked_tokens[-1]) == block_size else 1
73
+ num_immutable_blocks_per_alloc = len(
74
+ chunked_tokens) - num_mutable_blocks_per_alloc
75
+
76
+ block_tables = []
77
+ for alloc_i in range(1, 6):
78
+
79
+ block_tables.append(
80
+ BlockTable(
81
+ block_size=block_size,
82
+ block_allocator=allocator,
83
+ ))
84
+ block_tables[-1].allocate(token_ids=token_ids, device=Device.GPU)
85
+
86
+ # Expect all sequences to share allocations, except for their last block
87
+ # (which may be mutable).
88
+ assert allocator.get_num_free_blocks(
89
+ device=Device.GPU) == num_gpu_blocks - (
90
+ num_immutable_blocks_per_alloc + num_mutable_blocks_per_alloc *
91
+ (alloc_i))
92
+
93
+
94
+@pytest.mark.parametrize("block_size", [16])
95
+@pytest.mark.parametrize("sequence_len", [1, 16, 129])
96
+@pytest.mark.parametrize("allocator_type", ["naive", "prefix_caching"])
97
+@pytest.mark.parametrize("device", ["cpu", "gpu"])
98
+def test_allocate_free(block_size: int, sequence_len: int, allocator_type: str,
99
+ device: str):
100
+ """Test the allocation and freeing of blocks using different allocators and
101
+ devices.
102
+
103
+ This test creates a CpuGpuBlockAllocator with the specified block size,
104
+ number of blocks, allocator type, and device. It then allocates a BlockTable
105
+ multiple times with the same sequence and verifies that the number of free
106
+ blocks remains consistent after each allocation and freeing.
107
+ """
108
+ device = Device[device.upper()]
109
+
110
+ num_device_blocks = 1024
111
+ allocator = CpuGpuBlockAllocator.create(
112
+ allocator_type=allocator_type,
113
+ num_gpu_blocks=num_device_blocks,
114
+ num_cpu_blocks=num_device_blocks,
115
+ block_size=block_size,
116
+ )
117
+
118
+ token_ids = list(range(sequence_len))
119
+ num_blocks_per_alloc = len(list(chunk_list(token_ids, block_size)))
120
+
121
+ block_table = BlockTable(
122
+ block_size=block_size,
123
+ block_allocator=allocator,
124
+ )
125
+
126
+ for i in range(5):
127
+ block_table.allocate(token_ids=token_ids, device=device)
128
+ assert allocator.get_num_free_blocks(
129
+ device) == num_device_blocks - num_blocks_per_alloc
130
+ assert all(block_id is not None
131
+ for block_id in block_table.physical_block_ids)
132
+
133
+ block_table.free()
134
+ assert allocator.get_num_free_blocks(device) == num_device_blocks
135
+
136
+
137
+@pytest.mark.parametrize("block_size", [1, 8])
138
+@pytest.mark.parametrize("sequence_len", [1, 16, 129])
139
+@pytest.mark.parametrize("append_len", [1, 16, 129])
140
+@pytest.mark.parametrize("allocator_type", ["naive", "prefix_caching"])
141
+def test_append_token_ids_allocation(block_size: int, sequence_len: int,
142
+ append_len: int, allocator_type: str):
143
+ """Test the allocation behavior when appending token IDs to a BlockTable.
144
+
145
+ This test creates a CpuGpuBlockAllocator with the specified block size,
146
+ number of blocks, and allocator type. It then allocates a BlockTable with an
147
+ initial sequence and appends additional token IDs to it. The test verifies
148
+ that the number of allocated blocks before and after appending matches the
149
+ expected values.
150
+ """
151
+
152
+ num_gpu_blocks = 1024
153
+
154
+ allocator = CpuGpuBlockAllocator.create(
155
+ allocator_type=allocator_type,
156
+ num_gpu_blocks=num_gpu_blocks,
157
+ num_cpu_blocks=1024,
158
+ block_size=block_size,
159
+ )
160
+
161
+ token_ids = list(range(sequence_len))
162
+ token_ids_to_append = list(range(append_len))
163
+
164
+ block_table = BlockTable(
165
+ block_size=block_size,
166
+ block_allocator=allocator,
167
+ )
168
+
169
+ num_expected_blocks_before_append = len(
170
+ list(chunk_list(token_ids, block_size)))
171
+ num_expected_appended_blocks = len(
172
+ list(chunk_list(token_ids + token_ids_to_append,
173
+ block_size))) - num_expected_blocks_before_append
174
+
175
+ block_table.allocate(token_ids=token_ids, device=Device.GPU)
176
+
177
+ assert len(
178
+ block_table.physical_block_ids) == num_expected_blocks_before_append
179
+ block_table.append_token_ids(token_ids_to_append)
180
+ assert len(
181
+ block_table.physical_block_ids
182
+ ) == num_expected_blocks_before_append + num_expected_appended_blocks
183
+
184
+
185
+@pytest.mark.parametrize("block_size", [1, 8])
186
+@pytest.mark.parametrize("sequence_len", [1, 16, 129])
187
+@pytest.mark.parametrize("num_empty_slots", [1, 16, 129])
188
+@pytest.mark.parametrize("allocator_type", ["naive", "prefix_caching"])
189
+def test_ensure_num_empty_slots_allocation(block_size: int, sequence_len: int,
190
+ num_empty_slots: int,
191
+ allocator_type: str):
192
+ """Test the allocation behavior when ensuring a certain number of empty
193
+ slots in a BlockTable.
194
+
195
+ This test creates a CpuGpuBlockAllocator with the specified block size,
196
+ number of blocks, and allocator type. It then allocates a BlockTable with an
197
+ initial sequence and ensures a certain number of empty slots. The test
198
+ verifies that the number of allocated blocks before and after ensuring empty
199
+ slots matches the expected values. It also checks that filling up the empty
200
+ slots does not consume additional blocks.
201
+ """
202
+ num_gpu_blocks = 1024
203
+
204
+ allocator = CpuGpuBlockAllocator.create(
205
+ allocator_type=allocator_type,
206
+ num_gpu_blocks=num_gpu_blocks,
207
+ num_cpu_blocks=1024,
208
+ block_size=block_size,
209
+ )
210
+
211
+ token_ids = list(range(sequence_len))
212
+
213
+ block_table = BlockTable(
214
+ block_size=block_size,
215
+ block_allocator=allocator,
216
+ )
217
+
218
+ num_expected_blocks_before_append = len(
219
+ list(chunk_list(token_ids, block_size)))
220
+ num_expected_appended_blocks = len(
221
+ list(chunk_list(token_ids + [-1] * num_empty_slots,
222
+ block_size))) - num_expected_blocks_before_append
223
+
224
+ block_table.allocate(token_ids=token_ids, device=Device.GPU)
225
+
226
+ # Assert that the empty slots consume the expected number of additional
227
+ # blocks.
228
+ assert len(
229
+ block_table.physical_block_ids) == num_expected_blocks_before_append
230
+ block_table.ensure_num_empty_slots(num_empty_slots)
231
+ assert len(
232
+ block_table.physical_block_ids
233
+ ) == num_expected_blocks_before_append + num_expected_appended_blocks
234
+
235
+ # Now, ensure no additional blocks consumed as we fill up the empty slots.
236
+ num_free_blocks = allocator.get_num_free_blocks(device=Device.GPU)
237
+ block_table.append_token_ids(token_ids=list(range(num_empty_slots)))
238
+ assert num_free_blocks == allocator.get_num_free_blocks(device=Device.GPU)
239
+
240
+
241
+@pytest.mark.parametrize("block_size", [1, 8])
242
+@pytest.mark.parametrize("sequence_len", [1, 9])
243
+@pytest.mark.parametrize("append_len", [1, 16, 129])
244
+@pytest.mark.parametrize("append_size", [1, 4, 129])
245
+@pytest.mark.parametrize("allocator_type", ["naive", "prefix_caching"])
246
+def test_append_token_ids_correct_content(block_size: int, sequence_len: int,
247
+ append_len: int, allocator_type: str,
248
+ append_size: int):
249
+ """Verify token ids are correctly appended. Appends various amounts of
250
+ token ids in various append sizes, and verifies the final sequence is
251
+ correct.
252
+ """
253
+ num_gpu_blocks = 1024
254
+
255
+ allocator = CpuGpuBlockAllocator.create(
256
+ allocator_type=allocator_type,
257
+ num_gpu_blocks=num_gpu_blocks,
258
+ num_cpu_blocks=1024,
259
+ block_size=block_size,
260
+ )
261
+
262
+ token_ids = list(range(sequence_len))
263
+ token_ids_to_append = list(range(append_len))
264
+
265
+ block_table = BlockTable(
266
+ block_size=block_size,
267
+ block_allocator=allocator,
268
+ )
269
+ block_table.allocate(token_ids=token_ids, device=Device.GPU)
270
+
271
+ appended_so_far = []
272
+ for append in chunk_list(token_ids_to_append, append_size):
273
+ block_table.append_token_ids(append)
274
+ appended_so_far.extend(append)
275
+
276
+ assert block_table._get_all_token_ids() == token_ids + appended_so_far
277
+
278
+ assert block_table._get_all_token_ids() == token_ids + token_ids_to_append
279
+
280
+
281
+@pytest.mark.parametrize("seq_len", [1, 9, 129])
282
+@pytest.mark.parametrize("block_size", [1, 8])
283
+@pytest.mark.parametrize("allocator_type", ["naive", "prefix_caching"])
284
+def test_fork(seq_len: int, block_size: int, allocator_type: str):
285
+ """Create a sequence using the specified allocator.
286
+ 1. Assert that after forking the sequence, the free block count is the
287
+ same.
288
+ 2. Assert that the forked sequence has the same physical mappings.
289
+ 3. Then free the original sequence; verify that the free block count is
290
+ the same.
291
+ 4. Finally, free the forked sequence and verify that the free block
292
+ count drops to zero.
293
+ """
294
+ num_gpu_blocks = 1024
295
+
296
+ allocator = CpuGpuBlockAllocator.create(
297
+ allocator_type=allocator_type,
298
+ num_gpu_blocks=num_gpu_blocks,
299
+ num_cpu_blocks=0,
300
+ block_size=block_size,
301
+ )
302
+
303
+ token_ids = list(range(seq_len))
304
+
305
+ block_table = BlockTable(
306
+ block_size=block_size,
307
+ block_allocator=allocator,
308
+ )
309
+
310
+ block_table.allocate(token_ids)
311
+
312
+ num_free_blocks_before_fork = allocator.get_num_free_blocks(
313
+ device=Device.GPU)
314
+
315
+ forked_block_table = block_table.fork()
316
+
317
+ # Expect physical_block_ids and token_ids to match.
318
+ assert (block_table.physical_block_ids ==
319
+ forked_block_table.physical_block_ids)
320
+ assert block_table._get_all_token_ids(
321
+ ) == forked_block_table._get_all_token_ids()
322
+
323
+ # Do not expect any additional allocations.
324
+ assert allocator.get_num_free_blocks(
325
+ device=Device.GPU) == num_free_blocks_before_fork
326
+
327
+ # Free the original blocks. Assert num free blocks does not change, since
328
+ # refcount is nonzero.
329
+ block_table.free()
330
+ assert allocator.get_num_free_blocks(
331
+ device=Device.GPU) == num_free_blocks_before_fork
332
+
333
+ # Expect the forked block table to be unaffected by the free.
334
+ assert all(block_id is not None
335
+ for block_id in forked_block_table.physical_block_ids)
336
+
337
+ # Free the forked blocks. Assert num free blocks does change, since
338
+ # refcount is now zero.
339
+ forked_block_table.free()
340
+ assert allocator.get_num_free_blocks(device=Device.GPU) == num_gpu_blocks
341
+
342
+
343
+@pytest.mark.parametrize("block_size", [8])
344
+@pytest.mark.parametrize("sequence_len", [1, 16, 129])
345
+@pytest.mark.parametrize("append_len", [1, 16, 129])
346
+@pytest.mark.parametrize("appender", ["forked", "original"])
347
+@pytest.mark.parametrize("allocator_type", ["naive", "prefix_caching"])
348
+def test_cow(block_size: int, sequence_len: int, append_len: int,
349
+ allocator_type: str, appender: str):
350
+ """Fork a sequence; append to the forked sequence; verify there's a CoW.
351
+ """
352
+ num_gpu_blocks = 1024
353
+
354
+ allocator = CpuGpuBlockAllocator.create(
355
+ allocator_type=allocator_type,
356
+ num_gpu_blocks=num_gpu_blocks,
357
+ num_cpu_blocks=0,
358
+ block_size=block_size,
359
+ )
360
+
361
+ token_ids = list(range(sequence_len))
362
+ token_ids_to_append = list(range(append_len))
363
+
364
+ original_block_table = BlockTable(
365
+ block_size=block_size,
366
+ block_allocator=allocator,
367
+ )
368
+
369
+ num_expected_non_cow_blocks = cdiv(sequence_len, block_size)
370
+ num_expected_cow_blocks = cdiv(sequence_len + append_len,
371
+ block_size) - (sequence_len // block_size)
372
+
373
+ original_block_table.allocate(token_ids=token_ids, device=Device.GPU)
374
+ original_block_ids = original_block_table.physical_block_ids
375
+
376
+ forked_block_table = original_block_table.fork()
377
+
378
+ # Expect no additional allocation (copy on _write_).
379
+ assert allocator.get_num_free_blocks(
380
+ Device.GPU) == (num_gpu_blocks - num_expected_non_cow_blocks)
381
+
382
+ if appender == "forked":
383
+ appender_block_table = forked_block_table
384
+ static_block_table = original_block_table
385
+ elif appender == "original":
386
+ appender_block_table = original_block_table
387
+ static_block_table = forked_block_table
388
+ else:
389
+ raise ValueError(f"unknown test config {appender=}")
390
+
391
+ # Write tokens.
392
+ appender_block_table.append_token_ids(token_ids_to_append)
393
+
394
+ # Expect the non-appending block table to have no change.
395
+ assert static_block_table.physical_block_ids == original_block_ids
396
+ assert appender_block_table.physical_block_ids != original_block_ids
397
+
398
+ # Expect the blocks changed during append to have a CoW.
399
+ assert allocator.get_num_free_blocks(
400
+ Device.GPU) == num_gpu_blocks - (num_expected_non_cow_blocks +
401
+ num_expected_cow_blocks)
402
+
403
+ cows = allocator.clear_copy_on_writes()
404
+ if sequence_len % block_size > 0:
405
+ # If the last block in the sequence is not full, then when appending we
406
+ # expect a CoW.
407
+ assert cows
408
+
409
+ cow_block_id = sequence_len // block_size
410
+ expected_src = static_block_table.physical_block_ids[cow_block_id]
411
+ expected_dst = appender_block_table.physical_block_ids[cow_block_id]
412
+
413
+ assert expected_src in cows
414
+ assert expected_dst in cows[expected_src]
415
+ else:
416
+ # Otherwise, there should be no copy-on-write.
417
+ assert not cows
418
+
419
+ static_block_table.free()
420
+ appender_block_table.free()
421
+
422
+ # After free, expect all blocks to be freed.
423
+ assert allocator.get_num_free_blocks(Device.GPU) == num_gpu_blocks
424
+
425
+
426
+@pytest.mark.parametrize("block_size", [8])
427
+@pytest.mark.parametrize("sequence_len", [1, 16, 129])
428
+@pytest.mark.parametrize("append_len", [1, 16, 129])
429
+@pytest.mark.parametrize("lookahead_slots", [1, 16, 129])
430
+@pytest.mark.parametrize("appender", ["forked", "original"])
431
+@pytest.mark.parametrize("allocator_type", ["naive", "prefix_caching"])
432
+def test_cow_lookahead_simple(block_size: int, sequence_len: int,
433
+ append_len: int, lookahead_slots: int,
434
+ allocator_type: str, appender: str):
435
+ """Similar to test_cow, except with lookahead allocation. The assertions are
436
+ less rigorous due to the complexity of the property under test.
437
+ """
438
+ num_gpu_blocks = 1024
439
+
440
+ allocator = CpuGpuBlockAllocator.create(
441
+ allocator_type=allocator_type,
442
+ num_gpu_blocks=num_gpu_blocks,
443
+ num_cpu_blocks=0,
444
+ block_size=block_size,
445
+ )
446
+
447
+ token_ids = list(range(sequence_len))
448
+ token_ids_to_append = list(range(append_len))
449
+
450
+ original_block_table = BlockTable(
451
+ block_size=block_size,
452
+ block_allocator=allocator,
453
+ )
454
+
455
+ original_block_table.allocate(token_ids=token_ids, device=Device.GPU)
456
+
457
+ # Allocate lookahead slots.
458
+ original_block_table.ensure_num_empty_slots(lookahead_slots)
459
+ original_block_ids = original_block_table.physical_block_ids
460
+
461
+ forked_block_table = original_block_table.fork()
462
+
463
+ if appender == "forked":
464
+ appender_block_table = forked_block_table
465
+ static_block_table = original_block_table
466
+ elif appender == "original":
467
+ appender_block_table = original_block_table
468
+ static_block_table = forked_block_table
469
+ else:
470
+ raise ValueError(f"unknown test config {appender=}")
471
+
472
+ # Write tokens.
473
+ appender_block_table.append_token_ids(token_ids_to_append)
474
+
475
+ # Expect the non-appending block table to have no change.
476
+ assert static_block_table.physical_block_ids == original_block_ids
477
+ assert appender_block_table.physical_block_ids != original_block_ids
478
+
479
+ cows = allocator.clear_copy_on_writes()
480
+
481
+ # Always expect copy-on-write
482
+ assert cows
483
+
484
+ if sequence_len % block_size > 0:
485
+ # If the last block in the sequence is not full, then when appending we
486
+ # expect a CoW.
487
+ assert cows
488
+
489
+ cow_block_id = sequence_len // block_size
490
+ expected_src = static_block_table.physical_block_ids[cow_block_id]
491
+ expected_dst = appender_block_table.physical_block_ids[cow_block_id]
492
+
493
+ assert expected_src in cows
494
+ assert expected_dst in cows[expected_src]
495
+
496
+ static_block_table.free()
497
+ appender_block_table.free()
498
+
499
+ # After free, expect all blocks to be freed.
500
+ assert allocator.get_num_free_blocks(Device.GPU) == num_gpu_blocks
tests/core/block/test_common.py
new
+42
@@ -0,0 +1,42 @@
1
+import random
2
+
3
+import pytest
4
+
5
+from vllm.core.block.common import RefCounter
6
+
7
+
8
+@pytest.mark.parametrize("seed", list(range(20)))
9
+@pytest.mark.parametrize("num_incrs", [1, 100])
10
+@pytest.mark.parametrize("num_blocks", [1024])
11
+def test_incr(seed: int, num_incrs: int, num_blocks: int):
12
+ random.seed(seed)
13
+
14
+ all_block_indices = list(range(num_blocks))
15
+ counter = RefCounter(all_block_indices=all_block_indices)
16
+
17
+ block_id = random.randint(0, num_blocks - 1)
18
+ for i in range(num_incrs):
19
+ value = counter.incr(block_id)
20
+ assert value == i + 1
21
+
22
+
23
+@pytest.mark.parametrize("seed", list(range(20)))
24
+@pytest.mark.parametrize("num_incrs", [1, 100])
25
+@pytest.mark.parametrize("num_blocks", [1024])
26
+def test_incr_decr(seed: int, num_incrs: int, num_blocks: int):
27
+ random.seed(seed)
28
+
29
+ all_block_indices = list(range(num_blocks))
30
+ counter = RefCounter(all_block_indices=all_block_indices)
31
+
32
+ block_id = random.randint(0, num_blocks - 1)
33
+ for i in range(num_incrs):
34
+ value = counter.incr(block_id)
35
+ assert value == i + 1
36
+
37
+ for i in range(num_incrs):
38
+ value = counter.decr(block_id)
39
+ assert value == num_incrs - (i + 1)
40
+
41
+ with pytest.raises(AssertionError):
42
+ counter.decr(block_id)
tests/core/block/test_cpu_gpu_block_allocator.py
new
+93
@@ -0,0 +1,93 @@
1
+import pytest
2
+
3
+from vllm.core.block.cpu_gpu_block_allocator import CpuGpuBlockAllocator
4
+from vllm.utils import Device, chunk_list
5
+
6
+
7
+@pytest.mark.parametrize("num_cpu_blocks", [0, 512])
8
+@pytest.mark.parametrize("num_gpu_blocks", [1024])
9
+@pytest.mark.parametrize("block_size", [16])
10
+@pytest.mark.parametrize("allocator_type", ["naive", "prefix_caching"])
11
+def test_allocate_mutable(num_cpu_blocks: int, num_gpu_blocks: int,
12
+ block_size: int, allocator_type: str):
13
+ allocator = CpuGpuBlockAllocator.create(
14
+ allocator_type=allocator_type,
15
+ num_gpu_blocks=num_gpu_blocks,
16
+ num_cpu_blocks=num_cpu_blocks,
17
+ block_size=block_size,
18
+ )
19
+
20
+ assert allocator.get_num_free_blocks(Device.CPU) == num_cpu_blocks
21
+ assert allocator.get_num_free_blocks(Device.GPU) == num_gpu_blocks
22
+
23
+ cpu_blocks = [
24
+ allocator.allocate_mutable(prev_block=None, device=Device.CPU)
25
+ for _ in range(num_cpu_blocks)
26
+ ]
27
+ assert allocator.get_num_free_blocks(Device.CPU) == 0
28
+ assert allocator.get_num_free_blocks(Device.GPU) == num_gpu_blocks
29
+
30
+ gpu_blocks = [
31
+ allocator.allocate_mutable(prev_block=None, device=Device.GPU)
32
+ for _ in range(num_gpu_blocks)
33
+ ]
34
+ assert allocator.get_num_free_blocks(Device.CPU) == 0
35
+ assert allocator.get_num_free_blocks(Device.GPU) == 0
36
+
37
+ _ = [allocator.free(block) for block in cpu_blocks]
38
+ assert allocator.get_num_free_blocks(Device.CPU) == num_cpu_blocks
39
+ assert allocator.get_num_free_blocks(Device.GPU) == 0
40
+
41
+ _ = [allocator.free(block) for block in gpu_blocks]
42
+ assert allocator.get_num_free_blocks(Device.CPU) == num_cpu_blocks
43
+ assert allocator.get_num_free_blocks(Device.GPU) == num_gpu_blocks
44
+
45
+
46
+@pytest.mark.parametrize("num_cpu_blocks", [0, 512])
47
+@pytest.mark.parametrize("num_gpu_blocks", [1024])
48
+@pytest.mark.parametrize("block_size", [2])
49
+@pytest.mark.parametrize("allocator_type", ["naive", "prefix_caching"])
50
+def test_allocate_immutable(num_cpu_blocks: int, num_gpu_blocks: int,
51
+ block_size: int, allocator_type: str):
52
+ allocator = CpuGpuBlockAllocator.create(
53
+ allocator_type=allocator_type,
54
+ num_gpu_blocks=num_gpu_blocks,
55
+ num_cpu_blocks=num_cpu_blocks,
56
+ block_size=block_size,
57
+ )
58
+
59
+ unique_token_ids = list(
60
+ range((num_cpu_blocks + num_gpu_blocks) * block_size))
61
+ gpu_token_ids = chunk_list(unique_token_ids[:num_gpu_blocks * block_size],
62
+ block_size)
63
+ cpu_token_ids = chunk_list(unique_token_ids[num_gpu_blocks * block_size:],
64
+ block_size)
65
+
66
+ assert allocator.get_num_free_blocks(Device.CPU) == num_cpu_blocks
67
+ assert allocator.get_num_free_blocks(Device.GPU) == num_gpu_blocks
68
+
69
+ cpu_blocks = [
70
+ allocator.allocate_immutable(prev_block=None,
71
+ token_ids=token_ids,
72
+ device=Device.CPU)
73
+ for token_ids in cpu_token_ids
74
+ ]
75
+ assert allocator.get_num_free_blocks(Device.CPU) == 0
76
+ assert allocator.get_num_free_blocks(Device.GPU) == num_gpu_blocks
77
+
78
+ gpu_blocks = [
79
+ allocator.allocate_immutable(prev_block=None,
80
+ token_ids=token_ids,
81
+ device=Device.GPU)
82
+ for token_ids in gpu_token_ids
83
+ ]
84
+ assert allocator.get_num_free_blocks(Device.CPU) == 0
85
+ assert allocator.get_num_free_blocks(Device.GPU) == 0
86
+
87
+ _ = [allocator.free(block) for block in cpu_blocks]
88
+ assert allocator.get_num_free_blocks(Device.CPU) == num_cpu_blocks
89
+ assert allocator.get_num_free_blocks(Device.GPU) == 0
90
+
91
+ _ = [allocator.free(block) for block in gpu_blocks]
92
+ assert allocator.get_num_free_blocks(Device.CPU) == num_cpu_blocks
93
+ assert allocator.get_num_free_blocks(Device.GPU) == num_gpu_blocks
tests/core/block/test_naive_block.py
new
+102
@@ -0,0 +1,102 @@
1
+from typing import List, Optional
2
+
3
+import pytest
4
+
5
+from vllm.core.block.interfaces import Block, BlockAllocator
6
+from vllm.core.block.naive_block import NaiveBlock, NaiveBlockAllocator
7
+
8
+
9
+class TestNaiveBlockAllocator:
10
+
11
+ @staticmethod
12
+ def create_allocate_lambda(allocate_type: str,
13
+ allocator: NaiveBlockAllocator,
14
+ prev_block: Optional[Block],
15
+ token_ids: List[int]):
16
+ if allocate_type == "immutable":
17
+ allocate_block = lambda: allocator.allocate_immutable(
18
+ prev_block=prev_block, token_ids=token_ids)
19
+ elif allocate_type == "mutable":
20
+ allocate_block = lambda: allocator.allocate_mutable(prev_block=
21
+ prev_block)
22
+ else:
23
+ raise ValueError()
24
+
25
+ return allocate_block
26
+
27
+ @staticmethod
28
+ @pytest.mark.parametrize("allocate_type", ["immutable", "mutable"])
29
+ @pytest.mark.parametrize("num_blocks", [1, 1024])
30
+ @pytest.mark.parametrize("block_size", [1, 16])
31
+ def test_allocate_ooms(allocate_type: str, num_blocks: int,
32
+ block_size: int):
33
+ allocator = NaiveBlockAllocator(create_block=NaiveBlock,
34
+ num_blocks=num_blocks,
35
+ block_size=block_size)
36
+ allocate_block = TestNaiveBlockAllocator.create_allocate_lambda(
37
+ allocate_type,
38
+ allocator,
39
+ prev_block=None,
40
+ token_ids=list(range(block_size)))
41
+
42
+ [allocate_block() for _ in range(num_blocks)]
43
+ with pytest.raises(BlockAllocator.NoFreeBlocksError):
44
+ allocate_block()
45
+
46
+ @staticmethod
47
+ @pytest.mark.parametrize("allocate_type", ["immutable", "mutable"])
48
+ @pytest.mark.parametrize("num_blocks", [1, 1024])
49
+ @pytest.mark.parametrize("block_size", [1, 16])
50
+ def test_free_prevents_oom(allocate_type: str, num_blocks: int,
51
+ block_size: int):
52
+ allocator = NaiveBlockAllocator(create_block=NaiveBlock,
53
+ num_blocks=num_blocks,
54
+ block_size=block_size)
55
+ allocate_block = TestNaiveBlockAllocator.create_allocate_lambda(
56
+ allocate_type,
57
+ allocator,
58
+ prev_block=None,
59
+ token_ids=list(range(block_size)))
60
+
61
+ blocks = [allocate_block() for _ in range(num_blocks)]
62
+
63
+ with pytest.raises(BlockAllocator.NoFreeBlocksError):
64
+ allocate_block()
65
+
66
+ block_to_free = blocks.pop()
67
+
68
+ for _ in range(100):
69
+ block_id = block_to_free.block_id
70
+ allocator.free(block_to_free)
71
+ assert block_to_free.block_id is None
72
+
73
+ new_block = allocate_block()
74
+ assert new_block.block_id == block_id
75
+
76
+ with pytest.raises(BlockAllocator.NoFreeBlocksError):
77
+ allocate_block()
78
+
79
+ block_to_free = new_block
80
+
81
+ @staticmethod
82
+ @pytest.mark.parametrize("allocate_type", ["immutable", "mutable"])
83
+ @pytest.mark.parametrize("num_blocks", [1024])
84
+ @pytest.mark.parametrize("block_size", [16])
85
+ def test_get_num_free_blocks(allocate_type: str, num_blocks: int,
86
+ block_size: int):
87
+ allocator = NaiveBlockAllocator(create_block=NaiveBlock,
88
+ num_blocks=num_blocks,
89
+ block_size=block_size)
90
+ allocate_block = TestNaiveBlockAllocator.create_allocate_lambda(
91
+ allocate_type,
92
+ allocator,
93
+ prev_block=None,
94
+ token_ids=list(range(block_size)))
95
+
96
+ assert allocator.get_num_free_blocks() == num_blocks
97
+
98
+ blocks = [allocate_block() for _ in range(num_blocks)]
99
+
100
+ for i, block in enumerate(blocks):
101
+ assert allocator.get_num_free_blocks() == i
102
+ allocator.free(block)
tests/core/block/test_prefix_caching_block.py
new
+384
@@ -0,0 +1,384 @@
1
+import math
2
+import random
3
+from typing import List, Optional
4
+from unittest.mock import MagicMock
5
+
6
+import pytest
7
+
8
+from vllm.core.block.interfaces import Block, BlockAllocator
9
+from vllm.core.block.prefix_caching_block import (PrefixCachingBlock,
10
+ PrefixCachingBlockAllocator)
11
+
12
+
13
+class TestPrefixCachingBlock:
14
+
15
+ @staticmethod
16
+ @pytest.mark.parametrize("seed", list(range(10)))
17
+ @pytest.mark.parametrize("block_size", [1, 16])
18
+ @pytest.mark.parametrize("is_curr_block_full", [True, False])
19
+ def test_first_block_has_correct_content_hash(seed: int, block_size: int,
20
+ is_curr_block_full: bool):
21
+ """Verify a block which is first in the sequence has the correct hash.
22
+ """
23
+ random.seed(seed)
24
+ num_to_fill = block_size if is_curr_block_full else random.randint(
25
+ 0, block_size - 1)
26
+ token_ids = list(range(num_to_fill))
27
+ mock_allocator = MagicMock(spec=PrefixCachingBlockAllocator)
28
+
29
+ block_with_prev = PrefixCachingBlock(
30
+ prev_block=None,
31
+ token_ids=token_ids,
32
+ block_size=block_size,
33
+ prefix_caching_allocator=mock_allocator)
34
+
35
+ if is_curr_block_full:
36
+ # Expect hash since block is full.
37
+ assert block_with_prev.content_hash == (
38
+ PrefixCachingBlock.hash_block_tokens(
39
+ is_first_block=True,
40
+ prev_block_hash=None,
41
+ cur_block_token_ids=token_ids))
42
+ else:
43
+ # Do not expect hash since block is not full.
44
+ assert block_with_prev.content_hash is None
45
+
46
+ @staticmethod
47
+ @pytest.mark.parametrize("seed", list(range(10)))
48
+ @pytest.mark.parametrize("block_size", [1, 16])
49
+ @pytest.mark.parametrize("is_curr_block_full", [True, False])
50
+ @pytest.mark.parametrize("prev_block_has_hash", [True, False])
51
+ def test_nth_block_has_correct_content_hash(seed: int, block_size: int,
52
+ is_curr_block_full: bool,
53
+ prev_block_has_hash: bool):
54
+ """Verify a block which is not first in the sequence has the correct
55
+ hash.
56
+ """
57
+
58
+ random.seed(seed)
59
+
60
+ previous_block = MagicMock(spec=PrefixCachingBlock)
61
+ prev_block_hash = random.randint(0, 1000)
62
+ previous_block.content_hash = (prev_block_hash
63
+ if prev_block_has_hash else None)
64
+
65
+ num_to_fill = block_size if is_curr_block_full else random.randint(
66
+ 0, block_size - 1)
67
+ token_ids = list(range(num_to_fill))
68
+ mock_allocator = MagicMock(spec=PrefixCachingBlockAllocator)
69
+
70
+ block_with_prev = PrefixCachingBlock(
71
+ prev_block=previous_block,
72
+ token_ids=token_ids,
73
+ block_size=block_size,
74
+ prefix_caching_allocator=mock_allocator,
75
+ )
76
+
77
+ if is_curr_block_full and prev_block_has_hash:
78
+ # Expect hash since block is full and previous block has hash.
79
+ assert (block_with_prev.content_hash ==
80
+ PrefixCachingBlock.hash_block_tokens(
81
+ is_first_block=False,
82
+ prev_block_hash=prev_block_hash,
83
+ cur_block_token_ids=token_ids))
84
+ else:
85
+ # Do not expect hash since block is not full or the previous block
86
+ # does not have a hash.
87
+ assert block_with_prev.content_hash is None
88
+
89
+ @staticmethod
90
+ @pytest.mark.parametrize("block_size", [1, 2, 16])
91
+ @pytest.mark.parametrize("num_tokens", list(range(3)))
92
+ @pytest.mark.parametrize("num_empty_trailing_blocks", [0, 1, 10])
93
+ def test_blocks_have_correct_hash_in_chain(block_size: int,
94
+ num_tokens: int,
95
+ num_empty_trailing_blocks: int):
96
+ """Create two chains of logical blocks with the same contents.
97
+ Assert the hashes are equal.
98
+ """
99
+ random.seed(0)
100
+
101
+ token_ids = [random.randint(0, 50_000) for _ in range(num_tokens)]
102
+
103
+ first_chain, second_chain = [
104
+ TestPrefixCachingBlock.create_chain(
105
+ block_size=block_size,
106
+ token_ids=token_ids,
107
+ num_empty_trailing_blocks=num_empty_trailing_blocks)
108
+ for _ in range(2)
109
+ ]
110
+
111
+ for first_chain_block, second_chain_block in zip(
112
+ first_chain, second_chain):
113
+ assert (first_chain_block.content_hash ==
114
+ second_chain_block.content_hash)
115
+
116
+ if not first_chain or not second_chain:
117
+ assert first_chain == second_chain
118
+ assert num_tokens == 0
119
+
120
+ @staticmethod
121
+ def create_chain(block_size: int,
122
+ token_ids: List[int],
123
+ num_empty_trailing_blocks=0) -> List[PrefixCachingBlock]:
124
+ """Helper method which creates a chain of blocks.
125
+ """
126
+ blocks = []
127
+ num_blocks = math.ceil(
128
+ len(token_ids) / block_size) + num_empty_trailing_blocks
129
+
130
+ if num_blocks == 0:
131
+ return []
132
+
133
+ allocator = MagicMock(spec=PrefixCachingBlockAllocator)
134
+
135
+ prev_block = None
136
+ for block_number in range(0, num_blocks):
137
+ prev_block = PrefixCachingBlock(
138
+ prev_block=prev_block,
139
+ token_ids=[],
140
+ block_size=block_size,
141
+ prefix_caching_allocator=allocator,
142
+ )
143
+
144
+ tokens_to_append = token_ids[block_number *
145
+ block_size:(block_number + 1) *
146
+ block_size]
147
+ if tokens_to_append:
148
+ prev_block.append_token_ids(tokens_to_append)
149
+
150
+ blocks.append(prev_block)
151
+
152
+ return blocks
153
+
154
+
155
+class TestPrefixCachingBlockAllocator:
156
+
157
+ @staticmethod
158
+ def create_allocate_lambda(allocate_type: str, allocator: BlockAllocator,
159
+ prev_block: Optional[Block],
160
+ token_ids: List[int]):
161
+ if allocate_type == "immutable":
162
+ allocate_block = lambda: allocator.allocate_immutable(
163
+ prev_block=prev_block, token_ids=token_ids)
164
+ elif allocate_type == "mutable":
165
+ allocate_block = lambda: allocator.allocate_mutable(prev_block=
166
+ prev_block)
167
+ else:
168
+ raise ValueError()
169
+
170
+ return allocate_block
171
+
172
+ @staticmethod
173
+ @pytest.mark.parametrize("num_blocks", [1, 1024])
174
+ @pytest.mark.parametrize("block_size", [1, 16])
175
+ def test_allocate_mutable_ooms(num_blocks: int, block_size: int):
176
+ allocator = PrefixCachingBlockAllocator(num_blocks=num_blocks,
177
+ block_size=block_size)
178
+ allocate_block = TestPrefixCachingBlockAllocator.create_allocate_lambda(
179
+ allocate_type="mutable",
180
+ allocator=allocator,
181
+ prev_block=None,
182
+ token_ids=list(range(block_size)),
183
+ )
184
+
185
+ [allocate_block() for _ in range(num_blocks)]
186
+ with pytest.raises(BlockAllocator.NoFreeBlocksError):
187
+ allocate_block()
188
+
189
+ @staticmethod
190
+ @pytest.mark.parametrize("num_blocks", [1, 1024])
191
+ @pytest.mark.parametrize("block_size", [1, 16])
192
+ def test_allocate_immutable_does_not_oom_single_hash(
193
+ num_blocks: int, block_size: int):
194
+ allocator = PrefixCachingBlockAllocator(num_blocks=num_blocks,
195
+ block_size=block_size)
196
+ allocate_block = TestPrefixCachingBlockAllocator.create_allocate_lambda(
197
+ allocate_type="immutable",
198
+ allocator=allocator,
199
+ prev_block=None,
200
+ token_ids=list(range(block_size)),
201
+ )
202
+
203
+ blocks = [allocate_block() for _ in range(num_blocks)]
204
+
205
+ # Expect no OOM. If these were mutable blocks, this would OOM.
206
+ non_oom_block = allocate_block()
207
+
208
+ # Expect all blocks to have same physical block index.
209
+ for block in blocks:
210
+ assert (block.block_id == non_oom_block.block_id)
211
+
212
+ @staticmethod
213
+ @pytest.mark.parametrize("num_blocks", [1, 1024])
214
+ @pytest.mark.parametrize("block_size", [1, 16])
215
+ def test_allocate_immutable_ooms_many_hash(num_blocks: int,
216
+ block_size: int):
217
+ """Consume all blocks using many different hashes/block content.
218
+
219
+ Do this by creating a sequence that is very long.
220
+ Expect next block to OOM.
221
+ """
222
+ allocator = PrefixCachingBlockAllocator(num_blocks=num_blocks,
223
+ block_size=block_size)
224
+
225
+ # Create token ids that will exhaust all blocks.
226
+ token_ids = list(range(num_blocks * block_size))
227
+
228
+ chain = TestPrefixCachingBlockAllocator.create_immutable_chain(
229
+ block_size=block_size,
230
+ token_ids=token_ids,
231
+ allocator=allocator,
232
+ )
233
+
234
+ # Expect allocation with unseen hash to fail.
235
+ with pytest.raises(BlockAllocator.NoFreeBlocksError):
236
+ allocator.allocate_immutable(prev_block=chain[-1],
237
+ token_ids=list(range(block_size)))
238
+
239
+ # Expect mutable allocation to fail.
240
+ with pytest.raises(BlockAllocator.NoFreeBlocksError):
241
+ allocator.allocate_mutable(prev_block=chain[-1])
242
+
243
+ # Expect allocation of exact same chain to pass.
244
+ second_chain = TestPrefixCachingBlockAllocator.create_immutable_chain(
245
+ block_size=block_size,
246
+ token_ids=token_ids,
247
+ allocator=allocator,
248
+ )
249
+
250
+ # Expect physical block indices to be the same in both chains.
251
+ assert chain and second_chain
252
+ for first_chain_block, second_chain_block in zip(chain, second_chain):
253
+ assert (first_chain_block.block_id == second_chain_block.block_id)
254
+
255
+ @staticmethod
256
+ @pytest.mark.parametrize("num_blocks", [1, 1024])
257
+ @pytest.mark.parametrize("block_size", [1, 16])
258
+ def test_free_prevents_oom(num_blocks: int, block_size: int):
259
+ allocator = PrefixCachingBlockAllocator(num_blocks=num_blocks,
260
+ block_size=block_size)
261
+
262
+ # Create token ids that will exhaust all blocks.
263
+ token_ids = list(range(num_blocks * block_size))
264
+
265
+ chain = TestPrefixCachingBlockAllocator.create_immutable_chain(
266
+ block_size=block_size,
267
+ token_ids=token_ids,
268
+ allocator=allocator,
269
+ )
270
+
271
+ # Expect mutable allocation to fail.
272
+ with pytest.raises(BlockAllocator.NoFreeBlocksError):
273
+ allocator.allocate_mutable(prev_block=None)
274
+
275
+ block_to_free = chain[-1]
276
+
277
+ # Expect free/allocate loop to succeed many times.
278
+ for i in range(100):
279
+ block_id = block_to_free.block_id
280
+ allocator.free(block_to_free)
281
+ assert block_to_free.block_id is None, i
282
+
283
+ new_block = allocator.allocate_mutable(prev_block=None)
284
+ assert new_block.block_id == block_id, i
285
+
286
+ with pytest.raises(BlockAllocator.NoFreeBlocksError):
287
+ allocator.allocate_mutable(prev_block=None)
288
+
289
+ block_to_free = new_block
290
+
291
+ @staticmethod
292
+ @pytest.mark.parametrize("num_blocks", [1024])
293
+ @pytest.mark.parametrize("block_size", [16])
294
+ @pytest.mark.parametrize("seed", list(range(20)))
295
+ def test_get_num_free_blocks(num_blocks: int, block_size: int, seed: int):
296
+ random.seed(seed)
297
+ allocator = PrefixCachingBlockAllocator(num_blocks=num_blocks,
298
+ block_size=block_size)
299
+ num_blocks_to_consume = random.randint(1, num_blocks - 1)
300
+
301
+ # Create token ids that will exhaust all blocks.
302
+ token_ids = list(range(num_blocks_to_consume * block_size))
303
+
304
+ chain = TestPrefixCachingBlockAllocator.create_immutable_chain(
305
+ block_size=block_size,
306
+ token_ids=token_ids,
307
+ allocator=allocator,
308
+ )
309
+
310
+ # Free each block in chain, assert num free blocks includes new free
311
+ # block.
312
+ for i, block in enumerate(chain):
313
+ assert allocator.get_num_free_blocks() == (num_blocks -
314
+ num_blocks_to_consume +
315
+ i)
316
+ allocator.free(block)
317
+
318
+ @staticmethod
319
+ @pytest.mark.parametrize("num_blocks", [1024])
320
+ @pytest.mark.parametrize("block_size", [16])
321
+ @pytest.mark.parametrize("seed", list(range(20)))
322
+ def test_get_num_free_blocks_shared(num_blocks: int, block_size: int,
323
+ seed: int):
324
+ """Verify sharing occurs by allocating two sequences that share prefixes
325
+ and incrementally freeing blocks.
326
+ """
327
+ random.seed(seed)
328
+ allocator = PrefixCachingBlockAllocator(num_blocks=num_blocks,
329
+ block_size=block_size)
330
+ num_blocks_to_consume = random.randint(1, num_blocks - 1)
331
+
332
+ # Create token ids that will exhaust all blocks.
333
+ token_ids = list(range(num_blocks_to_consume * block_size))
334
+
335
+ first_chain = TestPrefixCachingBlockAllocator.create_immutable_chain(
336
+ block_size=block_size,
337
+ token_ids=token_ids,
338
+ allocator=allocator,
339
+ )
340
+ second_chain = TestPrefixCachingBlockAllocator.create_immutable_chain(
341
+ block_size=block_size,
342
+ token_ids=token_ids,
343
+ allocator=allocator,
344
+ )
345
+
346
+ # Free each block in the first chain. Since all blocks are shared, the
347
+ # free count should stay constant.
348
+ for i, block in enumerate(first_chain):
349
+ assert allocator.get_num_free_blocks() == (num_blocks -
350
+ num_blocks_to_consume)
351
+ allocator.free(block)
352
+
353
+ # Free each block in the second chain. Since the refcount is now zero,
354
+ # the free count should increment with each free.
355
+ for i, block in enumerate(second_chain):
356
+ assert allocator.get_num_free_blocks() == (num_blocks -
357
+ num_blocks_to_consume +
358
+ i)
359
+ allocator.free(block)
360
+
361
+ @staticmethod
362
+ def create_immutable_chain(
363
+ block_size: int,
364
+ token_ids: List[int],
365
+ allocator: PrefixCachingBlockAllocator,
366
+ ) -> List[PrefixCachingBlock]:
367
+ """Helper method which creates a chain of blocks.
368
+ """
369
+ blocks = []
370
+ num_blocks = math.ceil(len(token_ids) / block_size)
371
+
372
+ if num_blocks == 0:
373
+ return []
374
+
375
+ prev_block = None
376
+ for block_number in range(0, num_blocks):
377
+ block_token_ids = token_ids[block_number *
378
+ block_size:(block_number + 1) *
379
+ block_size]
380
+ prev_block = allocator.allocate_immutable(
381
+ prev_block=prev_block, token_ids=block_token_ids)
382
+ blocks.append(prev_block)
383
+
384
+ return blocks
tests/core/test_block_manager.py
+40
-39
@@ -5,8 +5,9 @@ import pytest
5
6
from vllm import SamplingParams
7
from vllm.block import PhysicalTokenBlock
8
-from vllm.core.block_manager import (AllocStatus, BlockSpaceManager,
9
- UncachedBlockAllocator)
8
+from vllm.core.block_manager_v1 import (BlockSpaceManagerV1,
9
+ UncachedBlockAllocator)
10
+from vllm.core.interfaces import AllocStatus
11
from vllm.sequence import Logprob, Sequence, SequenceGroup, SequenceStatus
12
from vllm.utils import Device
13
@@ -63,10 +64,10 @@ def test_allocate():
64
block_size = 4
65
num_cpu_blocks = 4
66
num_gpu_blocks = 4
66
- block_manager = BlockSpaceManager(block_size,
67
- num_cpu_blocks,
68
- num_gpu_blocks,
69
- watermark=0)
67
+ block_manager = BlockSpaceManagerV1(block_size,
68
+ num_cpu_blocks,
69
+ num_gpu_blocks,
70
+ watermark=0)
71
72
# Allocate same sequence group to all available gpu blocks.
73
for i in range(num_gpu_blocks):
@@ -77,10 +78,10 @@ def test_allocate():
78
79
# Allocate same sequence group to all available gpu blocks.
80
# Use watermark to reserve one gpu block.
80
- block_manager = BlockSpaceManager(block_size,
81
- num_cpu_blocks,
82
- num_gpu_blocks,
83
- watermark=1 / num_gpu_blocks)
81
+ block_manager = BlockSpaceManagerV1(block_size,
82
+ num_cpu_blocks,
83
+ num_gpu_blocks,
84
+ watermark=1 / num_gpu_blocks)
85
for i in range(num_gpu_blocks - 1):
86
_, seq_group = create_dummy_prompt(str(i), block_size)
87
assert block_manager.can_allocate(seq_group)
@@ -92,10 +93,10 @@ def test_append_slot_single_seq():
93
block_size = 4
94
num_cpu_blocks = 4
95
num_gpu_blocks = 4
95
- block_manager = BlockSpaceManager(block_size,
96
- num_cpu_blocks,
97
- num_gpu_blocks,
98
- watermark=0)
96
+ block_manager = BlockSpaceManagerV1(block_size,
97
+ num_cpu_blocks,
98
+ num_gpu_blocks,
99
+ watermark=0)
100
101
# Allocate single seq to gpu block.
102
prompt, seq_group = create_dummy_prompt("1", block_size)
@@ -124,10 +125,10 @@ def test_append_slot_cow():
125
block_size = 4
126
num_cpu_blocks = 4
127
num_gpu_blocks = 4
127
- block_manager = BlockSpaceManager(block_size=block_size,
128
- num_cpu_blocks=num_cpu_blocks,
129
- num_gpu_blocks=num_gpu_blocks,
130
- watermark=0)
128
+ block_manager = BlockSpaceManagerV1(block_size=block_size,
129
+ num_cpu_blocks=num_cpu_blocks,
130
+ num_gpu_blocks=num_gpu_blocks,
131
+ watermark=0)
132
133
# Allocate prompt to gpu block. There is one slot left in the block.
134
prompt = Sequence(seq_id=1,
@@ -165,10 +166,10 @@ def test_fork():
166
block_size = 4
167
num_cpu_blocks = 4
168
num_gpu_blocks = 4
168
- block_manager = BlockSpaceManager(block_size,
169
- num_cpu_blocks,
170
- num_gpu_blocks,
171
- watermark=0)
169
+ block_manager = BlockSpaceManagerV1(block_size,
170
+ num_cpu_blocks,
171
+ num_gpu_blocks,
172
+ watermark=0)
173
174
prompt, seq_group = create_dummy_prompt("1",
175
block_size - 1,
@@ -192,10 +193,10 @@ def test_swap():
193
block_size = 4
194
num_cpu_blocks = 4
195
num_gpu_blocks = 4
195
- block_manager = BlockSpaceManager(block_size,
196
- num_cpu_blocks,
197
- num_gpu_blocks,
198
- watermark=0)
196
+ block_manager = BlockSpaceManagerV1(block_size,
197
+ num_cpu_blocks,
198
+ num_gpu_blocks,
199
+ watermark=0)
200
201
prompt, seq_group = create_dummy_prompt("1", prompt_length=block_size - 1)
202
prompt.status = SequenceStatus.WAITING
@@ -238,10 +239,10 @@ def test_free():
239
block_size = 4
240
num_cpu_blocks = 4
241
num_gpu_blocks = 4
241
- block_manager = BlockSpaceManager(block_size,
242
- num_cpu_blocks,
243
- num_gpu_blocks,
244
- watermark=0)
242
+ block_manager = BlockSpaceManagerV1(block_size,
243
+ num_cpu_blocks,
244
+ num_gpu_blocks,
245
+ watermark=0)
246
247
prompt, seq_group = create_dummy_prompt("1", block_size)
248
block_manager.allocate(seq_group)
@@ -262,10 +263,10 @@ def test_reset():
263
block_size = 4
264
num_cpu_blocks = 4
265
num_gpu_blocks = 4
265
- block_manager = BlockSpaceManager(block_size,
266
- num_cpu_blocks,
267
- num_gpu_blocks,
268
- watermark=0)
266
+ block_manager = BlockSpaceManagerV1(block_size,
267
+ num_cpu_blocks,
268
+ num_gpu_blocks,
269
+ watermark=0)
270
271
# Allocate same seq group on all available gpu blocks.
272
original_blocks = block_manager.get_num_free_gpu_blocks()
@@ -289,11 +290,11 @@ def test_sliding_window_multi_seq():
290
num_cpu_blocks = 8
291
num_gpu_blocks = 8
292
sliding_window = 2
292
- block_manager = BlockSpaceManager(block_size,
293
- num_cpu_blocks,
294
- num_gpu_blocks,
295
- sliding_window=sliding_window,
296
- watermark=0)
293
+ block_manager = BlockSpaceManagerV1(block_size,
294
+ num_cpu_blocks,
295
+ num_gpu_blocks,
296
+ sliding_window=sliding_window,
297
+ watermark=0)
298
299
assert block_manager.get_num_free_gpu_blocks() == num_gpu_blocks
300
tests/core/test_scheduler.py
+1
-1
@@ -13,7 +13,7 @@ from .utils import create_dummy_prompt
13
def test_scheduler_add_seq_group():
14
block_size = 4
15
scheduler_config = SchedulerConfig(100, 64, 1)
16
- cache_config = CacheConfig(block_size, 1.0, 1, "auto")
16
+ cache_config = CacheConfig(block_size, 1.0, 1, cache_dtype="auto")
17
cache_config.num_cpu_blocks = 4
18
cache_config.num_gpu_blocks = 4
19
scheduler = Scheduler(scheduler_config, cache_config, None)
tests/core/utils.py
+38
-1
@@ -2,7 +2,7 @@ import time
2
from typing import Tuple
3
4
from vllm import SamplingParams
5
-from vllm.sequence import Sequence, SequenceGroup
5
+from vllm.sequence import Logprob, Sequence, SequenceGroup
6
7
8
def create_dummy_prompt(
@@ -23,5 +23,42 @@ def create_dummy_prompt(
23
return prompt, seq_group
24
25
26
+def create_seq_group(
27
+ seq_prompt_lens=1024,
28
+ seq_output_lens=(128, ),
29
+ request_id='0',
30
+ seq_id_start=0,
31
+) -> SequenceGroup:
32
+
33
+ assert len(seq_output_lens) > 0
34
+
35
+ prompt_token_ids = [0] * seq_prompt_lens
36
+
37
+ seqs = []
38
+ for seq_id_offset, output_len in enumerate(seq_output_lens):
39
+ seq = Sequence(
40
+ seq_id=seq_id_start + seq_id_offset,
41
+ prompt="",
42
+ prompt_token_ids=prompt_token_ids,
43
+ block_size=16,
44
+ )
45
+
46
+ for i in range(output_len):
47
+ seq.append_token_id(
48
+ token_id=i,
49
+ logprobs={i: Logprob(0.0)},
50
+ )
51
+ seqs.append(seq)
52
+
53
+ seq_group = SequenceGroup(
54
+ request_id=request_id,
55
+ seqs=seqs,
56
+ sampling_params=SamplingParams(),
57
+ arrival_time=time.time(),
58
+ )
59
+
60
+ return seq_group
61
+
62
+
63
def round_up_to_next_block(seq_len: int, block_size: int) -> int:
64
return (seq_len + block_size - 1) // block_size
tests/prefix_caching/test_prefix_caching.py
+1
-1
@@ -4,7 +4,7 @@ Run `pytest tests/prefix_caching/test_prefix_caching.py`.
4
"""
5
import pytest
6
7
-from vllm.core.block_manager import CachedBlockAllocator
7
+from vllm.core.block_manager_v1 import CachedBlockAllocator
8
from vllm.utils import Device
9
10
vllm/config.py
+7
@@ -324,6 +324,8 @@ class CacheConfig:
324
vLLM execution.
325
swap_space: Size of the CPU swap space per GPU (in GiB).
326
cache_dtype: Data type for kv cache storage.
327
+ forced_num_gpu_blocks: Number of GPU blocks to use. This overrides the
328
+ profiled num_gpu_blocks if specified. Does nothing if None.
329
"""
330
331
def __init__(
@@ -332,12 +334,14 @@ class CacheConfig:
334
gpu_memory_utilization: float,
335
swap_space: int,
336
cache_dtype: str,
337
+ forced_num_gpu_blocks: Optional[int] = None,
338
sliding_window: Optional[int] = None,
339
enable_prefix_caching: bool = False,
340
) -> None:
341
self.block_size = block_size
342
self.gpu_memory_utilization = gpu_memory_utilization
343
self.swap_space_bytes = swap_space * _GB
344
+ self.forced_num_gpu_blocks = forced_num_gpu_blocks
345
self.cache_dtype = cache_dtype
346
self.sliding_window = sliding_window
347
self.enable_prefix_caching = enable_prefix_caching
@@ -528,6 +532,7 @@ class SchedulerConfig:
532
and generated text).
533
delay_factor: Apply a delay (of delay factor multiplied by previous
534
prompt latency) before scheduling next prompt.
535
+ use_v2_block_manager: Whether to use the BlockSpaceManagerV2 or not.
536
"""
537
538
def __init__(
@@ -535,6 +540,7 @@ class SchedulerConfig:
540
max_num_batched_tokens: Optional[int],
541
max_num_seqs: int,
542
max_model_len: int,
543
+ use_v2_block_manager: bool = False,
544
delay_factor: float = 0.0,
545
) -> None:
546
if max_num_batched_tokens is not None:
@@ -546,6 +552,7 @@ class SchedulerConfig:
552
self.max_num_seqs = max_num_seqs
553
self.max_model_len = max_model_len
554
self.delay_factor = delay_factor
555
+ self.use_v2_block_manager = use_v2_block_manager
556
self._verify_args()
557
558
def _verify_args(self) -> None:
vllm/core/block/block_table.py
new
+245
@@ -0,0 +1,245 @@
1
+from typing import List, Optional
2
+
3
+from vllm.core.block.interfaces import Block, DeviceAwareBlockAllocator
4
+from vllm.utils import Device, cdiv, chunk_list
5
+
6
+
7
+class BlockTable:
8
+ """A class to manage blocks for a specific sequence.
9
+
10
+ The BlockTable maps a sequence of tokens to a list of blocks, where each
11
+ block represents a contiguous memory allocation for a portion of the
12
+ sequence. The blocks are managed by a DeviceAwareBlockAllocator, which is
13
+ responsible for allocating and freeing memory for the blocks.
14
+
15
+ Args:
16
+ block_size (int): The maximum number of tokens that can be stored in a
17
+ single block.
18
+ block_allocator (DeviceAwareBlockAllocator): The block allocator used to
19
+ manage memory for the blocks.
20
+ _blocks (Optional[List[Block]], optional): An optional list of existing
21
+ blocks to initialize the BlockTable with. If not provided, an empty
22
+ BlockTable is created.
23
+
24
+ Attributes:
25
+ _block_size (int): The maximum number of tokens that can be stored in a
26
+ single block.
27
+ _allocator (DeviceAwareBlockAllocator): The block allocator used to
28
+ manage memory for the blocks.
29
+ _blocks (Optional[List[Block]]): The list of blocks managed by this
30
+ BlockTable.
31
+ _num_full_slots (int): The number of tokens currently stored in the
32
+ blocks.
33
+ """
34
+
35
+ def __init__(
36
+ self,
37
+ block_size: int,
38
+ block_allocator: DeviceAwareBlockAllocator,
39
+ _blocks: Optional[List[Block]] = None,
40
+ ):
41
+ self._block_size = block_size
42
+ self._allocator = block_allocator
43
+ self._blocks: Optional[List[Block]] = _blocks
44
+
45
+ # Use helper method instead of directly calculating, as blocks
46
+ # may not be allocated.
47
+ self._num_full_slots = len(self._get_all_token_ids())
48
+
49
+ @staticmethod
50
+ def get_num_required_blocks(token_ids: List[int], block_size: int) -> int:
51
+ """Calculates the minimum number of blocks required to store a given
52
+ sequence of token IDs.
53
+
54
+ This assumes worst-case scenario, where every block requires a new
55
+ allocation (e.g. ignoring prefix caching).
56
+
57
+ Args:
58
+ token_ids (List[int]): The sequence of token IDs to be stored.
59
+ block_size (int): The maximum number of tokens that can be stored in
60
+ a single block.
61
+
62
+ Returns:
63
+ int: The minimum number of blocks required to store the given
64
+ sequence of token IDs.
65
+ """
66
+ return cdiv(len(token_ids), block_size)
67
+
68
+ def allocate(self,
69
+ token_ids: List[int],
70
+ device: Device = Device.GPU) -> None:
71
+ """Allocates memory blocks for storing the given sequence of token IDs.
72
+
73
+ This method allocates the required number of blocks to store the given
74
+ sequence of token IDs.
75
+
76
+ Args:
77
+ token_ids (List[int]): The sequence of token IDs to be stored.
78
+ device (Device, optional): The device on which the blocks should be
79
+ allocated. Defaults to Device.GPU.
80
+ """
81
+ assert not self._is_allocated
82
+ assert token_ids
83
+ self._blocks = self._allocate_blocks_for_token_ids(prev_block=None,
84
+ token_ids=token_ids,
85
+ device=device)
86
+ self._num_full_slots = len(token_ids)
87
+
88
+ def append_token_ids(self, token_ids: List[int]) -> None:
89
+ """Appends a sequence of token IDs to the existing blocks in the
90
+ BlockTable.
91
+
92
+ This method appends the given sequence of token IDs to the existing
93
+ blocks in the BlockTable. If there is not enough space in the existing
94
+ blocks, new blocks are allocated using the `ensure_num_empty_slots`
95
+ method to accommodate the additional tokens.
96
+
97
+ The token IDs are divided into chunks of size `block_size` (except for
98
+ the first chunk, which may be smaller), and each chunk is appended to a
99
+ separate block.
100
+
101
+ Args:
102
+ token_ids (List[int]): The sequence of token IDs to be appended.
103
+ """
104
+ assert self._is_allocated
105
+
106
+ self.ensure_num_empty_slots(num_empty_slots=len(token_ids))
107
+
108
+ blocks = self._blocks[self._num_full_slots // self._block_size:]
109
+ first_chunk_size = self._block_size - (self._num_full_slots %
110
+ self._block_size)
111
+ token_blocks = [token_ids[:first_chunk_size]] + chunk_list(
112
+ token_ids[first_chunk_size:], self._block_size)
113
+
114
+ for block, token_block in zip(blocks, token_blocks):
115
+ block.append_token_ids(token_block)
116
+
117
+ self._num_full_slots += len(token_ids)
118
+
119
+ def ensure_num_empty_slots(self, num_empty_slots: int) -> None:
120
+ """Ensures that the BlockTable has at least the specified number of
121
+ empty slots available.
122
+
123
+ This method checks if the BlockTable has enough empty slots (i.e.,
124
+ available space) to accommodate the requested number of tokens. If not,
125
+ it allocates additional blocks on the GPU to ensure that the required
126
+ number of empty slots is available.
127
+
128
+ Args:
129
+ num_empty_slots (int): The minimum number of empty slots required.
130
+ """
131
+ # Currently the block table only supports
132
+ # appending tokens to GPU blocks.
133
+ device = Device.GPU
134
+ assert self._is_allocated
135
+
136
+ if self._num_empty_slots >= num_empty_slots:
137
+ return
138
+
139
+ slots_to_allocate = num_empty_slots - self._num_empty_slots
140
+ blocks_to_allocate = cdiv(slots_to_allocate, self._block_size)
141
+
142
+ for _ in range(blocks_to_allocate):
143
+ self._blocks.append(
144
+ self._allocator.allocate_mutable(prev_block=self._blocks[-1],
145
+ device=device))
146
+
147
+ def fork(self) -> "BlockTable":
148
+ """Creates a new BlockTable instance with a copy of the blocks from the
149
+ current instance.
150
+
151
+ This method creates a new BlockTable instance with the same block size,
152
+ block allocator, and a copy of the blocks from the current instance. The
153
+ new BlockTable has its own independent set of blocks, but shares the
154
+ same underlying memory allocation with the original BlockTable.
155
+
156
+ Returns:
157
+ BlockTable: A new BlockTable instance with a copy of the blocks from
158
+ the current instance.
159
+ """
160
+ assert self._is_allocated
161
+ forked_blocks = self._allocator.fork(self._blocks[-1])
162
+ return BlockTable(
163
+ block_size=self._block_size,
164
+ block_allocator=self._allocator,
165
+ _blocks=forked_blocks,
166
+ )
167
+
168
+ def free(self) -> None:
169
+ """Frees the memory occupied by the blocks in the BlockTable.
170
+
171
+ This method iterates over all the blocks in the `_blocks` list and calls
172
+ the `free` method of the `_allocator` object to release the memory
173
+ occupied by each block. After freeing all the blocks, the `_blocks` list
174
+ is set to `None`.
175
+ """
176
+ assert self._is_allocated
177
+ for block in self._blocks:
178
+ self._allocator.free(block)
179
+ self._blocks = None
180
+
181
+ @property
182
+ def physical_block_ids(self) -> List[int]:
183
+ """Returns a list of physical block indices for the blocks in the
184
+ BlockTable.
185
+
186
+ This property returns a list of integers, where each integer represents
187
+ the physical block index of a corresponding block in the `_blocks` list.
188
+ The physical block index is a unique identifier for the memory location
189
+ occupied by the block.
190
+
191
+ Returns:
192
+ List[int]: A list of physical block indices for the blocks in the
193
+ BlockTable.
194
+ """
195
+ assert self._is_allocated
196
+ return [block.block_id for block in self._blocks]
197
+
198
+ def _allocate_blocks_for_token_ids(self, prev_block: Optional[Block],
199
+ token_ids: List[int],
200
+ device: Device) -> List[Block]:
201
+ blocks = []
202
+ for block_token_ids in chunk_list(token_ids, self._block_size):
203
+ if len(block_token_ids) == self._block_size:
204
+ # If the block is full, create an immutable block.
205
+ prev_block = self._allocator.allocate_immutable(
206
+ prev_block, token_ids=block_token_ids, device=device)
207
+ else:
208
+ # Else, partially fill a mutable block with token ids.
209
+ prev_block = self._allocator.allocate_mutable(
210
+ prev_block=prev_block, device=device)
211
+ prev_block.append_token_ids(block_token_ids)
212
+ blocks.append(prev_block)
213
+
214
+ return blocks
215
+
216
+ def _get_all_token_ids(self) -> List[int]:
217
+ # NOTE: This function is O(seq_len); use sparingly.
218
+ token_ids = []
219
+
220
+ if not self._is_allocated:
221
+ return token_ids
222
+
223
+ for block in self._blocks:
224
+ token_ids.extend(block.token_ids)
225
+
226
+ return token_ids
227
+
228
+ @property
229
+ def _is_allocated(self) -> bool:
230
+ return self._blocks is not None
231
+
232
+ @property
233
+ def _num_empty_slots(self) -> int:
234
+ assert self._is_allocated
235
+ return len(self._blocks) * self._block_size - self._num_full_slots
236
+
237
+ @property
238
+ def num_full_slots(self) -> int:
239
+ """Returns the total number of tokens currently stored in the
240
+ BlockTable.
241
+
242
+ Returns:
243
+ int: The total number of tokens currently stored in the BlockTable.
244
+ """
245
+ return self._num_full_slots
vllm/core/block/common.py
new
+185
@@ -0,0 +1,185 @@
1
+from collections import defaultdict
2
+from typing import Dict, Iterable, List, Optional
3
+
4
+from vllm.core.block.interfaces import Block, BlockAllocator
5
+
6
+BlockId = int
7
+RefCount = int
8
+
9
+
10
+class RefCounter:
11
+ """A class for managing reference counts for a set of block indices.
12
+
13
+ The RefCounter class maintains a dictionary that maps block indices to their
14
+ corresponding reference counts. It provides methods to increment, decrement,
15
+ and retrieve the reference count for a given block index.
16
+
17
+ Args:
18
+ all_block_indices (Iterable[BlockId]): An iterable of block indices
19
+ to initialize the reference counter with.
20
+ """
21
+
22
+ def __init__(self, all_block_indices: Iterable[BlockId]):
23
+ deduped = set(all_block_indices)
24
+ self._refcounts: Dict[BlockId,
25
+ RefCount] = {index: 0
26
+ for index in deduped}
27
+
28
+ def incr(self, block_id: BlockId) -> RefCount:
29
+ assert block_id in self._refcounts
30
+ pre_incr_refcount = self._refcounts[block_id]
31
+
32
+ assert pre_incr_refcount >= 0
33
+
34
+ post_incr_refcount = pre_incr_refcount + 1
35
+ self._refcounts[block_id] = post_incr_refcount
36
+ return post_incr_refcount
37
+
38
+ def decr(self, block_id: BlockId) -> RefCount:
39
+ assert block_id in self._refcounts
40
+ refcount = self._refcounts[block_id]
41
+
42
+ assert refcount > 0
43
+ refcount -= 1
44
+
45
+ self._refcounts[block_id] = refcount
46
+
47
+ return refcount
48
+
49
+ def get(self, block_id: BlockId) -> RefCount:
50
+ assert block_id in self._refcounts
51
+ return self._refcounts[block_id]
52
+
53
+ def as_readonly(self) -> "ReadOnlyRefCounter":
54
+ return ReadOnlyRefCounter(self)
55
+
56
+
57
+class ReadOnlyRefCounter:
58
+ """A read-only view of the RefCounter class.
59
+
60
+ The ReadOnlyRefCounter class provides a read-only interface to access the
61
+ reference counts maintained by a RefCounter instance. It does not allow
62
+ modifications to the reference counts.
63
+
64
+ Args:
65
+ refcounter (RefCounter): The RefCounter instance to create a read-only
66
+ view for.
67
+ """
68
+
69
+ def __init__(self, refcounter: RefCounter):
70
+ self._refcounter = refcounter
71
+
72
+ def incr(self, block_id: BlockId) -> RefCount:
73
+ raise ValueError("Incr not allowed")
74
+
75
+ def decr(self, block_id: BlockId) -> RefCount:
76
+ raise ValueError("Decr not allowed")
77
+
78
+ def get(self, block_id: BlockId) -> RefCount:
79
+ return self._refcounter.get(block_id)
80
+
81
+
82
+class CopyOnWriteTracker:
83
+ """A class for tracking and managing copy-on-write operations for blocks.
84
+
85
+ The CopyOnWriteTracker class maintains a mapping of source block indices to
86
+ their corresponding copy-on-write destination block indices. It works in
87
+ conjunction with a RefCounter and a BlockAllocator to handle reference
88
+ counting and block allocation.
89
+
90
+ Args:
91
+ refcounter (RefCounter): The reference counter used to track block
92
+ reference counts.
93
+ allocator (BlockAllocator): The block allocator used to allocate and
94
+ free blocks.
95
+ """
96
+
97
+ def __init__(
98
+ self,
99
+ refcounter: RefCounter,
100
+ allocator: BlockAllocator,
101
+ ):
102
+ self._copy_on_writes = defaultdict(list)
103
+ self._refcounter = refcounter
104
+ self._allocator = allocator
105
+
106
+ def cow_block_if_not_appendable(self, block: Block) -> Optional[BlockId]:
107
+ """Performs a copy-on-write operation on the given block if it is not
108
+ appendable.
109
+
110
+ This method checks the reference count of the given block. If the
111
+ reference count is greater than 1, indicating that the block is shared,
112
+ a copy-on-write operation is performed. The original block is freed,
113
+ and a new block is allocated with the same content. The new block index
114
+ is returned.
115
+
116
+ Args:
117
+ block (Block): The block to check for copy-on-write.
118
+
119
+ Returns:
120
+ Optional[BlockId]: The block index of the new block if a copy-on
121
+ -write operation was performed, or the original block index if
122
+ no copy-on-write was necessary.
123
+ """
124
+ block_id = block.block_id
125
+ if block_id is None:
126
+ return block_id
127
+
128
+ refcount = self._refcounter.get(block_id)
129
+ assert refcount != 0
130
+ if refcount > 1:
131
+ src_block_id = block_id
132
+
133
+ # Decrement refcount of the old block.
134
+ self._allocator.free(block)
135
+
136
+ # Allocate a fresh new block.
137
+ block_id = self._allocator.allocate_mutable(
138
+ prev_block=block.prev_block).block_id
139
+
140
+ # Track src/dst copy.
141
+ self._copy_on_writes[src_block_id].append(block_id)
142
+
143
+ return block_id
144
+
145
+ def clear_cows(self) -> Dict[BlockId, List[BlockId]]:
146
+ """Clears the copy-on-write tracking information and returns the current
147
+ state.
148
+
149
+ This method returns a dictionary mapping source block indices to lists
150
+ of destination block indices for the current copy-on-write operations.
151
+ It then clears the internal tracking information.
152
+
153
+ Returns:
154
+ Dict[BlockId, List[BlockId]]: A dictionary mapping source
155
+ block indices to lists of destination block indices for the
156
+ current copy-on-write operations.
157
+ """
158
+ cows = dict(self._copy_on_writes)
159
+ self._copy_on_writes.clear()
160
+ return cows
161
+
162
+
163
+def get_all_blocks_recursively(last_block: Block) -> List[Block]:
164
+ """Retrieves all the blocks in a sequence starting from the last block.
165
+
166
+ This function recursively traverses the sequence of blocks in reverse order,
167
+ starting from the given last block, and returns a list of all the blocks in
168
+ the sequence.
169
+
170
+ Args:
171
+ last_block (Block): The last block in the sequence.
172
+
173
+ Returns:
174
+ List[Block]: A list of all the blocks in the sequence, in the order they
175
+ appear.
176
+ """
177
+
178
+ def recurse(block: Block, lst: List[Block]) -> None:
179
+ if block.prev_block is not None:
180
+ recurse(block.prev_block, lst)
181
+ lst.append(block)
182
+
183
+ all_blocks = []
184
+ recurse(last_block, all_blocks)
185
+ return all_blocks
vllm/core/block/cpu_gpu_block_allocator.py
new
+206
@@ -0,0 +1,206 @@
1
+from typing import Dict, List, Optional
2
+
3
+from vllm.core.block.interfaces import (Block, BlockAllocator,
4
+ DeviceAwareBlockAllocator)
5
+from vllm.core.block.naive_block import NaiveBlock, NaiveBlockAllocator
6
+from vllm.core.block.prefix_caching_block import PrefixCachingBlockAllocator
7
+from vllm.utils import Device
8
+
9
+
10
+class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
11
+ """A block allocator that can allocate blocks on both CPU and GPU memory.
12
+
13
+ This class implements the `DeviceAwareBlockAllocator` interface and provides
14
+ functionality for allocating and managing blocks of memory on both CPU and
15
+ GPU devices.
16
+
17
+ The `CpuGpuBlockAllocator` maintains separate memory pools for CPU and GPU
18
+ blocks, and allows for allocation, deallocation, forking, and swapping of
19
+ blocks across these memory pools.
20
+ """
21
+
22
+ @staticmethod
23
+ def create(
24
+ allocator_type: str,
25
+ num_gpu_blocks: int,
26
+ num_cpu_blocks: int,
27
+ block_size: int,
28
+ ) -> DeviceAwareBlockAllocator:
29
+ """Creates a CpuGpuBlockAllocator instance with the specified
30
+ configuration.
31
+
32
+ This static method creates and returns a CpuGpuBlockAllocator instance
33
+ based on the provided parameters. It initializes the CPU and GPU block
34
+ allocators with the specified number of blocks, block size, and
35
+ allocator type.
36
+
37
+ Args:
38
+ allocator_type (str): The type of block allocator to use for CPU
39
+ and GPU blocks. Currently supported values are "naive" and
40
+ "prefix_caching".
41
+ num_gpu_blocks (int): The number of blocks to allocate for GPU
42
+ memory.
43
+ num_cpu_blocks (int): The number of blocks to allocate for CPU
44
+ memory.
45
+ block_size (int): The size of each block in number of tokens.
46
+
47
+ Returns:
48
+ DeviceAwareBlockAllocator: A CpuGpuBlockAllocator instance with the
49
+ specified configuration.
50
+
51
+ Notes:
52
+ - The block IDs are assigned contiguously, with GPU block IDs coming
53
+ before CPU block IDs.
54
+ """
55
+ block_ids = list(range(num_gpu_blocks + num_cpu_blocks))
56
+ gpu_block_ids = block_ids[:num_gpu_blocks]
57
+ cpu_block_ids = block_ids[num_gpu_blocks:]
58
+
59
+ if allocator_type == "naive":
60
+ gpu_allocator = NaiveBlockAllocator(
61
+ create_block=NaiveBlock,
62
+ num_blocks=num_gpu_blocks,
63
+ block_size=block_size,
64
+ block_ids=gpu_block_ids,
65
+ )
66
+
67
+ cpu_allocator = NaiveBlockAllocator(
68
+ create_block=NaiveBlock,
69
+ num_blocks=num_cpu_blocks,
70
+ block_size=block_size,
71
+ block_ids=cpu_block_ids,
72
+ )
73
+ elif allocator_type == "prefix_caching":
74
+ gpu_allocator = PrefixCachingBlockAllocator(
75
+ num_blocks=num_gpu_blocks,
76
+ block_size=block_size,
77
+ block_ids=gpu_block_ids,
78
+ )
79
+
80
+ cpu_allocator = PrefixCachingBlockAllocator(
81
+ num_blocks=num_cpu_blocks,
82
+ block_size=block_size,
83
+ block_ids=cpu_block_ids,
84
+ )
85
+ else:
86
+ raise ValueError(f"Unknown allocator type {allocator_type=}")
87
+
88
+ return CpuGpuBlockAllocator(
89
+ cpu_block_allocator=cpu_allocator,
90
+ gpu_block_allocator=gpu_allocator,
91
+ )
92
+
93
+ def __init__(
94
+ self,
95
+ cpu_block_allocator: BlockAllocator,
96
+ gpu_block_allocator: BlockAllocator,
97
+ ):
98
+ assert not (
99
+ cpu_block_allocator.all_block_ids
100
+ & gpu_block_allocator.all_block_ids
101
+ ), "cpu and gpu block allocators can't have intersection of block ids"
102
+
103
+ self._allocators = {
104
+ Device.CPU: cpu_block_allocator,
105
+ Device.GPU: gpu_block_allocator,
106
+ }
107
+
108
+ self._block_ids_to_allocator = {}
109
+ for _, allocator in self._allocators.items():
110
+ for block_id in allocator.all_block_ids:
111
+ self._block_ids_to_allocator[block_id] = allocator
112
+
113
+ def allocate_mutable(self, prev_block: Optional[Block],
114
+ device: Device) -> Block:
115
+ """Allocates a new mutable block on the specified device.
116
+
117
+ Args:
118
+ prev_block (Optional[Block]): The previous block to in the sequence.
119
+ Used for prefix hashing.
120
+ device (Device): The device on which to allocate the new block.
121
+
122
+ Returns:
123
+ Block: The newly allocated mutable block.
124
+ """
125
+ return self._allocators[device].allocate_mutable(prev_block)
126
+
127
+ def allocate_immutable(self, prev_block: Optional[Block],
128
+ token_ids: List[int], device: Device) -> Block:
129
+ """Allocates a new immutable block with the provided token IDs on the
130
+ specified device.
131
+
132
+ Args:
133
+ prev_block (Optional[Block]): The previous block in the sequence.
134
+ Used for prefix hashing.
135
+ token_ids (List[int]): The list of token IDs to be stored in the new
136
+ block.
137
+ device (Device): The device on which to allocate the new block.
138
+
139
+ Returns:
140
+ Block: The newly allocated immutable block containing the provided
141
+ token IDs.
142
+ """
143
+ return self._allocators[device].allocate_immutable(
144
+ prev_block, token_ids)
145
+
146
+ def free(self, block: Block) -> None:
147
+ """Frees the memory occupied by the given block.
148
+
149
+ Args:
150
+ block (Block): The block to be freed.
151
+ """
152
+ allocator = self._block_ids_to_allocator[block.block_id]
153
+ return allocator.free(block)
154
+
155
+ def fork(self, last_block: Block) -> List[Block]:
156
+ """Creates a new sequence of blocks that shares the same underlying
157
+ memory as the original sequence.
158
+
159
+ Args:
160
+ last_block (Block): The last block in the original sequence.
161
+
162
+ Returns:
163
+ List[Block]: A new list of blocks that shares the same memory as the
164
+ original sequence.
165
+ """
166
+ allocator = self._block_ids_to_allocator[last_block.block_id]
167
+ return allocator.fork(last_block)
168
+
169
+ def get_num_free_blocks(self, device: Device) -> int:
170
+ """Returns the number of free blocks available on the specified device.
171
+
172
+ Args:
173
+ device (Device): The device for which to query the number of free
174
+ blocks.
175
+
176
+ Returns:
177
+ int: The number of free blocks available on the specified device.
178
+ """
179
+ return self._allocators[device].get_num_free_blocks()
180
+
181
+ def clear_copy_on_writes(self) -> Dict[int, List[int]]:
182
+ """Clears the copy-on-write (CoW) state and returns the mapping of
183
+ source to destination block IDs.
184
+
185
+ Returns:
186
+ Dict[int, List[int]]: A dictionary mapping source block IDs to lists
187
+ of destination block IDs.
188
+ """
189
+ # CoW only supported on GPU
190
+ device = Device.GPU
191
+ return self._allocators[device].clear_copy_on_writes()
192
+
193
+ def mark_blocks_as_computed(self) -> None:
194
+ # Prefix caching only supported on GPU.
195
+ device = Device.GPU
196
+ return self._allocators[device].mark_blocks_as_computed()
197
+
198
+ def get_common_computed_block_ids(
199
+ self, seq_block_ids: List[List[int]]) -> List[int]:
200
+ # Prefix caching only supported on GPU.
201
+ device = Device.GPU
202
+ return self._allocators[device].get_common_computed_block_ids(
203
+ seq_block_ids)
204
+
205
+ def all_block_ids(self) -> frozenset[int]:
206
+ return frozenset(self._block_ids_to_allocator.keys())
vllm/core/block/interfaces.py
new
+105
@@ -0,0 +1,105 @@
1
+from abc import ABC, abstractmethod, abstractproperty
2
+from typing import Dict, List, Optional, Protocol
3
+
4
+from vllm.utils import Device
5
+
6
+
7
+class Block(ABC):
8
+
9
+ @abstractmethod
10
+ def append_token_ids(self, token_ids: List[int]) -> None:
11
+ pass
12
+
13
+ @abstractproperty
14
+ def block_id(self) -> Optional[int]:
15
+ pass
16
+
17
+ @abstractproperty
18
+ def token_ids(self) -> List[int]:
19
+ pass
20
+
21
+ @abstractproperty
22
+ def num_empty_slots(self) -> int:
23
+ pass
24
+
25
+ @abstractproperty
26
+ def is_full(self) -> bool:
27
+ pass
28
+
29
+ @abstractproperty
30
+ def prev_block(self) -> Optional["Block"]:
31
+ pass
32
+
33
+ class Factory(Protocol):
34
+
35
+ @abstractmethod
36
+ def __call__(
37
+ self,
38
+ prev_block: Optional["Block"],
39
+ token_ids: List[int],
40
+ block_size: int,
41
+ allocator: "BlockAllocator",
42
+ block_id: Optional[int] = None,
43
+ ) -> "Block":
44
+ pass
45
+
46
+
47
+class BlockAllocator(ABC):
48
+
49
+ @abstractmethod
50
+ def allocate_mutable(self, prev_block: Optional[Block]) -> Block:
51
+ pass
52
+
53
+ @abstractmethod
54
+ def allocate_immutable(self, prev_block: Optional[Block],
55
+ token_ids: List[int]) -> Block:
56
+ pass
57
+
58
+ @abstractmethod
59
+ def free(self, block: Block) -> None:
60
+ pass
61
+
62
+ @abstractmethod
63
+ def fork(self, last_block: Block) -> List[Block]:
64
+ pass
65
+
66
+ @abstractmethod
67
+ def get_num_free_blocks(self) -> int:
68
+ pass
69
+
70
+ @abstractproperty
71
+ def all_block_ids(self) -> frozenset[int]:
72
+ pass
73
+
74
+ @abstractmethod
75
+ def clear_copy_on_writes(self) -> Dict[int, List[int]]:
76
+ pass
77
+
78
+ @abstractmethod
79
+ def mark_blocks_as_computed(self) -> None:
80
+ pass
81
+
82
+ @abstractmethod
83
+ def get_common_computed_block_ids(
84
+ self, seq_block_ids: List[List[int]]) -> List[int]:
85
+ pass
86
+
87
+ class NoFreeBlocksError(ValueError):
88
+ pass
89
+
90
+
91
+class DeviceAwareBlockAllocator(BlockAllocator):
92
+
93
+ @abstractmethod
94
+ def allocate_mutable(self, prev_block: Optional[Block],
95
+ device: Device) -> Block:
96
+ pass
97
+
98
+ @abstractmethod
99
+ def allocate_immutable(self, prev_block: Optional[Block],
100
+ token_ids: List[int], device: Device) -> Block:
101
+ pass
102
+
103
+ @abstractmethod
104
+ def get_num_free_blocks(self, device: Device) -> int:
105
+ pass
vllm/core/block/naive_block.py
new
+275
@@ -0,0 +1,275 @@
1
+from typing import Dict, Iterable, List, Optional, Set
2
+
3
+from vllm.core.block.common import (CopyOnWriteTracker, RefCounter,
4
+ get_all_blocks_recursively)
5
+from vllm.core.block.interfaces import Block, BlockAllocator
6
+
7
+BlockId = int
8
+Refcount = int
9
+
10
+
11
+class NaiveBlockAllocator(BlockAllocator):
12
+ """A simple block allocator that manages blocks of memory without prefix
13
+ caching.
14
+
15
+ Args:
16
+ create_block (Block.Factory): A factory function for creating new
17
+ blocks. This is used when a NaiveBlockAllocator is composed within
18
+ a prefix caching allocator -- the naive block allocator must
19
+ construct prefix caching blocks (but shouldn't know anything else
20
+ about them).
21
+ num_blocks (int): The total number of blocks to manage.
22
+ block_size (int): The size of each block in tokens.
23
+ block_ids (Optional[Iterable[int]], optional): An optional iterable of
24
+ block IDs. If not provided, block IDs will be assigned sequentially
25
+ from 0 to num_blocks - 1.
26
+ """
27
+
28
+ def __init__(
29
+ self,
30
+ create_block: Block.Factory,
31
+ num_blocks: int,
32
+ block_size: int,
33
+ block_ids: Optional[Iterable[int]] = None,
34
+ ):
35
+ if block_ids is None:
36
+ block_ids = range(num_blocks)
37
+
38
+ self._free_block_indices: Set[BlockId] = set(block_ids)
39
+ self._all_block_indices = frozenset(block_ids)
40
+ assert len(self._all_block_indices) == num_blocks
41
+
42
+ self._refcounter = RefCounter(
43
+ all_block_indices=self._free_block_indices)
44
+ self._create_block = create_block
45
+ self._block_size = block_size
46
+
47
+ self._cow_tracker = CopyOnWriteTracker(
48
+ refcounter=self._refcounter.as_readonly(),
49
+ allocator=self,
50
+ )
51
+
52
+ def allocate_immutable(self, prev_block: Optional[Block],
53
+ token_ids: List[int]) -> Block:
54
+ """Allocates a new immutable block with the given token IDs, linked to
55
+ the previous block.
56
+
57
+ Args:
58
+ prev_block (Optional[Block]): The previous block in the sequence. If
59
+ None, then the block to be allocated is the first block in the
60
+ sequence.
61
+ token_ids (List[int]): The token IDs to be stored in the new block.
62
+
63
+ Returns:
64
+ Block: The newly allocated immutable block.
65
+ """
66
+ block = self.allocate_mutable(prev_block=prev_block)
67
+ block.append_token_ids(token_ids)
68
+ return block
69
+
70
+ def allocate_mutable(self, prev_block: Optional[Block]) -> Block:
71
+ """Allocates a new mutable block, linked to the previous block.
72
+
73
+ Args:
74
+ prev_block (Optional[Block]): The previous block in the sequence. If
75
+ None, then the block to be allocated is the first block in the
76
+ sequence.
77
+
78
+ Returns:
79
+ Block: The newly allocated mutable block.
80
+ """
81
+ block_id = self._allocate_new_block_id()
82
+ return self._create_block(
83
+ prev_block=prev_block,
84
+ token_ids=[],
85
+ block_id=block_id,
86
+ block_size=self._block_size,
87
+ allocator=self,
88
+ )
89
+
90
+ def free(self, block: Block) -> None:
91
+ self._free_block_id(block.block_id)
92
+
93
+ # Mark the block as having no allocation.
94
+ block.block_id = None
95
+
96
+ def fork(self, last_block: Block) -> List[Block]:
97
+ """Creates a new sequence of blocks that shares the same underlying
98
+ memory as the original sequence.
99
+
100
+ Args:
101
+ last_block (Block): The last block in the original sequence.
102
+
103
+ Returns:
104
+ List[Block]: The new sequence of blocks that shares the same memory
105
+ as the original sequence.
106
+ """
107
+ source_blocks = get_all_blocks_recursively(last_block)
108
+
109
+ forked_blocks = []
110
+ prev_block = None
111
+ for block in source_blocks:
112
+
113
+ # Increment refcount for each block.
114
+ refcount = self._refcounter.incr(block.block_id)
115
+ assert refcount != 1, "can't fork free'd block"
116
+
117
+ forked_blocks.append(
118
+ self._create_block(
119
+ prev_block=prev_block,
120
+ token_ids=block.token_ids,
121
+ block_id=block.block_id,
122
+ block_size=self._block_size,
123
+ allocator=self,
124
+ ))
125
+ prev_block = forked_blocks[-1]
126
+
127
+ return forked_blocks
128
+
129
+ def get_num_free_blocks(self) -> int:
130
+ return len(self._free_block_indices)
131
+
132
+ def _allocate_new_block_id(self) -> BlockId:
133
+ if not self._free_block_indices:
134
+ raise BlockAllocator.NoFreeBlocksError()
135
+
136
+ block_id = next(iter(self._free_block_indices))
137
+ self._refcounter.incr(block_id)
138
+ self._free_block_indices.remove(block_id)
139
+ return block_id
140
+
141
+ def _free_block_id(self, block_id: BlockId) -> None:
142
+ refcount = self._refcounter.decr(block_id)
143
+ if refcount == 0:
144
+ self._free_block_indices.add(block_id)
145
+
146
+ @property
147
+ def refcounter(self):
148
+ return self._refcounter
149
+
150
+ @property
151
+ def all_block_ids(self):
152
+ return self._all_block_indices
153
+
154
+ def cow_block_if_not_appendable(self, block: Block) -> Optional[BlockId]:
155
+ """Performs a copy-on-write operation on the given block if it is not
156
+ appendable.
157
+
158
+ Args:
159
+ block (Block): The block to check for copy-on-write.
160
+
161
+ Returns:
162
+ Optional[BlockId]: The block index of the new block if a copy-on
163
+ -write operation was performed, or the original block index if
164
+ no copy-on-write was necessary.
165
+ """
166
+ return self._cow_tracker.cow_block_if_not_appendable(block)
167
+
168
+ def clear_copy_on_writes(self) -> Dict[BlockId, List[BlockId]]:
169
+ """Returns the copy-on-write source->destination mapping and clears it.
170
+
171
+ Returns:
172
+ Dict[BlockId, List[BlockId]]: A dictionary mapping source
173
+ block indices to lists of destination block indices.
174
+ """
175
+ return self._cow_tracker.clear_cows()
176
+
177
+ def mark_blocks_as_computed(self) -> None:
178
+ """Mark blocks as computed, used in prefix caching.
179
+
180
+ Since the naive allocator does not implement prefix caching, we do
181
+ nothing.
182
+ """
183
+ pass
184
+
185
+ def get_common_computed_block_ids(
186
+ self, seq_block_ids: List[List[int]]) -> List[int]:
187
+ """Determine blocks that can be skipped in prefill.
188
+
189
+ Since the naive allocator does not support prefix caching, always return
190
+ an empty list.
191
+ """
192
+ return []
193
+
194
+
195
+class NaiveBlock(Block):
196
+ """An implementation of the Block class that does not support prefix
197
+ caching.
198
+
199
+ The NaiveBlock class represents a block of token IDs with a fixed size. It
200
+ provides methods for appending token IDs to the block and manages copy-on
201
+ -write operations when necessary.
202
+
203
+ Args:
204
+ prev_block (Block): The previous block in the sequence.
205
+ token_ids (List[int]): The initial token IDs to be stored in the block.
206
+ block_size (int): The maximum number of token IDs that can be stored in
207
+ the block.
208
+ allocator (BlockAllocator): The block allocator associated with this
209
+ block.
210
+ block_id (Optional[int], optional): The physical block index
211
+ of this block. Defaults to None, which means no allocation has been
212
+ made.
213
+ _cow_target (Optional[Block], optional): The copy-on-write target block.
214
+ If not provided, it defaults to self.
215
+ """
216
+
217
+ def __init__(self,
218
+ prev_block: Block,
219
+ token_ids: List[int],
220
+ block_size: int,
221
+ allocator: BlockAllocator,
222
+ block_id: Optional[int] = None,
223
+ _cow_target: Optional[Block] = None):
224
+ self._token_ids = []
225
+ self._block_size = block_size
226
+ self._prev_block = prev_block
227
+ self._block_id = block_id
228
+ self._allocator = allocator
229
+ self._cow_target = _cow_target if _cow_target is not None else self
230
+
231
+ self._append_token_ids_no_cow(token_ids)
232
+
233
+ def append_token_ids(self, token_ids: List[int]) -> None:
234
+ """Appends the given token IDs to the block, instructing the allocator
235
+ to perform a copy-on-write if necessary.
236
+
237
+ Args:
238
+ token_ids (List[int]): The token IDs to be appended to the block.
239
+ """
240
+ self._append_token_ids_no_cow(token_ids)
241
+
242
+ if self._block_id is not None:
243
+ self._block_id = (self._allocator.cow_block_if_not_appendable(
244
+ self._cow_target))
245
+
246
+ def _append_token_ids_no_cow(self, token_ids: List[int]) -> None:
247
+ assert self.num_empty_slots >= len(token_ids)
248
+ self._token_ids.extend(token_ids)
249
+
250
+ @property
251
+ def block_id(self) -> Optional[int]:
252
+ return self._block_id
253
+
254
+ @block_id.setter
255
+ def block_id(self, value: Optional[int]) -> None:
256
+ self._block_id = value
257
+
258
+ @property
259
+ def is_full(self) -> bool:
260
+ return self.num_empty_slots == 0
261
+
262
+ @property
263
+ def num_empty_slots(self) -> int:
264
+ return self._block_size - len(self._token_ids)
265
+
266
+ @property
267
+ def token_ids(self) -> List[int]:
268
+ return self._token_ids
269
+
270
+ def block_size(self) -> int:
271
+ return self._block_size
272
+
273
+ @property
274
+ def prev_block(self) -> Optional["Block"]:
275
+ return self._prev_block
vllm/core/block/prefix_caching_block.py
new
+472
@@ -0,0 +1,472 @@
1
+"""Token blocks."""
2
+from itertools import takewhile
3
+from os.path import commonprefix
4
+from typing import Dict, Iterable, List, Optional
5
+
6
+from vllm.core.block.common import (CopyOnWriteTracker,
7
+ get_all_blocks_recursively)
8
+from vllm.core.block.interfaces import Block, BlockAllocator
9
+from vllm.core.block.naive_block import NaiveBlock, NaiveBlockAllocator
10
+
11
+PrefixHash = int
12
+BlockId = int
13
+
14
+
15
+class PrefixCachingBlockAllocator(BlockAllocator):
16
+ """A block allocator that implements prefix caching.
17
+
18
+ The PrefixCachingBlockAllocator maintains a cache of blocks based on their
19
+ content hash. It reuses blocks with the same content hash to avoid redundant
20
+ memory allocation. The allocator also supports copy-on-write operations.
21
+
22
+ Args:
23
+ num_blocks (int): The total number of blocks to manage.
24
+ block_size (int): The size of each block in tokens.
25
+ block_ids(Optional[Iterable[int]], optional): An optional iterable of
26
+ block IDs. If not provided, block IDs will be assigned sequentially
27
+ from 0 to num_blocks - 1.
28
+ """
29
+
30
+ # TODO last access time / evictor integration
31
+
32
+ def __init__(
33
+ self,
34
+ num_blocks: int,
35
+ block_size: int,
36
+ block_ids: Optional[Iterable[int]] = None,
37
+ ):
38
+ # A mapping of prefix hash to block index. All blocks which have a
39
+ # prefix hash will be in this dict, even if they have refcount 0.
40
+ self._cached_blocks: Dict[PrefixHash, BlockId] = {}
41
+
42
+ # A mapping of prefix hash to block index. All blocks which have a
43
+ # prefix hash AND refcount 0 will be in this dict. Thus, it is a subset
44
+ # of self._cached_blocks.
45
+ self._unused_cached_blocks: Dict[PrefixHash, BlockId] = {}
46
+
47
+ # An allocator for blocks that do not have prefix hashes.
48
+ self._hashless_allocator = NaiveBlockAllocator(
49
+ create_block=self._create_block,
50
+ num_blocks=num_blocks,
51
+ block_size=block_size,
52
+ block_ids=block_ids,
53
+ )
54
+
55
+ self._block_size = block_size
56
+
57
+ # We share the refcounter between allocators. This allows us to promote
58
+ # blocks originally allocated in the hashless allocator to immutable
59
+ # blocks.
60
+ self._refcounter = self._hashless_allocator.refcounter
61
+
62
+ self._cow_tracker = CopyOnWriteTracker(
63
+ refcounter=self._refcounter.as_readonly(),
64
+ allocator=self,
65
+ )
66
+
67
+ # Implements Block.Factory.
68
+ def _create_block(
69
+ self,
70
+ prev_block: Optional[Block],
71
+ token_ids: List[int],
72
+ block_size: int,
73
+ allocator: BlockAllocator,
74
+ block_id: Optional[int] = None,
75
+ ) -> Block:
76
+ # Bind block to self.
77
+ allocator = self
78
+
79
+ return PrefixCachingBlock(
80
+ prev_block=prev_block,
81
+ token_ids=token_ids,
82
+ block_size=block_size,
83
+ block_id=block_id,
84
+ prefix_caching_allocator=allocator,
85
+ )
86
+
87
+ def allocate_immutable(self, prev_block: Optional[Block],
88
+ token_ids: List[int]) -> Block:
89
+ """Allocates an immutable block with the given token IDs, reusing cached
90
+ blocks if possible.
91
+
92
+ Args:
93
+ prev_block (Optional[Block]): The previous block in the sequence.
94
+ token_ids (List[int]): The token IDs to be stored in the block.
95
+
96
+ Returns:
97
+ Block: The allocated immutable block.
98
+ """
99
+ assert_prefix_caching_block_or_none(prev_block)
100
+
101
+ block = self._create_block(
102
+ prev_block=prev_block,
103
+ token_ids=token_ids,
104
+ block_size=self._block_size,
105
+ allocator=self,
106
+ )
107
+ assert block.content_hash is not None
108
+
109
+ cached_block_id = self._cached_blocks.get(block.content_hash, None)
110
+ if cached_block_id is not None:
111
+ block.block_id = cached_block_id
112
+ self._incr_refcount_cached_block(block.content_hash,
113
+ block.block_id)
114
+ return block
115
+
116
+ block = self.allocate_mutable(prev_block)
117
+ block.append_token_ids(token_ids)
118
+ assert block.content_hash is not None
119
+ # TODO computed bit
120
+
121
+ return block
122
+
123
+ def allocate_mutable(self, prev_block: Block) -> Block:
124
+ """Allocates a mutable block. If there are no free blocks, this will
125
+ evict unused cached blocks.
126
+
127
+ Args:
128
+ prev_block (Block): The previous block in the sequence.
129
+
130
+ Returns:
131
+ Block: The allocated mutable block.
132
+ """
133
+ assert_prefix_caching_block_or_none(prev_block)
134
+
135
+ try:
136
+ return self._hashless_allocator.allocate_mutable(
137
+ prev_block=prev_block)
138
+ except BlockAllocator.NoFreeBlocksError:
139
+ # We must check the unused cached blocks before raising OOM.
140
+ pass
141
+
142
+ if self._unused_cached_blocks:
143
+ # TODO policy for selecting block to remove
144
+ content_hash_to_evict = next(iter(self._unused_cached_blocks))
145
+
146
+ # Clear content hash mapping; the block will be overwritten.
147
+ del self._cached_blocks[content_hash_to_evict]
148
+
149
+ block_id = self._unused_cached_blocks.pop(content_hash_to_evict)
150
+ refcount = self._refcounter.incr(block_id)
151
+ assert refcount == 1
152
+ block = self._create_block(
153
+ prev_block=prev_block,
154
+ token_ids=[],
155
+ block_size=self._block_size,
156
+ allocator=self,
157
+ block_id=block_id,
158
+ )
159
+ assert block.content_hash is None
160
+ return block
161
+
162
+ # No block available in hashless allocator, nor in unused cache blocks.
163
+ raise BlockAllocator.NoFreeBlocksError()
164
+
165
+ def _incr_refcount_cached_block(self, content_hash: int,
166
+ block_id: BlockId) -> None:
167
+ refcount = self._refcounter.incr(block_id)
168
+ if refcount == 1:
169
+ assert content_hash in self._unused_cached_blocks
170
+ del self._unused_cached_blocks[content_hash]
171
+
172
+ def free(self, block: Block) -> None:
173
+ """Decrement the refcount of the block. If the decremented refcount is
174
+ zero, store the block in the freelist.
175
+
176
+ If the block has a content hash (meaning it is immutable), then we will
177
+ keep the block around in case future allocations require it.
178
+ """
179
+ assert (block.block_id
180
+ is not None), "freeing unallocated block is undefined"
181
+
182
+ self._free_block_id_for_block(block.block_id, block)
183
+ block.block_id = None
184
+
185
+ def _free_block_id_for_block(self, block_id: BlockId,
186
+ block: Block) -> None:
187
+ assert isinstance(block, PrefixCachingBlock)
188
+
189
+ if block.content_hash is None:
190
+ return self._hashless_allocator.free(block)
191
+
192
+ refcount = self._refcounter.decr(block_id)
193
+
194
+ # If no longer used, add the block to the unused cached blocks.
195
+ if refcount == 0:
196
+ assert block.content_hash not in self._unused_cached_blocks
197
+ assert block.content_hash in self._cached_blocks
198
+ self._unused_cached_blocks[block.content_hash] = block_id
199
+
200
+ def fork(self, last_block: Block) -> List[Block]:
201
+ """Creates a new sequence of blocks that shares the same underlying
202
+ memory as the original sequence.
203
+
204
+ Args:
205
+ last_block (Block): The last block in the original sequence.
206
+
207
+ Returns:
208
+ List[Block]: The new sequence of blocks that shares the same memory
209
+ as the original sequence.
210
+ """
211
+ source_blocks = get_all_blocks_recursively(last_block)
212
+
213
+ forked_blocks = []
214
+ prev_block = None
215
+ for block in source_blocks:
216
+ refcount = self._refcounter.incr(block.block_id)
217
+ assert refcount != 1, "can't fork free'd block"
218
+
219
+ forked_blocks.append(
220
+ self._create_block(
221
+ prev_block=prev_block,
222
+ token_ids=block.token_ids,
223
+ block_id=block.block_id,
224
+ block_size=self._block_size,
225
+ allocator=self,
226
+ ))
227
+ prev_block = forked_blocks[-1]
228
+
229
+ return forked_blocks
230
+
231
+ def get_num_free_blocks(self) -> int:
232
+ # The number of free blocks is the number of hashless free blocks
233
+ # plus the number of hashful blocks that are unused.
234
+ return self._hashless_allocator.get_num_free_blocks() + len(
235
+ self._unused_cached_blocks)
236
+
237
+ @property
238
+ def all_block_ids(self) -> frozenset[int]:
239
+ return self._hashless_allocator.all_block_ids
240
+
241
+ def promote_to_immutable_block(self,
242
+ block: "PrefixCachingBlock") -> BlockId:
243
+ """Once a mutable block is full, it can be promoted to an immutable
244
+ block. This means that its content can be referenced by future blocks
245
+ having the same prefix.
246
+
247
+ Note that if we already have a cached block with the same content, we
248
+ will replace the newly-promoted block's mapping with the existing cached
249
+ block.
250
+
251
+ Args:
252
+ block (PrefixCachingBlock): The mutable block to be promoted.
253
+
254
+ Returns:
255
+ BlockId: Either the original block index, or the block index of
256
+ the previously cached block matching the same content.
257
+ """
258
+ assert block.content_hash is not None
259
+ assert block.block_id is not None
260
+ assert self._refcounter.get(block.block_id) > 0
261
+
262
+ # If the content hash does not have a corresponding cached block,
263
+ # set this block as the cached block.
264
+ if block.content_hash not in self._cached_blocks:
265
+ self._cached_blocks[block.content_hash] = block.block_id
266
+ else:
267
+ self._free_block_id_for_block(block.block_id, block)
268
+ self._incr_refcount_cached_block(
269
+ block.content_hash, self._cached_blocks[block.content_hash])
270
+
271
+ return self._cached_blocks[block.content_hash]
272
+
273
+ def cow_block_if_not_appendable(self, block: Block) -> Optional[BlockId]:
274
+ """Performs a copy-on-write operation on the given block if it is not
275
+ appendable.
276
+
277
+ Args:
278
+ block (Block): The block to check for copy-on-write.
279
+
280
+ Returns:
281
+ Optional[BlockId]: The block index of the new block if a copy-on
282
+ -write operation was performed, or the original block index if
283
+ no copy-on-write was necessary.
284
+ """
285
+ return self._cow_tracker.cow_block_if_not_appendable(block)
286
+
287
+ def clear_copy_on_writes(self) -> Dict[BlockId, List[BlockId]]:
288
+ """Returns the copy-on-write source->destination mapping and clears it.
289
+
290
+ Returns:
291
+ Dict[BlockId, List[BlockId]]: A dictionary mapping source
292
+ block indices to lists of destination block indices.
293
+ """
294
+ return self._cow_tracker.clear_cows()
295
+
296
+ def mark_blocks_as_computed(self) -> None:
297
+ """Mark blocks as computed, used in prefix caching."""
298
+ # TODO Track computed blocks.
299
+ pass
300
+
301
+ def get_common_computed_block_ids(
302
+ self, seq_block_ids: List[List[int]]) -> List[int]:
303
+ """Return the block ids that are common for a given sequence group.
304
+
305
+ Used in prefill (can skip prefill of some blocks).
306
+ """
307
+
308
+ # TODO: Track computed blocks.
309
+ computed = lambda block_id: False
310
+
311
+ # NOTE We exclude the last block to avoid the case where the entire
312
+ # prompt is cached. This would cause erroneous behavior in model
313
+ # runner.
314
+ ids_list = [
315
+ takewhile(lambda block_id: computed(block_id), seq[:-1])
316
+ for seq in seq_block_ids
317
+ ]
318
+ return commonprefix([ids for ids in ids_list if ids != []])
319
+
320
+
321
+class PrefixCachingBlock(Block):
322
+ """A block implementation that supports prefix caching.
323
+
324
+ The PrefixCachingBlock class represents a block of token IDs with prefix
325
+ caching capabilities. It wraps a NaiveBlock internally and provides
326
+ additional functionality for content hashing and promoting immutable blocks
327
+ with the prefix caching allocator.
328
+
329
+ Args:
330
+ prev_block (Optional[PrefixCachingBlock]): The previous block in the
331
+ sequence.
332
+ token_ids (List[int]): The initial token IDs to be stored in the block.
333
+ block_size (int): The maximum number of token IDs that can be stored in
334
+ the block.
335
+ prefix_caching_allocator (PrefixCachingBlockAllocator): The prefix
336
+ caching block allocator associated with this block.
337
+ block_id (Optional[int], optional): The physical block index
338
+ of this block. Defaults to None.
339
+ """
340
+
341
+ def __init__(
342
+ self,
343
+ prev_block: Optional["PrefixCachingBlock"],
344
+ token_ids: List[int],
345
+ block_size: int,
346
+ prefix_caching_allocator: PrefixCachingBlockAllocator,
347
+ block_id: Optional[int] = None,
348
+ ):
349
+ assert_prefix_caching_block_or_none(prev_block)
350
+
351
+ self._prev_block = prev_block
352
+ self._cached_content_hash: Optional[int] = None
353
+ self._prefix_caching_allocator = prefix_caching_allocator
354
+
355
+ self._block = NaiveBlock(
356
+ prev_block=prev_block,
357
+ token_ids=token_ids,
358
+ block_size=block_size,
359
+ block_id=block_id,
360
+ allocator=prefix_caching_allocator,
361
+ _cow_target=self,
362
+ )
363
+
364
+ def append_token_ids(self, token_ids: List[int]) -> None:
365
+ """Appends the given token IDs to the block and registers the block as
366
+ immutable if the block becomes full.
367
+
368
+ Internally, the naive block handles CoW.
369
+
370
+ Args:
371
+ token_ids (List[int]): The token IDs to be appended to the block.
372
+ """
373
+ assert token_ids
374
+
375
+ # naive block handles CoW.
376
+ self._block.append_token_ids(token_ids)
377
+
378
+ # If the content hash is present, then the block can be made immutable.
379
+ # Register ourselves with the allocator, potentially replacing the
380
+ # physical block index.
381
+ if self.content_hash is not None:
382
+ self.block_id = (self._prefix_caching_allocator.
383
+ promote_to_immutable_block(self))
384
+
385
+ @property
386
+ def block_id(self) -> Optional[int]:
387
+ return self._block.block_id
388
+
389
+ @block_id.setter
390
+ def block_id(self, value) -> None:
391
+ self._block.block_id = value
392
+
393
+ @property
394
+ def is_full(self) -> bool:
395
+ return self._block.is_full
396
+
397
+ @property
398
+ def num_empty_slots(self) -> int:
399
+ return self._block.num_empty_slots
400
+
401
+ @property
402
+ def block_size(self) -> int:
403
+ return self._block.block_size
404
+
405
+ @property
406
+ def token_ids(self) -> List[int]:
407
+ return self._block.token_ids
408
+
409
+ @property
410
+ def prev_block(self) -> Optional[Block]:
411
+ return self._prev_block
412
+
413
+ @property
414
+ def content_hash(self) -> Optional[int]:
415
+ """Return the content-based hash of the current block, or None if it is
416
+ not yet defined.
417
+
418
+ For the content-based hash to be defined, the current block must be
419
+ full.
420
+ """
421
+
422
+ # If the hash is already computed, return it.
423
+ if self._cached_content_hash is not None:
424
+ return self._cached_content_hash
425
+
426
+ # We cannot compute a hash for the current block because it is not full.
427
+ if not self.is_full:
428
+ return None
429
+
430
+ is_first_block = self._prev_block is None
431
+ prev_block_hash = (None if is_first_block else
432
+ self._prev_block.content_hash)
433
+
434
+ # Previous block exists but does not yet have a hash.
435
+ # Return no hash in this case.
436
+ if prev_block_hash is None and not is_first_block:
437
+ return None
438
+
439
+ self._cached_content_hash = PrefixCachingBlock.hash_block_tokens(
440
+ is_first_block,
441
+ prev_block_hash,
442
+ cur_block_token_ids=self.token_ids)
443
+ return self._cached_content_hash
444
+
445
+ @staticmethod
446
+ def hash_block_tokens(is_first_block: bool, prev_block_hash: Optional[int],
447
+ cur_block_token_ids: List[int]) -> int:
448
+ """Computes a hash value corresponding to the contents of a block and
449
+ the contents of the preceding block(s). The hash value is used for
450
+ prefix caching.
451
+
452
+ NOTE: Content-based hashing does not yet support LoRA.
453
+
454
+ Parameters:
455
+ - is_first_block (bool): A flag indicating if the block is the first in
456
+ the sequence.
457
+ - prev_block_hash (Optional[int]): The hash of the previous block. None
458
+ if this is the first block.
459
+ - cur_block_token_ids (List[int]): A list of token ids in the current
460
+ block. The current block is assumed to be full.
461
+
462
+ Returns:
463
+ - int: The computed hash value for the block.
464
+ """
465
+ assert (prev_block_hash is None) == is_first_block
466
+ return hash((is_first_block, prev_block_hash, *cur_block_token_ids))
467
+
468
+
469
+def assert_prefix_caching_block_or_none(block: Optional[Block]):
470
+ if block is None:
471
+ return
472
+ assert isinstance(block, PrefixCachingBlock)
vllm/core/block_manager_v1.py
renamed
+20
-22
@@ -1,5 +1,4 @@
1
"""A block manager that manages token blocks."""
2
-import enum
2
from abc import ABC, abstractmethod
3
from itertools import count, takewhile
4
from os.path import commonprefix
@@ -7,6 +6,7 @@ from typing import Dict, List, Optional, Set, Tuple
6
7
from vllm.block import BlockTable, PhysicalTokenBlock
8
from vllm.core.evictor import EvictionPolicy, Evictor, make_evictor
9
+from vllm.core.interfaces import AllocStatus, BlockSpaceManager
10
from vllm.logger import init_logger
11
from vllm.sequence import Sequence, SequenceGroup, SequenceStatus
12
from vllm.utils import Device
@@ -196,21 +196,7 @@ class UncachedBlockAllocator(BlockAllocatorBase):
196
"Invalid codepath for uncached block allocator.")
197
198
199
-class AllocStatus(enum.Enum):
200
- """Result for BlockSpaceManager.can_allocate
201
-
202
- 1. Ok: seq_group can be allocated now.
203
- 2. Later: seq_group cannot be allocated.
204
- The capacity of allocator is larger than seq_group required.
205
- 3. Never: seq_group can never be allocated.
206
- The seq_group is too large to allocated in GPU.
207
- """
208
- OK = enum.auto()
209
- LATER = enum.auto()
210
- NEVER = enum.auto()
211
-
212
-
213
-class BlockSpaceManager:
199
+class BlockSpaceManagerV1(BlockSpaceManager):
200
"""Manages the mapping between logical and physical token blocks."""
201
202
def __init__(
@@ -355,6 +341,11 @@ class BlockSpaceManager:
341
self,
342
seq: Sequence,
343
) -> PhysicalTokenBlock:
344
+ # Called before a new block is appended.
345
+ # This is in charge of allocating a new physical block (to be appended).
346
+
347
+ # None if the last block is not full. Otherwise, we set it to the
348
+ # content hash.
349
if not self.enable_caching:
350
return self.gpu_allocator.allocate()
351
block_hash: Optional[int] = None
@@ -362,7 +353,14 @@ class BlockSpaceManager:
353
block_hash = seq.hash_of_block(len(seq.logical_token_blocks) - 1)
354
num_hashed_tokens = seq.num_hashed_tokens_of_block(
355
len(seq.logical_token_blocks) - 1)
356
+
357
+ # num_hashed_tokens is used to compute future hashes
358
+ # (e.g. in the hashing function, it is used to ask the sequence for
359
+ # prefix tokens)
360
new_block = self.gpu_allocator.allocate(block_hash, num_hashed_tokens)
361
+
362
+ # If the block has is None, then the block is not full.
363
+ # If the block is not full, then we expect it to have a refcount of 1.
364
if block_hash is None:
365
assert new_block.ref_count == 1
366
return new_block
@@ -576,16 +574,16 @@ class BlockSpaceManager:
574
for b in takewhile(lambda b: b.computed, block_table[:-1])
575
]
576
579
- def get_common_computed_block_ids(self,
580
- seq_group: SequenceGroup) -> List[int]:
577
+ def get_common_computed_block_ids(self, seqs: List[Sequence]) -> List[int]:
578
+ """Return the block ids that are common for a given sequence group.
579
+
580
+ Used in prefill (can skip prefill of some blocks).
581
+ """
582
# Can return non-empty result only with prefix caching enabled.
583
if not self.enable_caching:
584
return []
585
585
- ids_list = [
586
- self.get_all_computed_blocks(seq)
587
- for seq in iter(seq_group.seqs_dict.values())
588
- ]
586
+ ids_list = [self.get_all_computed_blocks(seq) for seq in seqs]
587
return commonprefix([ids for ids in ids_list if ids != []])
588
589
def mark_blocks_as_computed(self, seq_group: SequenceGroup):
vllm/core/block_manager_v2.py
new
+210
@@ -0,0 +1,210 @@
1
+"""A block manager that manages token blocks."""
2
+from typing import Dict, List, Optional, Tuple
3
+
4
+from vllm.core.block.block_table import BlockTable
5
+from vllm.core.block.cpu_gpu_block_allocator import CpuGpuBlockAllocator
6
+from vllm.core.interfaces import AllocStatus, BlockSpaceManager
7
+from vllm.sequence import Sequence, SequenceGroup, SequenceStatus
8
+from vllm.utils import Device
9
+
10
+SeqId = int
11
+
12
+
13
+class BlockSpaceManagerV2(BlockSpaceManager):
14
+ """BlockSpaceManager which manages the allocation of KV cache.
15
+
16
+ It owns responsibility for allocation, swapping, allocating memory for
17
+ autoregressively-generated tokens, and other advanced features such as
18
+ prefix caching, forking/copy-on-write, and sliding-window memory allocation.
19
+
20
+ The current implementation is partial; in particular prefix caching and
21
+ sliding-window are not feature complete. This class implements the design
22
+ described in https://github.com/vllm-project/vllm/pull/3492.
23
+
24
+ Args:
25
+ block_size (int): The size of each memory block.
26
+ num_gpu_blocks (int): The number of memory blocks allocated on GPU.
27
+ num_cpu_blocks (int): The number of memory blocks allocated on CPU.
28
+ watermark (float, optional): The threshold used for memory swapping.
29
+ Defaults to 0.01.
30
+ sliding_window (Optional[int], optional): The size of the sliding
31
+ window. Defaults to None.
32
+ enable_caching (bool, optional): Flag indicating whether caching is
33
+ enabled. Defaults to False.
34
+ """
35
+
36
+ def __init__(
37
+ self,
38
+ block_size: int,
39
+ num_gpu_blocks: int,
40
+ num_cpu_blocks: int,
41
+ watermark: float = 0.01,
42
+ sliding_window: Optional[int] = None,
43
+ enable_caching: bool = False,
44
+ ) -> None:
45
+ self.block_size = block_size
46
+ self.num_total_gpu_blocks = num_gpu_blocks
47
+ self.num_total_cpu_blocks = num_cpu_blocks
48
+
49
+ assert sliding_window is None, "Sliding window not yet supported"
50
+
51
+ self.block_sliding_window = None
52
+
53
+ self.watermark = watermark
54
+ assert watermark >= 0.0
55
+
56
+ assert not enable_caching, "Prefix caching not yet supported"
57
+ self.enable_caching = enable_caching
58
+
59
+ self.watermark_blocks = int(watermark * num_gpu_blocks)
60
+
61
+ self.block_allocator = CpuGpuBlockAllocator.create(
62
+ # Currently, only naive blocks are supported (no prefix caching).
63
+ allocator_type="naive",
64
+ num_gpu_blocks=num_gpu_blocks,
65
+ num_cpu_blocks=num_cpu_blocks,
66
+ block_size=block_size,
67
+ )
68
+
69
+ self.block_tables: Dict[SeqId, BlockTable] = {}
70
+
71
+ def can_allocate(self, seq_group: SequenceGroup) -> AllocStatus:
72
+ # FIXME(woosuk): Here we assume that all sequences in the group share
73
+ # the same prompt. This may not be true for preempted sequences.
74
+ seq = seq_group.get_seqs(status=SequenceStatus.WAITING)[0]
75
+
76
+ num_required_blocks = BlockTable.get_num_required_blocks(
77
+ seq.get_token_ids(),
78
+ block_size=self.block_size,
79
+ )
80
+
81
+ assert self.block_sliding_window is None
82
+ if self.block_sliding_window is not None:
83
+ num_required_blocks = min(num_required_blocks,
84
+ self.block_sliding_window)
85
+
86
+ num_free_gpu_blocks = self.block_allocator.get_num_free_blocks(
87
+ device=Device.GPU)
88
+
89
+ # Use watermark to avoid frequent cache eviction.
90
+ if (self.num_total_gpu_blocks - num_required_blocks <
91
+ self.watermark_blocks):
92
+ return AllocStatus.NEVER
93
+ if num_free_gpu_blocks - num_required_blocks >= self.watermark_blocks:
94
+ return AllocStatus.OK
95
+ else:
96
+ return AllocStatus.LATER
97
+
98
+ def allocate(self, seq_group: SequenceGroup) -> None:
99
+ waiting_seqs = seq_group.get_seqs(status=SequenceStatus.WAITING)
100
+ assert not (set(seq.seq_id for seq in waiting_seqs)
101
+ & self.block_tables.keys()), "block table already exists"
102
+
103
+ # NOTE: Here we assume that all sequences in the group have the same
104
+ # prompt.
105
+ seq = waiting_seqs[0]
106
+
107
+ block_table = BlockTable(
108
+ block_size=self.block_size,
109
+ block_allocator=self.block_allocator,
110
+ )
111
+ assert self.block_sliding_window is None
112
+ block_table.allocate(seq.get_token_ids())
113
+ self.block_tables[seq.seq_id] = block_table
114
+
115
+ # Assign the block table for each sequence.
116
+ for seq in waiting_seqs[1:]:
117
+ self.block_tables[seq.seq_id] = block_table.fork()
118
+
119
+ def can_append_slot(self, seq_group: SequenceGroup) -> bool:
120
+ # Simple heuristic: If there is at least one free block
121
+ # for each sequence, we can append.
122
+ num_free_gpu_blocks = self.block_allocator.get_num_free_blocks(
123
+ Device.GPU)
124
+ num_seqs = seq_group.num_seqs(status=SequenceStatus.RUNNING)
125
+ return num_seqs <= num_free_gpu_blocks
126
+
127
+ def append_slot(
128
+ self,
129
+ seq: Sequence,
130
+ ) -> Optional[Tuple[int, int]]:
131
+
132
+ block_table = self.block_tables[seq.seq_id]
133
+
134
+ # Get unseen token ids.
135
+ num_full_slots = block_table.num_full_slots
136
+ unseen_token_ids = seq.get_token_ids()[num_full_slots:]
137
+ assert unseen_token_ids
138
+
139
+ block_table.append_token_ids(unseen_token_ids)
140
+
141
+ # Return any copy-on-writes.
142
+ _ = self.block_allocator.clear_copy_on_writes()
143
+
144
+ # TODO extend append_slot interface to append_slots
145
+ # @cadedaniel will do in https://github.com/vllm-project/vllm/pull/3250
146
+
147
+ return None
148
+
149
+ def free(self, seq: Sequence) -> None:
150
+ if seq.seq_id not in self.block_tables:
151
+ # Already freed or haven't been scheduled yet.
152
+ return
153
+ self.block_tables[seq.seq_id].free()
154
+ del self.block_tables[seq.seq_id]
155
+
156
+ def get_block_table(self, seq: Sequence) -> List[int]:
157
+ assert seq.seq_id in self.block_tables
158
+ block_ids = self.block_tables[seq.seq_id].physical_block_ids
159
+ assert all(b is not None for b in block_ids)
160
+ return block_ids
161
+
162
+ def access_all_blocks_in_seq(self, seq, now):
163
+ # TODO add prefix caching support.
164
+ # Tracked here https://github.com/vllm-project/vllm/issues/3667
165
+ pass
166
+
167
+ def mark_blocks_as_computed(self, seq_group: SequenceGroup):
168
+ # We ignore the sequence group as its not necessary. After the batch is
169
+ # formed by the scheduler, we do not need to mark blocks from individual
170
+ # sequence groups as computed -- all blocks in the batch can be marked
171
+ # as computed.
172
+ self.block_allocator.mark_blocks_as_computed()
173
+
174
+ def get_common_computed_block_ids(self, seqs: List[Sequence]) -> List[int]:
175
+ """Determine which blocks for which we skip prefill.
176
+
177
+ With prefix caching we can skip prefill for previously-generated blocks.
178
+ Currently, the attention implementation only supports skipping cached
179
+ blocks if they are a contiguous prefix of cached blocks.
180
+
181
+ This method determines which blocks can be safely skipped for all
182
+ sequences in the sequence group.
183
+ """
184
+ seq_block_ids = [
185
+ self.block_tables[seq.seq_id].physical_block_ids for seq in seqs
186
+ ]
187
+ return self.block_allocator.get_common_computed_block_ids(
188
+ seq_block_ids)
189
+
190
+ def fork(self, parent_seq: Sequence, child_seq: Sequence) -> None:
191
+ src_block_table = self.block_tables[parent_seq.seq_id]
192
+ self.block_tables[child_seq.seq_id] = src_block_table.fork()
193
+
194
+ def can_swap_in(self, seq_group: SequenceGroup) -> bool:
195
+ return False
196
+
197
+ def swap_in(self, seq_group: SequenceGroup) -> Dict[int, int]:
198
+ raise NotImplementedError
199
+
200
+ def can_swap_out(self, seq_group: SequenceGroup) -> bool:
201
+ return False
202
+
203
+ def swap_out(self, seq_group: SequenceGroup) -> Dict[int, int]:
204
+ raise NotImplementedError
205
+
206
+ def get_num_free_gpu_blocks(self) -> int:
207
+ return self.block_allocator.get_num_free_blocks(Device.GPU)
208
+
209
+ def get_num_free_cpu_blocks(self) -> int:
210
+ return self.block_allocator.get_num_free_blocks(Device.CPU)
vllm/core/interfaces.py
new
+107
@@ -0,0 +1,107 @@
1
+import enum
2
+from abc import ABC, abstractmethod
3
+from typing import Dict, List, Optional, Tuple
4
+
5
+from vllm.sequence import Sequence, SequenceGroup
6
+
7
+
8
+class AllocStatus(enum.Enum):
9
+ """Result for BlockSpaceManager.can_allocate
10
+
11
+ 1. Ok: seq_group can be allocated now.
12
+ 2. Later: seq_group cannot be allocated.
13
+ The capacity of allocator is larger than seq_group required.
14
+ 3. Never: seq_group can never be allocated.
15
+ The seq_group is too large to allocated in GPU.
16
+ """
17
+ OK = enum.auto()
18
+ LATER = enum.auto()
19
+ NEVER = enum.auto()
20
+
21
+
22
+class BlockSpaceManager(ABC):
23
+
24
+ @staticmethod
25
+ def get_block_space_manager_class(version: str):
26
+ version = version.lower()
27
+
28
+ if version == "v1":
29
+ from vllm.core.block_manager_v1 import BlockSpaceManagerV1
30
+ return BlockSpaceManagerV1
31
+
32
+ if version == "v2":
33
+ from vllm.core.block_manager_v2 import BlockSpaceManagerV2
34
+ return BlockSpaceManagerV2
35
+
36
+ raise ValueError(f"Unknown version {version=}")
37
+
38
+ @abstractmethod
39
+ def can_allocate(self, seq_group: SequenceGroup) -> AllocStatus:
40
+ pass
41
+
42
+ @abstractmethod
43
+ def allocate(self, seq_group: SequenceGroup) -> None:
44
+ pass
45
+
46
+ @abstractmethod
47
+ def can_append_slot(self, seq_group: SequenceGroup) -> bool:
48
+ pass
49
+
50
+ @abstractmethod
51
+ def append_slot(
52
+ self,
53
+ seq: Sequence,
54
+ ) -> Optional[Tuple[int, int]]:
55
+ pass
56
+
57
+ @abstractmethod
58
+ def fork(self, parent_seq: Sequence, child_seq: Sequence) -> None:
59
+ pass
60
+
61
+ @abstractmethod
62
+ def can_swap_in(self, seq_group: SequenceGroup) -> bool:
63
+ pass
64
+
65
+ @abstractmethod
66
+ def swap_in(self, seq_group: SequenceGroup) -> Dict[int, int]:
67
+ pass
68
+
69
+ @abstractmethod
70
+ def can_swap_out(self, seq_group: SequenceGroup) -> bool:
71
+ pass
72
+
73
+ @abstractmethod
74
+ def swap_out(self, seq_group: SequenceGroup) -> Dict[int, int]:
75
+ pass
76
+
77
+ @abstractmethod
78
+ def free(self, seq: Sequence) -> None:
79
+ pass
80
+
81
+ @abstractmethod
82
+ def get_block_table(self, seq: Sequence) -> List[int]:
83
+ pass
84
+
85
+ @abstractmethod
86
+ def get_num_free_gpu_blocks(self) -> int:
87
+ pass
88
+
89
+ @abstractmethod
90
+ def get_num_free_cpu_blocks(self) -> int:
91
+ pass
92
+
93
+ @abstractmethod
94
+ def access_all_blocks_in_seq(
95
+ self,
96
+ seq: Sequence,
97
+ access_time: float,
98
+ ) -> None:
99
+ pass
100
+
101
+ @abstractmethod
102
+ def get_common_computed_block_ids(self, seqs: List[Sequence]) -> List[int]:
103
+ pass
104
+
105
+ @abstractmethod
106
+ def mark_blocks_as_computed(self, seq_group: SequenceGroup):
107
+ pass
vllm/core/scheduler.py
+20
-7
@@ -4,7 +4,7 @@ from collections import deque
4
from typing import Deque, Dict, Iterable, List, Optional, Set, Tuple, Union
5
6
from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig
7
-from vllm.core.block_manager import AllocStatus, BlockSpaceManager
7
+from vllm.core.interfaces import AllocStatus, BlockSpaceManager
8
from vllm.core.policy import PolicyFactory
9
from vllm.logger import init_logger
10
from vllm.lora.request import LoRARequest
@@ -88,8 +88,13 @@ class Scheduler:
88
89
# Instantiate the scheduling policy.
90
self.policy = PolicyFactory.get_policy(policy_name="fcfs")
91
+
92
+ BlockSpaceManagerImpl = BlockSpaceManager.get_block_space_manager_class(
93
+ version="v2" if self.scheduler_config.
94
+ use_v2_block_manager else "v1")
95
+
96
# Create the block space manager.
92
- self.block_manager = BlockSpaceManager(
97
+ self.block_manager = BlockSpaceManagerImpl(
98
block_size=self.cache_config.block_size,
99
num_gpu_blocks=self.cache_config.num_gpu_blocks,
100
num_cpu_blocks=self.cache_config.num_cpu_blocks,
@@ -378,6 +383,10 @@ class Scheduler:
383
block_tables[seq_id] = self.block_manager.get_block_table(seq)
384
self.block_manager.access_all_blocks_in_seq(seq, now)
385
386
+ common_computed_block_nums = (
387
+ self.block_manager.get_common_computed_block_ids(
388
+ seq_group.get_seqs(status=SequenceStatus.RUNNING)))
389
+
390
seq_group_metadata = SequenceGroupMetadata(
391
request_id=seq_group.request_id,
392
is_prompt=scheduler_outputs.prompt_run,
@@ -385,8 +394,7 @@ class Scheduler:
394
sampling_params=seq_group.sampling_params,
395
block_tables=block_tables,
396
lora_request=seq_group.lora_request,
388
- computed_block_nums=self.block_manager.
389
- get_common_computed_block_ids(seq_group),
397
+ computed_block_nums=common_computed_block_nums,
398
state=seq_group.state,
399
# `multi_modal_data` will only be present for the 1st comm
400
# between engine and worker.
@@ -396,6 +404,14 @@ class Scheduler:
404
if scheduler_outputs.prompt_run else None,
405
)
406
seq_group_metadata_list.append(seq_group_metadata)
407
+
408
+ # Now that the batch has been created, we can assume all blocks in the
409
+ # batch will have been computed before the next scheduling invocation.
410
+ # This is because the engine assumes that a failure in model execution
411
+ # will crash the vLLM instance / will not retry.
412
+ for seq_group in scheduler_outputs.scheduled_seq_groups:
413
+ self.block_manager.mark_blocks_as_computed(seq_group)
414
+
415
return seq_group_metadata_list, scheduler_outputs
416
417
def fork_seq(self, parent_seq: Sequence, child_seq: Sequence) -> None:
@@ -503,9 +519,6 @@ class Scheduler:
519
for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
520
seq.status = SequenceStatus.SWAPPED
521
506
- def mark_blocks_as_computed(self, seq_group: SequenceGroup):
507
- self.block_manager.mark_blocks_as_computed(seq_group)
508
-
522
def _passed_delay(self, now: float) -> bool:
523
if self.prev_prompt:
524
self.last_prompt_latency = now - self.prev_time
vllm/engine/arg_utils.py
+15
@@ -28,6 +28,7 @@ class EngineArgs:
28
max_parallel_loading_workers: Optional[int] = None
29
block_size: int = 16
30
enable_prefix_caching: bool = False
31
+ use_v2_block_manager: bool = False
32
swap_space: int = 4 # GiB
33
gpu_memory_utilization: float = 0.90
34
max_num_batched_tokens: Optional[int] = None
@@ -52,6 +53,9 @@ class EngineArgs:
53
max_cpu_loras: Optional[int] = None
54
device: str = 'auto'
55
ray_workers_use_nsight: bool = False
56
+
57
+ forced_num_gpu_blocks: Optional[int] = None
58
+
59
# Related to Vision-language models such as llava
60
image_input_type: Optional[str] = None
61
image_token_id: Optional[int] = None
@@ -194,6 +198,9 @@ class EngineArgs:
198
parser.add_argument('--enable-prefix-caching',
199
action='store_true',
200
help='Enables automatic prefix caching')
201
+ parser.add_argument('--use-v2-block-manager',
202
+ action='store_true',
203
+ help='Use BlockSpaceMangerV2')
204
205
parser.add_argument('--seed',
206
type=int,
@@ -210,6 +217,12 @@ class EngineArgs:
217
help='the fraction of GPU memory to be used for '
218
'the model executor, which can range from 0 to 1.'
219
'If unspecified, will use the default value of 0.9.')
220
+ parser.add_argument(
221
+ '--forced-num-gpu-blocks',
222
+ type=int,
223
+ default=None,
224
+ help='If specified, ignore GPU profiling result and use this number'
225
+ 'of GPU blocks. Used for testing preemption.')
226
parser.add_argument('--max-num-batched-tokens',
227
type=int,
228
default=EngineArgs.max_num_batched_tokens,
@@ -369,6 +382,7 @@ class EngineArgs:
382
cache_config = CacheConfig(self.block_size,
383
self.gpu_memory_utilization,
384
self.swap_space, self.kv_cache_dtype,
385
+ self.forced_num_gpu_blocks,
386
model_config.get_sliding_window(),
387
self.enable_prefix_caching)
388
parallel_config = ParallelConfig(
@@ -383,6 +397,7 @@ class EngineArgs:
397
scheduler_config = SchedulerConfig(self.max_num_batched_tokens,
398
self.max_num_seqs,
399
model_config.max_model_len,
400
+ self.use_v2_block_manager,
401
self.scheduler_delay_factor)
402
lora_config = LoRAConfig(
403
max_lora_rank=self.max_lora_rank,
vllm/engine/llm_engine.py
-6
@@ -553,12 +553,6 @@ class LLMEngine:
553
# Update the scheduled sequence groups with the model outputs.
554
scheduled_seq_groups = scheduler_outputs.scheduled_seq_groups
555
556
- # If prefix caching is enabled, mark all blocks in the sequence groups
557
- # as completed so that future requests don't attempt to recompute them
558
- if self.cache_config.enable_prefix_caching:
559
- for seq_group in scheduled_seq_groups:
560
- self.scheduler.mark_blocks_as_computed(seq_group)
561
-
556
for seq_group, outputs in zip(scheduled_seq_groups, output):
557
self._process_sequence_group_outputs(seq_group, outputs)
558
vllm/executor/gpu_executor.py
+6
@@ -85,6 +85,12 @@ class GPUExecutor(ExecutorBase):
85
cache_dtype=self.cache_config.cache_dtype,
86
))
87
88
+ if self.cache_config.forced_num_gpu_blocks is not None:
89
+ forced_num_gpu_blocks = self.cache_config.forced_num_gpu_blocks
90
+ logger.info(f"Replacing profiled {num_gpu_blocks=} with "
91
+ f"{forced_num_gpu_blocks=}")
92
+ num_gpu_blocks = forced_num_gpu_blocks
93
+
94
logger.info(f"# GPU blocks: {num_gpu_blocks}, "
95
f"# CPU blocks: {num_cpu_blocks}")
96
vllm/executor/ray_gpu_executor.py
+7
@@ -232,6 +232,13 @@ class RayGPUExecutor(ExecutorBase):
232
# operators can be applied to all workers.
233
num_gpu_blocks = min(b[0] for b in num_blocks)
234
num_cpu_blocks = min(b[1] for b in num_blocks)
235
+
236
+ if self.cache_config.forced_num_gpu_blocks is not None:
237
+ forced_num_gpu_blocks = self.cache_config.forced_num_gpu_blocks
238
+ logger.info(f"Replacing profiled {num_gpu_blocks=} with "
239
+ f"{forced_num_gpu_blocks=}")
240
+ num_gpu_blocks = forced_num_gpu_blocks
241
+
242
logger.info(f"# GPU blocks: {num_gpu_blocks}, "
243
f"# CPU blocks: {num_cpu_blocks}")
244
vllm/sequence.py
+2
@@ -196,6 +196,8 @@ class Sequence:
196
return self.lora_request.lora_int_id if self.lora_request else 0
197
198
def hash_of_block(self, logical_idx: int) -> int:
199
+ # TODO This can produce incorrect hash when block size > prompt size
200
+
201
# Compute the number of tokens in the sequence
202
# TODO: The current hashing function is O(L^2). We should optimize
203
# this in the future.
vllm/utils.py
+10
@@ -227,6 +227,16 @@ def set_cuda_visible_devices(device_ids: List[int]) -> None:
227
os.environ["CUDA_VISIBLE_DEVICES"] = ",".join(map(str, device_ids))
228
229
230
+def chunk_list(lst, chunk_size):
231
+ """Yield successive chunk_size chunks from lst."""
232
+ return [lst[i:i + chunk_size] for i in range(0, len(lst), chunk_size)]
233
+
234
+
235
+def cdiv(a: int, b: int) -> int:
236
+ """Ceiling division."""
237
+ return -(a // -b)
238
+
239
+
240
@lru_cache(maxsize=None)
241
def get_nvcc_cuda_version() -> Optional[Version]:
242
cuda_home = os.environ.get('CUDA_HOME')