A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[Tests] Add block manager and scheduler tests (#3108)
SangBin Cho committed
Mar 6, 2024 at 11:23 UTC
24aecf421a4ad5989697010963074904fead9a1b
5 files changed
+462
.buildkite/test-pipeline.yaml
+3
@@ -13,6 +13,9 @@ steps:
13
14
- label: Basic Correctness Test
15
command: pytest -v -s --forked basic_correctness
16
+
17
+- label: Core Test
18
+ command: pytest -v -s core
19
20
- label: Distributed Comm Ops Test
21
command: pytest -v -s --forked test_comm_ops.py
tests/core/__init__.py
tests/core/test_block_manager.py
new
+262
@@ -0,0 +1,262 @@
1
+import pytest
2
+import time
3
+from typing import List
4
+
5
+from vllm import SamplingParams
6
+from vllm.block import PhysicalTokenBlock
7
+from vllm.core.block_manager import BlockAllocator, BlockSpaceManager, AllocStatus
8
+from vllm.utils import Device
9
+from vllm.sequence import Sequence, SequenceGroup, SequenceStatus
10
+
11
+from .utils import create_dummy_prompt
12
+
13
+
14
+def test_block_allocator_allocate():
15
+ block_size = 4
16
+ num_cpu_blocks = 4
17
+ cpu_allocator = BlockAllocator(Device.CPU, block_size, num_cpu_blocks)
18
+
19
+ # Allocate all available cpu blocks.
20
+ num_free = num_cpu_blocks
21
+ assert cpu_allocator.get_num_free_blocks() == num_free
22
+ for _ in range(num_cpu_blocks):
23
+ block = cpu_allocator.allocate()
24
+ num_free -= 1
25
+ assert block not in cpu_allocator.free_blocks
26
+ assert cpu_allocator.get_num_free_blocks() == num_free
27
+
28
+ with pytest.raises(ValueError):
29
+ cpu_allocator.allocate()
30
+
31
+
32
+def test_block_allocator_free():
33
+ block_size = 4
34
+ num_cpu_blocks = 4
35
+ cpu_allocator = BlockAllocator(Device.CPU, block_size, num_cpu_blocks)
36
+
37
+ # Allocate all available cpu blocks.
38
+ blocks: List[PhysicalTokenBlock] = []
39
+ for _ in range(num_cpu_blocks):
40
+ block = cpu_allocator.allocate()
41
+ blocks.append(block)
42
+ assert block not in cpu_allocator.free_blocks
43
+
44
+ # Free all allocated cpu blocks.
45
+ num_free = 0
46
+ assert cpu_allocator.get_num_free_blocks() == num_free
47
+ for block in blocks:
48
+ cpu_allocator.free(block)
49
+ num_free += 1
50
+ assert block in cpu_allocator.free_blocks
51
+ assert cpu_allocator.get_num_free_blocks() == num_free
52
+
53
+ with pytest.raises(ValueError):
54
+ cpu_allocator.free(block)
55
+
56
+
57
+def test_allocate():
58
+ block_size = 4
59
+ num_cpu_blocks = 4
60
+ num_gpu_blocks = 4
61
+ block_manager = BlockSpaceManager(block_size,
62
+ num_cpu_blocks,
63
+ num_gpu_blocks,
64
+ watermark=0)
65
+
66
+ # Allocate same sequence group to all available gpu blocks.
67
+ for i in range(num_gpu_blocks):
68
+ _, seq_group = create_dummy_prompt(str(i), block_size)
69
+ assert block_manager.can_allocate(seq_group)
70
+ block_manager.allocate(seq_group)
71
+ assert block_manager.can_allocate(seq_group) != AllocStatus.OK
72
+
73
+ # Allocate same sequence group to all available gpu blocks.
74
+ # Use watermark to reserve one gpu block.
75
+ block_manager = BlockSpaceManager(block_size,
76
+ num_cpu_blocks,
77
+ num_gpu_blocks,
78
+ watermark=1 / num_gpu_blocks)
79
+ for i in range(num_gpu_blocks - 1):
80
+ _, seq_group = create_dummy_prompt(str(i), block_size)
81
+ assert block_manager.can_allocate(seq_group)
82
+ block_manager.allocate(seq_group)
83
+ assert block_manager.can_allocate(seq_group) != AllocStatus.OK
84
+
85
+
86
+def test_append_slot_single_seq():
87
+ block_size = 4
88
+ num_cpu_blocks = 4
89
+ num_gpu_blocks = 4
90
+ block_manager = BlockSpaceManager(block_size,
91
+ num_cpu_blocks,
92
+ num_gpu_blocks,
93
+ watermark=0)
94
+
95
+ # Allocate single seq to gpu block.
96
+ prompt, seq_group = create_dummy_prompt("1", block_size)
97
+ block_manager.allocate(seq_group)
98
+
99
+ # Nothing to append. Sequence has no new logical blocks.
100
+ assert block_manager.can_append_slot(seq_group)
101
+ before_blocks = block_manager.get_num_free_gpu_blocks()
102
+ assert not block_manager.append_slot(prompt)
103
+ after_blocks = block_manager.get_num_free_gpu_blocks()
104
+ assert before_blocks == after_blocks
105
+
106
+ # Add block_size number of new tokens and append slot.
107
+ for i in range(block_size):
108
+ token_id = i + 5
109
+ prompt.append_token_id(token_id, {token_id: 0.0})
110
+
111
+ assert block_manager.can_append_slot(seq_group)
112
+ before_blocks = block_manager.get_num_free_gpu_blocks()
113
+ assert not block_manager.append_slot(prompt)
114
+ after_blocks = block_manager.get_num_free_gpu_blocks()
115
+ assert before_blocks - after_blocks == 1
116
+
117
+
118
+def test_append_slot_cow():
119
+ block_size = 4
120
+ num_cpu_blocks = 4
121
+ num_gpu_blocks = 4
122
+ block_manager = BlockSpaceManager(block_size,
123
+ num_cpu_blocks,
124
+ num_gpu_blocks,
125
+ watermark=0)
126
+
127
+ # Allocate prompt to gpu block.
128
+ prompt = Sequence(1, "one two three", [1, 2, 3], block_size)
129
+ child = prompt.fork(2)
130
+ token_id = 4
131
+ child.append_token_id(token_id, {token_id: 0.0})
132
+ seq_group = SequenceGroup("1", [prompt, child], SamplingParams(),
133
+ time.time(), time.perf_counter)
134
+ block_manager.allocate(seq_group)
135
+
136
+ # Append slot for child token.
137
+ # Last block being modified is shared. Copy on write occurs.
138
+ assert block_manager.can_append_slot(seq_group)
139
+ before_blocks = block_manager.get_num_free_gpu_blocks()
140
+ src_block, dst_block = block_manager.append_slot(child)
141
+ assert src_block != dst_block
142
+
143
+ after_blocks = block_manager.get_num_free_gpu_blocks()
144
+ assert before_blocks - after_blocks == 1
145
+
146
+
147
+def test_fork():
148
+ block_size = 4
149
+ num_cpu_blocks = 4
150
+ num_gpu_blocks = 4
151
+ block_manager = BlockSpaceManager(block_size,
152
+ num_cpu_blocks,
153
+ num_gpu_blocks,
154
+ watermark=0)
155
+
156
+ prompt, seq_group = create_dummy_prompt("1",
157
+ block_size - 1,
158
+ block_size=block_size)
159
+ block_manager.allocate(seq_group)
160
+
161
+ # Fork prompt and copy block tables.
162
+ child = prompt.fork(2)
163
+ block_manager.fork(prompt, child)
164
+ assert block_manager.get_block_table(
165
+ prompt) == block_manager.get_block_table(child)
166
+ token_id = 4
167
+ # Append token to child. Block is shared so copy on write occurs.
168
+ child.append_token_id(token_id, {token_id: 0.0})
169
+ block_manager.append_slot(child)
170
+ assert block_manager.get_block_table(
171
+ prompt) != block_manager.get_block_table(child)
172
+
173
+
174
+def test_swap():
175
+ block_size = 4
176
+ num_cpu_blocks = 4
177
+ num_gpu_blocks = 4
178
+ block_manager = BlockSpaceManager(block_size,
179
+ num_cpu_blocks,
180
+ num_gpu_blocks,
181
+ watermark=0)
182
+
183
+ prompt, seq_group = create_dummy_prompt("1", prompt_length=block_size - 1)
184
+ prompt.status = SequenceStatus.WAITING
185
+ block_manager.allocate(seq_group)
186
+
187
+ # Emulate a forward pass by appending a single token.
188
+ # The block manager then knows how many unprocessed
189
+ # tokens will be written in the next forward pass.
190
+ token_id = 0
191
+ prompt.status = SequenceStatus.RUNNING
192
+ prompt.append_token_id(token_id, {token_id: 0.0})
193
+
194
+ # Swap seq group from GPU -> CPU.
195
+ gpu_blocks = block_manager.get_block_table(prompt)
196
+ assert block_manager.can_swap_out(seq_group)
197
+ before_cpu_blocks = block_manager.get_num_free_cpu_blocks()
198
+ before_gpu_blocks = block_manager.get_num_free_gpu_blocks()
199
+ mapping = block_manager.swap_out(seq_group)
200
+ assert list(mapping.keys()) == gpu_blocks
201
+ after_cpu_blocks = block_manager.get_num_free_cpu_blocks()
202
+ after_gpu_blocks = block_manager.get_num_free_gpu_blocks()
203
+ assert before_cpu_blocks == after_cpu_blocks + len(gpu_blocks)
204
+ assert before_gpu_blocks + len(gpu_blocks) == after_gpu_blocks
205
+ prompt.status = SequenceStatus.SWAPPED
206
+
207
+ # Swap seq group from CPU -> GPU.
208
+ cpu_blocks = block_manager.get_block_table(prompt)
209
+ assert block_manager.can_swap_in(seq_group)
210
+ before_cpu_blocks = block_manager.get_num_free_cpu_blocks()
211
+ before_gpu_blocks = block_manager.get_num_free_gpu_blocks()
212
+ mapping = block_manager.swap_in(seq_group)
213
+ assert list(mapping.keys()) == cpu_blocks
214
+ after_cpu_blocks = block_manager.get_num_free_cpu_blocks()
215
+ after_gpu_blocks = block_manager.get_num_free_gpu_blocks()
216
+ assert before_cpu_blocks + len(cpu_blocks) == after_cpu_blocks
217
+ assert before_gpu_blocks == after_gpu_blocks + len(cpu_blocks)
218
+
219
+
220
+def test_free():
221
+ block_size = 4
222
+ num_cpu_blocks = 4
223
+ num_gpu_blocks = 4
224
+ block_manager = BlockSpaceManager(block_size,
225
+ num_cpu_blocks,
226
+ num_gpu_blocks,
227
+ watermark=0)
228
+
229
+ prompt, seq_group = create_dummy_prompt("1", block_size)
230
+ block_manager.allocate(seq_group)
231
+
232
+ # Free allocated seq.
233
+ prompt_blocks = len(block_manager.get_block_table(prompt))
234
+ before_blocks = block_manager.get_num_free_gpu_blocks()
235
+ block_manager.free(prompt)
236
+ after_blocks = block_manager.get_num_free_gpu_blocks()
237
+ assert after_blocks == before_blocks + prompt_blocks
238
+
239
+ # Block table for freed seq is deleted.
240
+ with pytest.raises(KeyError):
241
+ block_manager.get_block_table(prompt)
242
+
243
+
244
+def test_reset():
245
+ block_size = 4
246
+ num_cpu_blocks = 4
247
+ num_gpu_blocks = 4
248
+ block_manager = BlockSpaceManager(block_size,
249
+ num_cpu_blocks,
250
+ num_gpu_blocks,
251
+ watermark=0)
252
+
253
+ # Allocate same seq group on all available gpu blocks.
254
+ original_blocks = block_manager.get_num_free_gpu_blocks()
255
+ for i in range(num_gpu_blocks):
256
+ _, seq_group = create_dummy_prompt(str(i), block_size)
257
+ block_manager.allocate(seq_group)
258
+ assert block_manager.get_num_free_gpu_blocks() == 0
259
+
260
+ # Resetting block manager frees all allocated blocks.
261
+ block_manager.reset()
262
+ assert block_manager.get_num_free_gpu_blocks() == original_blocks
tests/core/test_scheduler.py
new
+170
@@ -0,0 +1,170 @@
1
+from typing import List
2
+import pytest # noqa
3
+
4
+from vllm.config import CacheConfig, SchedulerConfig
5
+from vllm.core.scheduler import Scheduler
6
+from vllm.sequence import SequenceGroup
7
+
8
+from .utils import create_dummy_prompt
9
+
10
+
11
+def test_scheduler_add_seq_group():
12
+ block_size = 4
13
+ scheduler_config = SchedulerConfig(100, 64, 1, 256)
14
+ cache_config = CacheConfig(block_size, 1.0, 1, "auto")
15
+ cache_config.num_cpu_blocks = 4
16
+ cache_config.num_gpu_blocks = 4
17
+ scheduler = Scheduler(scheduler_config, cache_config, None)
18
+
19
+ # Add seq group to scheduler.
20
+ num_seq_group = 4
21
+ for i in range(num_seq_group):
22
+ _, seq_group = create_dummy_prompt(str(i), block_size)
23
+ scheduler.add_seq_group(seq_group)
24
+ assert scheduler.get_num_unfinished_seq_groups() == i + 1
25
+
26
+
27
+def test_scheduler_abort_seq_group():
28
+ block_size = 4
29
+ scheduler_config = SchedulerConfig(100, 64, 1, 256)
30
+ cache_config = CacheConfig(block_size, 1.0, 1, "auto")
31
+ cache_config.num_cpu_blocks = 4
32
+ cache_config.num_gpu_blocks = 4
33
+ scheduler = Scheduler(scheduler_config, cache_config, None)
34
+
35
+ # Add multiple seq groups to scheduler.
36
+ num_seq_group = 4
37
+ request_ids = set()
38
+ for i in range(num_seq_group):
39
+ _, seq_group = create_dummy_prompt(str(i), block_size)
40
+ scheduler.add_seq_group(seq_group)
41
+ request_ids.add(str(i))
42
+
43
+ # Abort all added seq groups.
44
+ assert scheduler.get_num_unfinished_seq_groups() == num_seq_group
45
+ scheduler.abort_seq_group(request_ids)
46
+ assert scheduler.get_num_unfinished_seq_groups() == 0
47
+
48
+
49
+def test_scheduler_schedule_simple():
50
+ block_size = 4
51
+ num_seq_group = 4
52
+ max_model_len = 16
53
+ scheduler_config = SchedulerConfig(64, num_seq_group, max_model_len, 256)
54
+ cache_config = CacheConfig(block_size, 1.0, 1, "auto")
55
+ cache_config.num_cpu_blocks = 8
56
+ cache_config.num_gpu_blocks = 8
57
+ scheduler = Scheduler(scheduler_config, cache_config, None)
58
+
59
+ # Add seq groups to scheduler.
60
+ running: List[SequenceGroup] = []
61
+ for i in range(num_seq_group):
62
+ _, seq_group = create_dummy_prompt(str(i), prompt_length=block_size)
63
+ scheduler.add_seq_group(seq_group)
64
+ running.append(seq_group)
65
+
66
+ # Schedule seq groups prompts.
67
+ seq_group_meta, out = scheduler.schedule()
68
+ assert set(out.scheduled_seq_groups) == set(running)
69
+ assert out.num_batched_tokens == num_seq_group * seq_group.get_seqs(
70
+ )[0].get_len()
71
+ assert (not out.blocks_to_copy and not out.blocks_to_swap_in
72
+ and not out.blocks_to_swap_out)
73
+ assert len(seq_group_meta) == num_seq_group
74
+
75
+ # Schedule seq groups generation.
76
+ seq_group_meta, out = scheduler.schedule()
77
+ assert set(out.scheduled_seq_groups) == set(running)
78
+ assert out.num_batched_tokens == num_seq_group
79
+ assert (not out.blocks_to_copy and not out.blocks_to_swap_in
80
+ and not out.blocks_to_swap_out)
81
+ assert len(seq_group_meta) == num_seq_group
82
+
83
+
84
+def test_scheduler_schedule_preempt_abort():
85
+ block_size = 4
86
+ max_model_len = 16
87
+ scheduler_config = SchedulerConfig(64, 2, max_model_len, 256)
88
+ cache_config = CacheConfig(block_size, 1.0, 1, "auto")
89
+ cache_config.num_cpu_blocks = 2
90
+ cache_config.num_gpu_blocks = 2
91
+ scheduler = Scheduler(scheduler_config, cache_config, None)
92
+
93
+ # Add seq groups to scheduler.
94
+ seq_a, seq_group_a = create_dummy_prompt("1", block_size)
95
+ seq_b, seq_group_b = create_dummy_prompt("2", block_size)
96
+ scheduler.add_seq_group(seq_group_a)
97
+ scheduler.add_seq_group(seq_group_b)
98
+
99
+ # Schedule seq groups prompts.
100
+ seq_group_meta, out = scheduler.schedule()
101
+ assert out.scheduled_seq_groups == [seq_group_a, seq_group_b]
102
+ assert out.num_batched_tokens == seq_group_a.get_seqs()[0].get_len() * 2
103
+ assert (not out.blocks_to_copy and not out.blocks_to_swap_in
104
+ and not out.blocks_to_swap_out)
105
+ assert len(seq_group_meta) == 2
106
+ assert scheduler.get_num_unfinished_seq_groups() == 2
107
+
108
+ # Append "generated" tokens, allowing the sequence to mark prompt tokens as
109
+ # processed.
110
+ token_id = 0
111
+ seq_a.append_token_id(token_id, {token_id: 0.0})
112
+ seq_b.append_token_id(token_id, {token_id: 0.0})
113
+
114
+ # Schedule seq groups generation and preempt seq group b.
115
+ seq_group_meta, out = scheduler.schedule()
116
+ assert out.scheduled_seq_groups == [seq_group_a]
117
+ assert out.num_batched_tokens == 1
118
+ assert (not out.blocks_to_copy and not out.blocks_to_swap_in
119
+ and not out.blocks_to_swap_out)
120
+ assert len(seq_group_meta) == 1
121
+ assert scheduler.get_num_unfinished_seq_groups() == 2
122
+
123
+ # Abort seq group a. Re-schedule seq group b prompt with recomputation.
124
+ scheduler.abort_seq_group("1")
125
+ seq_group_meta, out = scheduler.schedule()
126
+ assert out.scheduled_seq_groups == [seq_group_b]
127
+ assert out.num_batched_tokens == seq_group_b.get_seqs()[0].get_len()
128
+ assert (not out.blocks_to_copy and not out.blocks_to_swap_in
129
+ and not out.blocks_to_swap_out)
130
+ assert len(seq_group_meta) == 1
131
+ assert scheduler.get_num_unfinished_seq_groups() == 1
132
+
133
+
134
+def test_scheduler_max_seqs():
135
+ block_size = 4
136
+ num_seq_group = 4
137
+ max_seq_group = 2
138
+ max_model_len = 16
139
+ scheduler_config = SchedulerConfig(64, max_seq_group, max_model_len, 256)
140
+ cache_config = CacheConfig(block_size, 1.0, 1, "auto")
141
+ cache_config.num_cpu_blocks = 8
142
+ cache_config.num_gpu_blocks = 8
143
+ scheduler = Scheduler(scheduler_config, cache_config, None)
144
+
145
+ all_seq_groups: List[SequenceGroup] = []
146
+ # Add seq groups to scheduler.
147
+ for i in range(num_seq_group):
148
+ _, seq_group = create_dummy_prompt(str(i), prompt_length=block_size)
149
+ all_seq_groups.append(seq_group)
150
+
151
+ # Append 1 seq group
152
+ scheduler.add_seq_group(all_seq_groups[0])
153
+
154
+ # Schedule seq groups prompts.
155
+ _, out = scheduler.schedule()
156
+ assert set(out.scheduled_seq_groups) == set([all_seq_groups[0]])
157
+
158
+ # Schedule seq groups generation.
159
+ _, out = scheduler.schedule()
160
+ assert set(out.scheduled_seq_groups) == set([all_seq_groups[0]])
161
+
162
+ # Append 2 more seq group
163
+ scheduler.add_seq_group(all_seq_groups[1])
164
+ scheduler.add_seq_group(all_seq_groups[2])
165
+
166
+ # Schedule seq groups prompts.
167
+ # Only 1 seq group should be scheduled since max_seq_group is 2
168
+ # and one is prompting.
169
+ _, out = scheduler.schedule()
170
+ assert set(out.scheduled_seq_groups) == set([all_seq_groups[1]])
tests/core/utils.py
new
+27
@@ -0,0 +1,27 @@
1
+import time
2
+from typing import Tuple
3
+
4
+from vllm import SamplingParams
5
+from vllm.sequence import Sequence, SequenceGroup
6
+
7
+
8
+def create_dummy_prompt(
9
+ request_id: str,
10
+ prompt_length: int,
11
+ block_size: int = None) -> Tuple[Sequence, SequenceGroup]:
12
+ if not block_size:
13
+ block_size = prompt_length
14
+
15
+ # Create dummy prompt sequence with tokens 0...block_size-1
16
+ # and prompt "0 ... block_size".
17
+ prompt_tokens = list(range(prompt_length))
18
+ prompt_str = " ".join([str(t) for t in prompt_tokens])
19
+ prompt = Sequence(int(request_id), prompt_str, prompt_tokens, block_size)
20
+ seq_group = SequenceGroup(request_id, [prompt], SamplingParams(),
21
+ time.time(), None, None)
22
+
23
+ return prompt, seq_group
24
+
25
+
26
+def round_up_to_next_block(seq_len: int, block_size: int) -> int:
27
+ return (seq_len + block_size - 1) // block_size