@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 24aecf421a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[Tests] Add block manager and scheduler tests (#3108)

SangBin Cho committed Mar 6, 2024 at 11:23 UTC 24aecf421a4ad5989697010963074904fead9a1b
5 files changed +462
.buildkite/test-pipeline.yaml
+3
@@ -13,6 +13,9 @@ steps:
13
14 - label: Basic Correctness Test
15 command: pytest -v -s --forked basic_correctness
16 +
17 +- label: Core Test
18 + command: pytest -v -s core
19
20 - label: Distributed Comm Ops Test
21 command: pytest -v -s --forked test_comm_ops.py
tests/core/__init__.py
tests/core/test_block_manager.py new
+262
@@ -0,0 +1,262 @@
1 +import pytest
2 +import time
3 +from typing import List
4 +
5 +from vllm import SamplingParams
6 +from vllm.block import PhysicalTokenBlock
7 +from vllm.core.block_manager import BlockAllocator, BlockSpaceManager, AllocStatus
8 +from vllm.utils import Device
9 +from vllm.sequence import Sequence, SequenceGroup, SequenceStatus
10 +
11 +from .utils import create_dummy_prompt
12 +
13 +
14 +def test_block_allocator_allocate():
15 + block_size = 4
16 + num_cpu_blocks = 4
17 + cpu_allocator = BlockAllocator(Device.CPU, block_size, num_cpu_blocks)
18 +
19 + # Allocate all available cpu blocks.
20 + num_free = num_cpu_blocks
21 + assert cpu_allocator.get_num_free_blocks() == num_free
22 + for _ in range(num_cpu_blocks):
23 + block = cpu_allocator.allocate()
24 + num_free -= 1
25 + assert block not in cpu_allocator.free_blocks
26 + assert cpu_allocator.get_num_free_blocks() == num_free
27 +
28 + with pytest.raises(ValueError):
29 + cpu_allocator.allocate()
30 +
31 +
32 +def test_block_allocator_free():
33 + block_size = 4
34 + num_cpu_blocks = 4
35 + cpu_allocator = BlockAllocator(Device.CPU, block_size, num_cpu_blocks)
36 +
37 + # Allocate all available cpu blocks.
38 + blocks: List[PhysicalTokenBlock] = []
39 + for _ in range(num_cpu_blocks):
40 + block = cpu_allocator.allocate()
41 + blocks.append(block)
42 + assert block not in cpu_allocator.free_blocks
43 +
44 + # Free all allocated cpu blocks.
45 + num_free = 0
46 + assert cpu_allocator.get_num_free_blocks() == num_free
47 + for block in blocks:
48 + cpu_allocator.free(block)
49 + num_free += 1
50 + assert block in cpu_allocator.free_blocks
51 + assert cpu_allocator.get_num_free_blocks() == num_free
52 +
53 + with pytest.raises(ValueError):
54 + cpu_allocator.free(block)
55 +
56 +
57 +def test_allocate():
58 + block_size = 4
59 + num_cpu_blocks = 4
60 + num_gpu_blocks = 4
61 + block_manager = BlockSpaceManager(block_size,
62 + num_cpu_blocks,
63 + num_gpu_blocks,
64 + watermark=0)
65 +
66 + # Allocate same sequence group to all available gpu blocks.
67 + for i in range(num_gpu_blocks):
68 + _, seq_group = create_dummy_prompt(str(i), block_size)
69 + assert block_manager.can_allocate(seq_group)
70 + block_manager.allocate(seq_group)
71 + assert block_manager.can_allocate(seq_group) != AllocStatus.OK
72 +
73 + # Allocate same sequence group to all available gpu blocks.
74 + # Use watermark to reserve one gpu block.
75 + block_manager = BlockSpaceManager(block_size,
76 + num_cpu_blocks,
77 + num_gpu_blocks,
78 + watermark=1 / num_gpu_blocks)
79 + for i in range(num_gpu_blocks - 1):
80 + _, seq_group = create_dummy_prompt(str(i), block_size)
81 + assert block_manager.can_allocate(seq_group)
82 + block_manager.allocate(seq_group)
83 + assert block_manager.can_allocate(seq_group) != AllocStatus.OK
84 +
85 +
86 +def test_append_slot_single_seq():
87 + block_size = 4
88 + num_cpu_blocks = 4
89 + num_gpu_blocks = 4
90 + block_manager = BlockSpaceManager(block_size,
91 + num_cpu_blocks,
92 + num_gpu_blocks,
93 + watermark=0)
94 +
95 + # Allocate single seq to gpu block.
96 + prompt, seq_group = create_dummy_prompt("1", block_size)
97 + block_manager.allocate(seq_group)
98 +
99 + # Nothing to append. Sequence has no new logical blocks.
100 + assert block_manager.can_append_slot(seq_group)
101 + before_blocks = block_manager.get_num_free_gpu_blocks()
102 + assert not block_manager.append_slot(prompt)
103 + after_blocks = block_manager.get_num_free_gpu_blocks()
104 + assert before_blocks == after_blocks
105 +
106 + # Add block_size number of new tokens and append slot.
107 + for i in range(block_size):
108 + token_id = i + 5
109 + prompt.append_token_id(token_id, {token_id: 0.0})
110 +
111 + assert block_manager.can_append_slot(seq_group)
112 + before_blocks = block_manager.get_num_free_gpu_blocks()
113 + assert not block_manager.append_slot(prompt)
114 + after_blocks = block_manager.get_num_free_gpu_blocks()
115 + assert before_blocks - after_blocks == 1
116 +
117 +
118 +def test_append_slot_cow():
119 + block_size = 4
120 + num_cpu_blocks = 4
121 + num_gpu_blocks = 4
122 + block_manager = BlockSpaceManager(block_size,
123 + num_cpu_blocks,
124 + num_gpu_blocks,
125 + watermark=0)
126 +
127 + # Allocate prompt to gpu block.
128 + prompt = Sequence(1, "one two three", [1, 2, 3], block_size)
129 + child = prompt.fork(2)
130 + token_id = 4
131 + child.append_token_id(token_id, {token_id: 0.0})
132 + seq_group = SequenceGroup("1", [prompt, child], SamplingParams(),
133 + time.time(), time.perf_counter)
134 + block_manager.allocate(seq_group)
135 +
136 + # Append slot for child token.
137 + # Last block being modified is shared. Copy on write occurs.
138 + assert block_manager.can_append_slot(seq_group)
139 + before_blocks = block_manager.get_num_free_gpu_blocks()
140 + src_block, dst_block = block_manager.append_slot(child)
141 + assert src_block != dst_block
142 +
143 + after_blocks = block_manager.get_num_free_gpu_blocks()
144 + assert before_blocks - after_blocks == 1
145 +
146 +
147 +def test_fork():
148 + block_size = 4
149 + num_cpu_blocks = 4
150 + num_gpu_blocks = 4
151 + block_manager = BlockSpaceManager(block_size,
152 + num_cpu_blocks,
153 + num_gpu_blocks,
154 + watermark=0)
155 +
156 + prompt, seq_group = create_dummy_prompt("1",
157 + block_size - 1,
158 + block_size=block_size)
159 + block_manager.allocate(seq_group)
160 +
161 + # Fork prompt and copy block tables.
162 + child = prompt.fork(2)
163 + block_manager.fork(prompt, child)
164 + assert block_manager.get_block_table(
165 + prompt) == block_manager.get_block_table(child)
166 + token_id = 4
167 + # Append token to child. Block is shared so copy on write occurs.
168 + child.append_token_id(token_id, {token_id: 0.0})
169 + block_manager.append_slot(child)
170 + assert block_manager.get_block_table(
171 + prompt) != block_manager.get_block_table(child)
172 +
173 +
174 +def test_swap():
175 + block_size = 4
176 + num_cpu_blocks = 4
177 + num_gpu_blocks = 4
178 + block_manager = BlockSpaceManager(block_size,
179 + num_cpu_blocks,
180 + num_gpu_blocks,
181 + watermark=0)
182 +
183 + prompt, seq_group = create_dummy_prompt("1", prompt_length=block_size - 1)
184 + prompt.status = SequenceStatus.WAITING
185 + block_manager.allocate(seq_group)
186 +
187 + # Emulate a forward pass by appending a single token.
188 + # The block manager then knows how many unprocessed
189 + # tokens will be written in the next forward pass.
190 + token_id = 0
191 + prompt.status = SequenceStatus.RUNNING
192 + prompt.append_token_id(token_id, {token_id: 0.0})
193 +
194 + # Swap seq group from GPU -> CPU.
195 + gpu_blocks = block_manager.get_block_table(prompt)
196 + assert block_manager.can_swap_out(seq_group)
197 + before_cpu_blocks = block_manager.get_num_free_cpu_blocks()
198 + before_gpu_blocks = block_manager.get_num_free_gpu_blocks()
199 + mapping = block_manager.swap_out(seq_group)
200 + assert list(mapping.keys()) == gpu_blocks
201 + after_cpu_blocks = block_manager.get_num_free_cpu_blocks()
202 + after_gpu_blocks = block_manager.get_num_free_gpu_blocks()
203 + assert before_cpu_blocks == after_cpu_blocks + len(gpu_blocks)
204 + assert before_gpu_blocks + len(gpu_blocks) == after_gpu_blocks
205 + prompt.status = SequenceStatus.SWAPPED
206 +
207 + # Swap seq group from CPU -> GPU.
208 + cpu_blocks = block_manager.get_block_table(prompt)
209 + assert block_manager.can_swap_in(seq_group)
210 + before_cpu_blocks = block_manager.get_num_free_cpu_blocks()
211 + before_gpu_blocks = block_manager.get_num_free_gpu_blocks()
212 + mapping = block_manager.swap_in(seq_group)
213 + assert list(mapping.keys()) == cpu_blocks
214 + after_cpu_blocks = block_manager.get_num_free_cpu_blocks()
215 + after_gpu_blocks = block_manager.get_num_free_gpu_blocks()
216 + assert before_cpu_blocks + len(cpu_blocks) == after_cpu_blocks
217 + assert before_gpu_blocks == after_gpu_blocks + len(cpu_blocks)
218 +
219 +
220 +def test_free():
221 + block_size = 4
222 + num_cpu_blocks = 4
223 + num_gpu_blocks = 4
224 + block_manager = BlockSpaceManager(block_size,
225 + num_cpu_blocks,
226 + num_gpu_blocks,
227 + watermark=0)
228 +
229 + prompt, seq_group = create_dummy_prompt("1", block_size)
230 + block_manager.allocate(seq_group)
231 +
232 + # Free allocated seq.
233 + prompt_blocks = len(block_manager.get_block_table(prompt))
234 + before_blocks = block_manager.get_num_free_gpu_blocks()
235 + block_manager.free(prompt)
236 + after_blocks = block_manager.get_num_free_gpu_blocks()
237 + assert after_blocks == before_blocks + prompt_blocks
238 +
239 + # Block table for freed seq is deleted.
240 + with pytest.raises(KeyError):
241 + block_manager.get_block_table(prompt)
242 +
243 +
244 +def test_reset():
245 + block_size = 4
246 + num_cpu_blocks = 4
247 + num_gpu_blocks = 4
248 + block_manager = BlockSpaceManager(block_size,
249 + num_cpu_blocks,
250 + num_gpu_blocks,
251 + watermark=0)
252 +
253 + # Allocate same seq group on all available gpu blocks.
254 + original_blocks = block_manager.get_num_free_gpu_blocks()
255 + for i in range(num_gpu_blocks):
256 + _, seq_group = create_dummy_prompt(str(i), block_size)
257 + block_manager.allocate(seq_group)
258 + assert block_manager.get_num_free_gpu_blocks() == 0
259 +
260 + # Resetting block manager frees all allocated blocks.
261 + block_manager.reset()
262 + assert block_manager.get_num_free_gpu_blocks() == original_blocks
tests/core/test_scheduler.py new
+170
@@ -0,0 +1,170 @@
1 +from typing import List
2 +import pytest # noqa
3 +
4 +from vllm.config import CacheConfig, SchedulerConfig
5 +from vllm.core.scheduler import Scheduler
6 +from vllm.sequence import SequenceGroup
7 +
8 +from .utils import create_dummy_prompt
9 +
10 +
11 +def test_scheduler_add_seq_group():
12 + block_size = 4
13 + scheduler_config = SchedulerConfig(100, 64, 1, 256)
14 + cache_config = CacheConfig(block_size, 1.0, 1, "auto")
15 + cache_config.num_cpu_blocks = 4
16 + cache_config.num_gpu_blocks = 4
17 + scheduler = Scheduler(scheduler_config, cache_config, None)
18 +
19 + # Add seq group to scheduler.
20 + num_seq_group = 4
21 + for i in range(num_seq_group):
22 + _, seq_group = create_dummy_prompt(str(i), block_size)
23 + scheduler.add_seq_group(seq_group)
24 + assert scheduler.get_num_unfinished_seq_groups() == i + 1
25 +
26 +
27 +def test_scheduler_abort_seq_group():
28 + block_size = 4
29 + scheduler_config = SchedulerConfig(100, 64, 1, 256)
30 + cache_config = CacheConfig(block_size, 1.0, 1, "auto")
31 + cache_config.num_cpu_blocks = 4
32 + cache_config.num_gpu_blocks = 4
33 + scheduler = Scheduler(scheduler_config, cache_config, None)
34 +
35 + # Add multiple seq groups to scheduler.
36 + num_seq_group = 4
37 + request_ids = set()
38 + for i in range(num_seq_group):
39 + _, seq_group = create_dummy_prompt(str(i), block_size)
40 + scheduler.add_seq_group(seq_group)
41 + request_ids.add(str(i))
42 +
43 + # Abort all added seq groups.
44 + assert scheduler.get_num_unfinished_seq_groups() == num_seq_group
45 + scheduler.abort_seq_group(request_ids)
46 + assert scheduler.get_num_unfinished_seq_groups() == 0
47 +
48 +
49 +def test_scheduler_schedule_simple():
50 + block_size = 4
51 + num_seq_group = 4
52 + max_model_len = 16
53 + scheduler_config = SchedulerConfig(64, num_seq_group, max_model_len, 256)
54 + cache_config = CacheConfig(block_size, 1.0, 1, "auto")
55 + cache_config.num_cpu_blocks = 8
56 + cache_config.num_gpu_blocks = 8
57 + scheduler = Scheduler(scheduler_config, cache_config, None)
58 +
59 + # Add seq groups to scheduler.
60 + running: List[SequenceGroup] = []
61 + for i in range(num_seq_group):
62 + _, seq_group = create_dummy_prompt(str(i), prompt_length=block_size)
63 + scheduler.add_seq_group(seq_group)
64 + running.append(seq_group)
65 +
66 + # Schedule seq groups prompts.
67 + seq_group_meta, out = scheduler.schedule()
68 + assert set(out.scheduled_seq_groups) == set(running)
69 + assert out.num_batched_tokens == num_seq_group * seq_group.get_seqs(
70 + )[0].get_len()
71 + assert (not out.blocks_to_copy and not out.blocks_to_swap_in
72 + and not out.blocks_to_swap_out)
73 + assert len(seq_group_meta) == num_seq_group
74 +
75 + # Schedule seq groups generation.
76 + seq_group_meta, out = scheduler.schedule()
77 + assert set(out.scheduled_seq_groups) == set(running)
78 + assert out.num_batched_tokens == num_seq_group
79 + assert (not out.blocks_to_copy and not out.blocks_to_swap_in
80 + and not out.blocks_to_swap_out)
81 + assert len(seq_group_meta) == num_seq_group
82 +
83 +
84 +def test_scheduler_schedule_preempt_abort():
85 + block_size = 4
86 + max_model_len = 16
87 + scheduler_config = SchedulerConfig(64, 2, max_model_len, 256)
88 + cache_config = CacheConfig(block_size, 1.0, 1, "auto")
89 + cache_config.num_cpu_blocks = 2
90 + cache_config.num_gpu_blocks = 2
91 + scheduler = Scheduler(scheduler_config, cache_config, None)
92 +
93 + # Add seq groups to scheduler.
94 + seq_a, seq_group_a = create_dummy_prompt("1", block_size)
95 + seq_b, seq_group_b = create_dummy_prompt("2", block_size)
96 + scheduler.add_seq_group(seq_group_a)
97 + scheduler.add_seq_group(seq_group_b)
98 +
99 + # Schedule seq groups prompts.
100 + seq_group_meta, out = scheduler.schedule()
101 + assert out.scheduled_seq_groups == [seq_group_a, seq_group_b]
102 + assert out.num_batched_tokens == seq_group_a.get_seqs()[0].get_len() * 2
103 + assert (not out.blocks_to_copy and not out.blocks_to_swap_in
104 + and not out.blocks_to_swap_out)
105 + assert len(seq_group_meta) == 2
106 + assert scheduler.get_num_unfinished_seq_groups() == 2
107 +
108 + # Append "generated" tokens, allowing the sequence to mark prompt tokens as
109 + # processed.
110 + token_id = 0
111 + seq_a.append_token_id(token_id, {token_id: 0.0})
112 + seq_b.append_token_id(token_id, {token_id: 0.0})
113 +
114 + # Schedule seq groups generation and preempt seq group b.
115 + seq_group_meta, out = scheduler.schedule()
116 + assert out.scheduled_seq_groups == [seq_group_a]
117 + assert out.num_batched_tokens == 1
118 + assert (not out.blocks_to_copy and not out.blocks_to_swap_in
119 + and not out.blocks_to_swap_out)
120 + assert len(seq_group_meta) == 1
121 + assert scheduler.get_num_unfinished_seq_groups() == 2
122 +
123 + # Abort seq group a. Re-schedule seq group b prompt with recomputation.
124 + scheduler.abort_seq_group("1")
125 + seq_group_meta, out = scheduler.schedule()
126 + assert out.scheduled_seq_groups == [seq_group_b]
127 + assert out.num_batched_tokens == seq_group_b.get_seqs()[0].get_len()
128 + assert (not out.blocks_to_copy and not out.blocks_to_swap_in
129 + and not out.blocks_to_swap_out)
130 + assert len(seq_group_meta) == 1
131 + assert scheduler.get_num_unfinished_seq_groups() == 1
132 +
133 +
134 +def test_scheduler_max_seqs():
135 + block_size = 4
136 + num_seq_group = 4
137 + max_seq_group = 2
138 + max_model_len = 16
139 + scheduler_config = SchedulerConfig(64, max_seq_group, max_model_len, 256)
140 + cache_config = CacheConfig(block_size, 1.0, 1, "auto")
141 + cache_config.num_cpu_blocks = 8
142 + cache_config.num_gpu_blocks = 8
143 + scheduler = Scheduler(scheduler_config, cache_config, None)
144 +
145 + all_seq_groups: List[SequenceGroup] = []
146 + # Add seq groups to scheduler.
147 + for i in range(num_seq_group):
148 + _, seq_group = create_dummy_prompt(str(i), prompt_length=block_size)
149 + all_seq_groups.append(seq_group)
150 +
151 + # Append 1 seq group
152 + scheduler.add_seq_group(all_seq_groups[0])
153 +
154 + # Schedule seq groups prompts.
155 + _, out = scheduler.schedule()
156 + assert set(out.scheduled_seq_groups) == set([all_seq_groups[0]])
157 +
158 + # Schedule seq groups generation.
159 + _, out = scheduler.schedule()
160 + assert set(out.scheduled_seq_groups) == set([all_seq_groups[0]])
161 +
162 + # Append 2 more seq group
163 + scheduler.add_seq_group(all_seq_groups[1])
164 + scheduler.add_seq_group(all_seq_groups[2])
165 +
166 + # Schedule seq groups prompts.
167 + # Only 1 seq group should be scheduled since max_seq_group is 2
168 + # and one is prompting.
169 + _, out = scheduler.schedule()
170 + assert set(out.scheduled_seq_groups) == set([all_seq_groups[1]])
tests/core/utils.py new
+27
@@ -0,0 +1,27 @@
1 +import time
2 +from typing import Tuple
3 +
4 +from vllm import SamplingParams
5 +from vllm.sequence import Sequence, SequenceGroup
6 +
7 +
8 +def create_dummy_prompt(
9 + request_id: str,
10 + prompt_length: int,
11 + block_size: int = None) -> Tuple[Sequence, SequenceGroup]:
12 + if not block_size:
13 + block_size = prompt_length
14 +
15 + # Create dummy prompt sequence with tokens 0...block_size-1
16 + # and prompt "0 ... block_size".
17 + prompt_tokens = list(range(prompt_length))
18 + prompt_str = " ".join([str(t) for t in prompt_tokens])
19 + prompt = Sequence(int(request_id), prompt_str, prompt_tokens, block_size)
20 + seq_group = SequenceGroup(request_id, [prompt], SamplingParams(),
21 + time.time(), None, None)
22 +
23 + return prompt, seq_group
24 +
25 +
26 +def round_up_to_next_block(seq_len: int, block_size: int) -> int:
27 + return (seq_len + block_size - 1) // block_size