A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[Testing] Fix core tests (#3224)
Cade Daniel committed
Mar 6, 2024 at 01:04 UTC
a33ce60c6629e8c22aaf002ae8478a685e726e3e
4 files changed
+36
-23
tests/core/test_block_manager.py
+31
-18
@@ -6,7 +6,7 @@ from vllm import SamplingParams
6
from vllm.block import PhysicalTokenBlock
7
from vllm.core.block_manager import BlockAllocator, BlockSpaceManager, AllocStatus
8
from vllm.utils import Device
9
-from vllm.sequence import Sequence, SequenceGroup, SequenceStatus
9
+from vllm.sequence import Sequence, SequenceGroup, SequenceStatus, Logprob
10
11
from .utils import create_dummy_prompt
12
@@ -22,7 +22,8 @@ def test_block_allocator_allocate():
22
for _ in range(num_cpu_blocks):
23
block = cpu_allocator.allocate()
24
num_free -= 1
25
- assert block not in cpu_allocator.free_blocks
25
+
26
+ assert block.block_hash not in cpu_allocator.evictor
27
assert cpu_allocator.get_num_free_blocks() == num_free
28
29
with pytest.raises(ValueError):
@@ -39,7 +40,7 @@ def test_block_allocator_free():
40
for _ in range(num_cpu_blocks):
41
block = cpu_allocator.allocate()
42
blocks.append(block)
42
- assert block not in cpu_allocator.free_blocks
43
+ assert block.block_hash not in cpu_allocator.evictor
44
45
# Free all allocated cpu blocks.
46
num_free = 0
@@ -47,7 +48,7 @@ def test_block_allocator_free():
48
for block in blocks:
49
cpu_allocator.free(block)
50
num_free += 1
50
- assert block in cpu_allocator.free_blocks
51
+ assert block.block_hash in cpu_allocator.evictor
52
assert cpu_allocator.get_num_free_blocks() == num_free
53
54
with pytest.raises(ValueError):
@@ -106,7 +107,7 @@ def test_append_slot_single_seq():
107
# Add block_size number of new tokens and append slot.
108
for i in range(block_size):
109
token_id = i + 5
109
- prompt.append_token_id(token_id, {token_id: 0.0})
110
+ prompt.append_token_id(token_id, {token_id: Logprob(0.0)})
111
112
assert block_manager.can_append_slot(seq_group)
113
before_blocks = block_manager.get_num_free_gpu_blocks()
@@ -119,25 +120,37 @@ def test_append_slot_cow():
120
block_size = 4
121
num_cpu_blocks = 4
122
num_gpu_blocks = 4
122
- block_manager = BlockSpaceManager(block_size,
123
- num_cpu_blocks,
124
- num_gpu_blocks,
123
+ block_manager = BlockSpaceManager(block_size=block_size,
124
+ num_cpu_blocks=num_cpu_blocks,
125
+ num_gpu_blocks=num_gpu_blocks,
126
watermark=0)
127
127
- # Allocate prompt to gpu block.
128
- prompt = Sequence(1, "one two three", [1, 2, 3], block_size)
129
- child = prompt.fork(2)
130
- token_id = 4
131
- child.append_token_id(token_id, {token_id: 0.0})
128
+ # Allocate prompt to gpu block. There is one slot left in the block.
129
+ prompt = Sequence(seq_id=1,
130
+ prompt="one two three",
131
+ prompt_token_ids=[1, 2, 3],
132
+ block_size=block_size)
133
+
134
+ # Fork the sequence, such that a COW will be required when we append a new
135
+ # token id.
136
+ child = prompt.fork(new_seq_id=2)
137
+
138
+ # Allocate space for the sequence group.
139
seq_group = SequenceGroup("1", [prompt, child], SamplingParams(),
140
time.time(), time.perf_counter)
141
block_manager.allocate(seq_group)
142
136
- # Append slot for child token.
137
- # Last block being modified is shared. Copy on write occurs.
143
+ # Fork and append a new token id. We expect a COW to be scheduled.
144
+ token_id = 4
145
+ child.append_token_id(token_id, {token_id: Logprob(0.0)})
146
+ block_manager.fork(prompt, child)
147
+
148
assert block_manager.can_append_slot(seq_group)
149
before_blocks = block_manager.get_num_free_gpu_blocks()
140
- src_block, dst_block = block_manager.append_slot(child)
150
+
151
+ maybe_src_dst_block = block_manager.append_slot(child)
152
+ assert maybe_src_dst_block is not None
153
+ src_block, dst_block = maybe_src_dst_block
154
assert src_block != dst_block
155
156
after_blocks = block_manager.get_num_free_gpu_blocks()
@@ -165,7 +178,7 @@ def test_fork():
178
prompt) == block_manager.get_block_table(child)
179
token_id = 4
180
# Append token to child. Block is shared so copy on write occurs.
168
- child.append_token_id(token_id, {token_id: 0.0})
181
+ child.append_token_id(token_id, {token_id: Logprob(0.0)})
182
block_manager.append_slot(child)
183
assert block_manager.get_block_table(
184
prompt) != block_manager.get_block_table(child)
@@ -189,7 +202,7 @@ def test_swap():
202
# tokens will be written in the next forward pass.
203
token_id = 0
204
prompt.status = SequenceStatus.RUNNING
192
- prompt.append_token_id(token_id, {token_id: 0.0})
205
+ prompt.append_token_id(token_id, {token_id: Logprob(0.0)})
206
207
# Swap seq group from GPU -> CPU.
208
gpu_blocks = block_manager.get_block_table(prompt)
tests/core/test_scheduler.py
+3
-3
@@ -3,7 +3,7 @@ import pytest # noqa
3
4
from vllm.config import CacheConfig, SchedulerConfig
5
from vllm.core.scheduler import Scheduler
6
-from vllm.sequence import SequenceGroup
6
+from vllm.sequence import SequenceGroup, Logprob
7
8
from .utils import create_dummy_prompt
9
@@ -108,8 +108,8 @@ def test_scheduler_schedule_preempt_abort():
108
# Append "generated" tokens, allowing the sequence to mark prompt tokens as
109
# processed.
110
token_id = 0
111
- seq_a.append_token_id(token_id, {token_id: 0.0})
112
- seq_b.append_token_id(token_id, {token_id: 0.0})
111
+ seq_a.append_token_id(token_id, {token_id: Logprob(0.0)})
112
+ seq_b.append_token_id(token_id, {token_id: Logprob(0.0)})
113
114
# Schedule seq groups generation and preempt seq group b.
115
seq_group_meta, out = scheduler.schedule()
tests/core/utils.py
+1
-1
@@ -18,7 +18,7 @@ def create_dummy_prompt(
18
prompt_str = " ".join([str(t) for t in prompt_tokens])
19
prompt = Sequence(int(request_id), prompt_str, prompt_tokens, block_size)
20
seq_group = SequenceGroup(request_id, [prompt], SamplingParams(),
21
- time.time(), None, None)
21
+ time.time(), None)
22
23
return prompt, seq_group
24
vllm/sequence.py
+1
-1
@@ -142,7 +142,7 @@ class Sequence:
142
prompt: str,
143
prompt_token_ids: List[int],
144
block_size: int,
145
- eos_token_id: int,
145
+ eos_token_id: Optional[int] = None,
146
lora_request: Optional[LoRARequest] = None,
147
) -> None:
148
self.seq_id = seq_id