@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a33ce60c66
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[Testing] Fix core tests (#3224)

Cade Daniel committed Mar 6, 2024 at 01:04 UTC a33ce60c6629e8c22aaf002ae8478a685e726e3e
4 files changed +36 -23
tests/core/test_block_manager.py
+31 -18
@@ -6,7 +6,7 @@ from vllm import SamplingParams
6 from vllm.block import PhysicalTokenBlock
7 from vllm.core.block_manager import BlockAllocator, BlockSpaceManager, AllocStatus
8 from vllm.utils import Device
9 -from vllm.sequence import Sequence, SequenceGroup, SequenceStatus
9 +from vllm.sequence import Sequence, SequenceGroup, SequenceStatus, Logprob
10
11 from .utils import create_dummy_prompt
12
@@ -22,7 +22,8 @@ def test_block_allocator_allocate():
22 for _ in range(num_cpu_blocks):
23 block = cpu_allocator.allocate()
24 num_free -= 1
25 - assert block not in cpu_allocator.free_blocks
25 +
26 + assert block.block_hash not in cpu_allocator.evictor
27 assert cpu_allocator.get_num_free_blocks() == num_free
28
29 with pytest.raises(ValueError):
@@ -39,7 +40,7 @@ def test_block_allocator_free():
40 for _ in range(num_cpu_blocks):
41 block = cpu_allocator.allocate()
42 blocks.append(block)
42 - assert block not in cpu_allocator.free_blocks
43 + assert block.block_hash not in cpu_allocator.evictor
44
45 # Free all allocated cpu blocks.
46 num_free = 0
@@ -47,7 +48,7 @@ def test_block_allocator_free():
48 for block in blocks:
49 cpu_allocator.free(block)
50 num_free += 1
50 - assert block in cpu_allocator.free_blocks
51 + assert block.block_hash in cpu_allocator.evictor
52 assert cpu_allocator.get_num_free_blocks() == num_free
53
54 with pytest.raises(ValueError):
@@ -106,7 +107,7 @@ def test_append_slot_single_seq():
107 # Add block_size number of new tokens and append slot.
108 for i in range(block_size):
109 token_id = i + 5
109 - prompt.append_token_id(token_id, {token_id: 0.0})
110 + prompt.append_token_id(token_id, {token_id: Logprob(0.0)})
111
112 assert block_manager.can_append_slot(seq_group)
113 before_blocks = block_manager.get_num_free_gpu_blocks()
@@ -119,25 +120,37 @@ def test_append_slot_cow():
120 block_size = 4
121 num_cpu_blocks = 4
122 num_gpu_blocks = 4
122 - block_manager = BlockSpaceManager(block_size,
123 - num_cpu_blocks,
124 - num_gpu_blocks,
123 + block_manager = BlockSpaceManager(block_size=block_size,
124 + num_cpu_blocks=num_cpu_blocks,
125 + num_gpu_blocks=num_gpu_blocks,
126 watermark=0)
127
127 - # Allocate prompt to gpu block.
128 - prompt = Sequence(1, "one two three", [1, 2, 3], block_size)
129 - child = prompt.fork(2)
130 - token_id = 4
131 - child.append_token_id(token_id, {token_id: 0.0})
128 + # Allocate prompt to gpu block. There is one slot left in the block.
129 + prompt = Sequence(seq_id=1,
130 + prompt="one two three",
131 + prompt_token_ids=[1, 2, 3],
132 + block_size=block_size)
133 +
134 + # Fork the sequence, such that a COW will be required when we append a new
135 + # token id.
136 + child = prompt.fork(new_seq_id=2)
137 +
138 + # Allocate space for the sequence group.
139 seq_group = SequenceGroup("1", [prompt, child], SamplingParams(),
140 time.time(), time.perf_counter)
141 block_manager.allocate(seq_group)
142
136 - # Append slot for child token.
137 - # Last block being modified is shared. Copy on write occurs.
143 + # Fork and append a new token id. We expect a COW to be scheduled.
144 + token_id = 4
145 + child.append_token_id(token_id, {token_id: Logprob(0.0)})
146 + block_manager.fork(prompt, child)
147 +
148 assert block_manager.can_append_slot(seq_group)
149 before_blocks = block_manager.get_num_free_gpu_blocks()
140 - src_block, dst_block = block_manager.append_slot(child)
150 +
151 + maybe_src_dst_block = block_manager.append_slot(child)
152 + assert maybe_src_dst_block is not None
153 + src_block, dst_block = maybe_src_dst_block
154 assert src_block != dst_block
155
156 after_blocks = block_manager.get_num_free_gpu_blocks()
@@ -165,7 +178,7 @@ def test_fork():
178 prompt) == block_manager.get_block_table(child)
179 token_id = 4
180 # Append token to child. Block is shared so copy on write occurs.
168 - child.append_token_id(token_id, {token_id: 0.0})
181 + child.append_token_id(token_id, {token_id: Logprob(0.0)})
182 block_manager.append_slot(child)
183 assert block_manager.get_block_table(
184 prompt) != block_manager.get_block_table(child)
@@ -189,7 +202,7 @@ def test_swap():
202 # tokens will be written in the next forward pass.
203 token_id = 0
204 prompt.status = SequenceStatus.RUNNING
192 - prompt.append_token_id(token_id, {token_id: 0.0})
205 + prompt.append_token_id(token_id, {token_id: Logprob(0.0)})
206
207 # Swap seq group from GPU -> CPU.
208 gpu_blocks = block_manager.get_block_table(prompt)
tests/core/test_scheduler.py
+3 -3
@@ -3,7 +3,7 @@ import pytest # noqa
3
4 from vllm.config import CacheConfig, SchedulerConfig
5 from vllm.core.scheduler import Scheduler
6 -from vllm.sequence import SequenceGroup
6 +from vllm.sequence import SequenceGroup, Logprob
7
8 from .utils import create_dummy_prompt
9
@@ -108,8 +108,8 @@ def test_scheduler_schedule_preempt_abort():
108 # Append "generated" tokens, allowing the sequence to mark prompt tokens as
109 # processed.
110 token_id = 0
111 - seq_a.append_token_id(token_id, {token_id: 0.0})
112 - seq_b.append_token_id(token_id, {token_id: 0.0})
111 + seq_a.append_token_id(token_id, {token_id: Logprob(0.0)})
112 + seq_b.append_token_id(token_id, {token_id: Logprob(0.0)})
113
114 # Schedule seq groups generation and preempt seq group b.
115 seq_group_meta, out = scheduler.schedule()
tests/core/utils.py
+1 -1
@@ -18,7 +18,7 @@ def create_dummy_prompt(
18 prompt_str = " ".join([str(t) for t in prompt_tokens])
19 prompt = Sequence(int(request_id), prompt_str, prompt_tokens, block_size)
20 seq_group = SequenceGroup(request_id, [prompt], SamplingParams(),
21 - time.time(), None, None)
21 + time.time(), None)
22
23 return prompt, seq_group
24
vllm/sequence.py
+1 -1
@@ -142,7 +142,7 @@ class Sequence:
142 prompt: str,
143 prompt_token_ids: List[int],
144 block_size: int,
145 - eos_token_id: int,
145 + eos_token_id: Optional[int] = None,
146 lora_request: Optional[LoRARequest] = None,
147 ) -> None:
148 self.seq_id = seq_id