@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 93deb0b38f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Speculative decoding 4/9] Lookahead scheduling for speculative decoding (#3250)

Cade Daniel committed Apr 1, 2024 at 15:55 UTC 93deb0b38f8c11a67fc559957661ceac7172dbfb
13 files changed +579 -123
tests/core/block/e2e/test_correctness.py
+153
@@ -77,6 +77,159 @@ def test_v1_v2_greedy_equality_with_preemption(baseline_llm_generator,
77 assert baseline_token_ids == test_token_ids
78
79
80 +@pytest.mark.parametrize(
81 + "common_llm_kwargs",
82 + [{
83 + # Use a small model for a fast test.
84 + "model": "facebook/opt-125m",
85 +
86 + # skip cuda graph creation for fast test.
87 + "enforce_eager": True,
88 +
89 + # Use a large block size to trigger more copy-on-writes.
90 + "block_size": 32,
91 + }])
92 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
93 +@pytest.mark.parametrize("baseline_llm_kwargs", [{
94 + "use_v2_block_manager": False
95 +}])
96 +@pytest.mark.parametrize("test_llm_kwargs", [{"use_v2_block_manager": True}])
97 +@pytest.mark.parametrize("batch_size", [10])
98 +@pytest.mark.parametrize("seed", [1])
99 +def test_v1_v2_greedy_equality_with_cow(baseline_llm_generator,
100 + test_llm_generator, batch_size):
101 + """Verify beam search equality with block manager v1 and v2.
102 +
103 + This requires copy-on-writes; if the v1 and v2 output is the same, then
104 + we have some confidence cow is working.
105 + """
106 + output_len = 128
107 + temperature = 0.0
108 +
109 + prompts = [
110 + "Hello, my name is",
111 + "The president of the United States is",
112 + "The capital of France is",
113 + "The future of AI is",
114 + ]
115 +
116 + prompts = [prompt for prompt, _ in zip(cycle(prompts), range(batch_size))]
117 +
118 + sampling_params = SamplingParams(
119 + max_tokens=output_len,
120 + ignore_eos=True,
121 + temperature=temperature,
122 + use_beam_search=True,
123 + best_of=2,
124 + )
125 +
126 + print('Getting token ids from block manager v1')
127 + baseline_token_ids = get_token_ids_from_llm_generator(
128 + baseline_llm_generator, prompts, sampling_params)
129 +
130 + print('Getting token ids from block manager v2')
131 + test_token_ids = get_token_ids_from_llm_generator(test_llm_generator,
132 + prompts, sampling_params)
133 +
134 + for expected_token_ids, actual_token_ids in zip(baseline_token_ids,
135 + test_token_ids):
136 + assert expected_token_ids == actual_token_ids
137 +
138 + assert baseline_token_ids == test_token_ids
139 +
140 +
141 +@pytest.mark.parametrize(
142 + "common_llm_kwargs",
143 + [{
144 + # Use a small model for a fast test.
145 + "model": "facebook/opt-125m",
146 +
147 + # Our prompts will generate 128 tokens; since the prompts themselves are
148 + # small, we don't need much KV space beyond 128.
149 + "max_model_len": 160,
150 +
151 + # skip cuda graph creation for fast test.
152 + "enforce_eager": True,
153 +
154 + # Lookahead scheduling only supported in v2 block manager.
155 + "use_v2_block_manager": True,
156 + }])
157 +@pytest.mark.parametrize(
158 + "per_test_common_llm_kwargs",
159 + [
160 + {
161 + "block_size": 16,
162 +
163 + # Allow only 2 sequences of ~128 tokens in worst case.
164 + # Note 8 = 128/block_size
165 + "forced_num_gpu_blocks": 2 * (8 + 1),
166 + },
167 + {
168 + "block_size": 8,
169 +
170 + # Allow only 2 sequences of ~128 tokens in worst case.
171 + # Note 16 = 128/block_size
172 + "forced_num_gpu_blocks": 2 * (16 + 1),
173 + }
174 + ])
175 +@pytest.mark.parametrize("baseline_llm_kwargs", [{
176 + "num_lookahead_slots": 0,
177 +}])
178 +@pytest.mark.parametrize(
179 + "test_llm_kwargs",
180 + [{
181 + # We run one test with block_size < lookahead_slots, one test with
182 + # block_size > lookahead_slots
183 + "num_lookahead_slots": 10,
184 + }])
185 +@pytest.mark.parametrize("batch_size", [4])
186 +@pytest.mark.parametrize("seed", [1])
187 +def test_lookahead_greedy_equality_with_preemption(baseline_llm_generator,
188 + test_llm_generator,
189 + batch_size):
190 + """Verify vLLM produces the same output with greedy sampling, when lookahead
191 + scheduling is used vs. not.
192 +
193 + Lookahead scheduling is not expected to modify the output, as it simply
194 + allocates empty slots ahead of the known token ids in a sliding fashion.
195 +
196 + This test constrains the total number of blocks to force preemption. It also
197 + varies the block size so that the lookahead size is less than and greater
198 + than the block size.
199 + """
200 + output_len = 128
201 + temperature = 0.0
202 +
203 + prompts = [
204 + "Hello, my name is",
205 + "The president of the United States is",
206 + "The capital of France is",
207 + "The future of AI is",
208 + ]
209 +
210 + prompts = [prompt for prompt, _ in zip(cycle(prompts), range(batch_size))]
211 +
212 + sampling_params = SamplingParams(
213 + max_tokens=output_len,
214 + ignore_eos=True,
215 + temperature=temperature,
216 + )
217 +
218 + print('Getting token ids without lookahead scheduling')
219 + baseline_token_ids = get_token_ids_from_llm_generator(
220 + baseline_llm_generator, prompts, sampling_params)
221 +
222 + print('Getting token ids with lookahead scheduling')
223 + test_token_ids = get_token_ids_from_llm_generator(test_llm_generator,
224 + prompts, sampling_params)
225 +
226 + for expected_token_ids, actual_token_ids in zip(baseline_token_ids,
227 + test_token_ids):
228 + assert expected_token_ids == actual_token_ids
229 +
230 + assert baseline_token_ids == test_token_ids
231 +
232 +
233 def get_token_ids_from_llm_generator(llm_generator, prompts, sampling_params):
234 for llm in llm_generator:
235 outputs = llm.generate(prompts, sampling_params, use_tqdm=True)
tests/core/block/test_block_manager_v2.py new
+103
@@ -0,0 +1,103 @@
1 +import pytest
2 +
3 +from vllm.core.block_manager_v2 import BlockSpaceManagerV2
4 +from vllm.core.interfaces import AllocStatus
5 +from vllm.sequence import Logprob, SequenceStatus
6 +from vllm.utils import chunk_list
7 +
8 +from ..utils import create_seq_group
9 +
10 +
11 +@pytest.mark.parametrize("block_size", [16])
12 +@pytest.mark.parametrize("num_gpu_blocks", [8, 40, 80])
13 +@pytest.mark.parametrize("num_seqs_per_group", [1, 4])
14 +@pytest.mark.parametrize("watermark", [0.0, 0.5])
15 +def test_can_allocate_seq_group(block_size: int, num_seqs_per_group: int,
16 + num_gpu_blocks: int, watermark: float):
17 + block_manager = BlockSpaceManagerV2(
18 + block_size=block_size,
19 + num_gpu_blocks=num_gpu_blocks,
20 + num_cpu_blocks=1024,
21 + watermark=watermark,
22 + )
23 + num_watermark_blocks = int(watermark * num_gpu_blocks)
24 +
25 + num_output_blocks_per_seq = 1
26 +
27 + # NOTE: This should be num_output_blocks_per_seq * num_seqs_per_group, but
28 + # the current implementation assumes all seqs are new prompts / don't have
29 + # different output lens.
30 + num_output_blocks = num_output_blocks_per_seq
31 +
32 + for num_prompt_blocks in range(1, num_gpu_blocks - num_output_blocks):
33 + seq_group = create_seq_group(
34 + seq_prompt_len=block_size * num_prompt_blocks,
35 + seq_output_lens=[
36 + block_size * num_output_blocks_per_seq
37 + for _ in range(num_seqs_per_group)
38 + ],
39 + )
40 +
41 + assert num_prompt_blocks + num_output_blocks <= num_gpu_blocks
42 +
43 + can_allocate_result = block_manager.can_allocate(seq_group)
44 +
45 + num_required_blocks = num_prompt_blocks + num_output_blocks
46 +
47 + if num_gpu_blocks - num_required_blocks < num_watermark_blocks:
48 + assert can_allocate_result == AllocStatus.NEVER
49 + elif num_gpu_blocks >= num_required_blocks:
50 + assert can_allocate_result == AllocStatus.OK
51 + else:
52 + assert can_allocate_result == AllocStatus.LATER
53 +
54 +
55 +@pytest.mark.parametrize("block_size", [1, 8])
56 +@pytest.mark.parametrize("prompt_len", [1, 7, 8])
57 +@pytest.mark.parametrize("num_slots_to_append", [1, 8, 129])
58 +@pytest.mark.parametrize("num_lookahead_slots", [0, 10])
59 +def test_append_slots(block_size, prompt_len, num_slots_to_append,
60 + num_lookahead_slots):
61 + """Verify append_slots consumes the correct number of blocks from the block
62 + table.
63 + """
64 +
65 + num_gpu_blocks = 1024
66 + watermark = 0.1
67 + block_manager = BlockSpaceManagerV2(
68 + block_size=block_size,
69 + num_gpu_blocks=num_gpu_blocks,
70 + num_cpu_blocks=0,
71 + watermark=watermark,
72 + )
73 +
74 + seq_group = create_seq_group(
75 + seq_prompt_len=prompt_len,
76 + seq_output_lens=[0],
77 + )
78 +
79 + # Allocate seq
80 + assert block_manager.can_allocate(seq_group)
81 + block_manager.allocate(seq_group)
82 +
83 + # Seq seq to RUNNING
84 + seq = seq_group.get_seqs()[0]
85 + seq.status = SequenceStatus.RUNNING
86 +
87 + # Append tokens to the sequeqnce
88 + for token_id in range(num_slots_to_append):
89 + seq.append_token_id(token_id, {token_id: Logprob(0.0)})
90 +
91 + # Append slots for new tokens and lookahead slots.
92 + free_blocks_before_append = block_manager.get_num_free_gpu_blocks()
93 + block_manager.append_slots(seq, num_lookahead_slots)
94 + num_consumed_blocks = (free_blocks_before_append -
95 + block_manager.get_num_free_gpu_blocks())
96 +
97 + # Expect consumed blocks to be new blocks required to support the new slots.
98 + expected_consumed_blocks = len(
99 + chunk_list(
100 + list(
101 + range(prompt_len + num_slots_to_append + num_lookahead_slots)),
102 + block_size)) - len(chunk_list(list(range(prompt_len)), block_size))
103 + assert num_consumed_blocks == expected_consumed_blocks
tests/core/block/test_block_space_manager.py deleted
-50
@@ -1,50 +0,0 @@
1 -import pytest
2 -
3 -from vllm.core.block_manager_v2 import BlockSpaceManagerV2
4 -from vllm.core.interfaces import AllocStatus
5 -
6 -from ..utils import create_seq_group
7 -
8 -
9 -@pytest.mark.parametrize("block_size", [16])
10 -@pytest.mark.parametrize("num_gpu_blocks", [8, 40, 80])
11 -@pytest.mark.parametrize("num_seqs_per_group", [1, 4])
12 -@pytest.mark.parametrize("watermark", [0.0, 0.5])
13 -def test_can_allocate_seq_group(block_size: int, num_seqs_per_group: int,
14 - num_gpu_blocks: int, watermark: float):
15 - block_manager = BlockSpaceManagerV2(
16 - block_size=block_size,
17 - num_gpu_blocks=num_gpu_blocks,
18 - num_cpu_blocks=1024,
19 - watermark=watermark,
20 - )
21 - num_watermark_blocks = int(watermark * num_gpu_blocks)
22 -
23 - num_output_blocks_per_seq = 1
24 -
25 - # NOTE: This should be num_output_blocks_per_seq * num_seqs_per_group, but
26 - # the current implementation assumes all seqs are new prompts / don't have
27 - # different output lens.
28 - num_output_blocks = num_output_blocks_per_seq
29 -
30 - for num_prompt_blocks in range(1, num_gpu_blocks - num_output_blocks):
31 - seq_group = create_seq_group(
32 - seq_prompt_lens=block_size * num_prompt_blocks,
33 - seq_output_lens=[
34 - block_size * num_output_blocks_per_seq
35 - for _ in range(num_seqs_per_group)
36 - ],
37 - )
38 -
39 - assert num_prompt_blocks + num_output_blocks <= num_gpu_blocks
40 -
41 - can_allocate_result = block_manager.can_allocate(seq_group)
42 -
43 - num_required_blocks = num_prompt_blocks + num_output_blocks
44 -
45 - if num_gpu_blocks - num_required_blocks < num_watermark_blocks:
46 - assert can_allocate_result == AllocStatus.NEVER
47 - elif num_gpu_blocks >= num_required_blocks:
48 - assert can_allocate_result == AllocStatus.OK
49 - else:
50 - assert can_allocate_result == AllocStatus.LATER
tests/core/block/test_block_table.py
+75
@@ -498,3 +498,78 @@ def test_cow_lookahead_simple(block_size: int, sequence_len: int,
498
499 # After free, expect all blocks to be freed.
500 assert allocator.get_num_free_blocks(Device.GPU) == num_gpu_blocks
501 +
502 +
503 +@pytest.mark.parametrize("block_size", [1, 8])
504 +@pytest.mark.parametrize("sequence_len", [1, 16, 129])
505 +@pytest.mark.parametrize("num_new_tokens", [1, 16, 129])
506 +@pytest.mark.parametrize("num_lookahead_slots", [1, 7, 8])
507 +@pytest.mark.parametrize("allocator_type", ["naive", "prefix_caching"])
508 +def test_num_blocks_touched_by_append_slots(block_size: int, sequence_len: int,
509 + num_new_tokens: int,
510 + num_lookahead_slots: int,
511 + allocator_type: str):
512 + """Verify correct calculation of get_num_blocks_touched_by_append_slots.
513 +
514 + This is done by using copy-on-write, which requires any modified block to
515 + be copied before write if the refcount > 1. We set the refcount>1 by forking
516 + a sequence, then measure the free blocks before and after an append. If the
517 + number of consumed blocks equals what `get_num_blocks_touched_by_append_
518 + slots` returns, then the calculation is correct.
519 + """
520 +
521 + num_gpu_blocks = 1024
522 +
523 + allocator = CpuGpuBlockAllocator.create(
524 + allocator_type=allocator_type,
525 + num_gpu_blocks=num_gpu_blocks,
526 + num_cpu_blocks=0,
527 + block_size=block_size,
528 + )
529 +
530 + token_ids = list(range(sequence_len))
531 + token_ids_to_append = list(range(num_new_tokens))
532 +
533 + block_table = BlockTable(
534 + block_size=block_size,
535 + block_allocator=allocator,
536 + )
537 +
538 + block_table.allocate(token_ids=token_ids, device=Device.GPU)
539 +
540 + # Add lookahead before fork so both sequences have the same lookahead
541 + # blocks.
542 + block_table.ensure_num_empty_slots(num_empty_slots=num_lookahead_slots)
543 +
544 + # Fork sequence so that every block has refcount > 1.
545 + _ = block_table.fork()
546 +
547 + # Determine how many blocks should be touched.
548 + expected_num_touched_blocks = (
549 + block_table.get_num_blocks_touched_by_append_slots(
550 + token_ids=token_ids_to_append,
551 + num_lookahead_slots=num_lookahead_slots))
552 +
553 + # Measure how many blocks are touched by measuring num_free_blocks before
554 + # and after the append.
555 + #
556 + # We expect append_token_ids to CoW all mutated blocks that have refcount>1.
557 + num_free_blocks_before_append = allocator.get_num_free_blocks(Device.GPU)
558 + block_table.append_token_ids(token_ids_to_append, num_lookahead_slots)
559 + num_consumed_blocks = (num_free_blocks_before_append -
560 + allocator.get_num_free_blocks(Device.GPU))
561 +
562 + # TODO(cade) ensure equality when num_lookahead_slots > 0.
563 + # The reason we have < is because lookahead blocks are not copied eagerly;
564 + # they are copied on first write. This will cause issues for beam search +
565 + # speculative decoding. This is acceptable for now as it is a large effort
566 + # to combine the two. To fix this, we can ensure single sequence ownership
567 + # of lookahead blocks by appending empty slots to each block, which will
568 + # trigger the CoW.
569 + #
570 + # Until then, we can accept that the consumed tokens are <= the expected
571 + # tokens when appending with lookahead.
572 + if num_lookahead_slots > 0:
573 + assert num_consumed_blocks <= expected_num_touched_blocks
574 + else:
575 + assert num_consumed_blocks == expected_num_touched_blocks
tests/core/test_block_manager.py
+12 -12
@@ -103,9 +103,9 @@ def test_append_slot_single_seq():
103 block_manager.allocate(seq_group)
104
105 # Nothing to append. Sequence has no new logical blocks.
106 - assert block_manager.can_append_slot(seq_group)
106 + assert block_manager.can_append_slots(seq_group)
107 before_blocks = block_manager.get_num_free_gpu_blocks()
108 - assert not block_manager.append_slot(prompt)
108 + assert not block_manager.append_slots(prompt)
109 after_blocks = block_manager.get_num_free_gpu_blocks()
110 assert before_blocks == after_blocks
111
@@ -114,9 +114,9 @@ def test_append_slot_single_seq():
114 token_id = i + 5
115 prompt.append_token_id(token_id, {token_id: Logprob(0.0)})
116
117 - assert block_manager.can_append_slot(seq_group)
117 + assert block_manager.can_append_slots(seq_group)
118 before_blocks = block_manager.get_num_free_gpu_blocks()
119 - assert not block_manager.append_slot(prompt)
119 + assert not block_manager.append_slots(prompt)
120 after_blocks = block_manager.get_num_free_gpu_blocks()
121 assert before_blocks - after_blocks == 1
122
@@ -150,13 +150,13 @@ def test_append_slot_cow():
150 child.append_token_id(token_id, {token_id: Logprob(0.0)})
151 block_manager.fork(prompt, child)
152
153 - assert block_manager.can_append_slot(seq_group)
153 + assert block_manager.can_append_slots(seq_group)
154 before_blocks = block_manager.get_num_free_gpu_blocks()
155
156 - maybe_src_dst_block = block_manager.append_slot(child)
157 - assert maybe_src_dst_block is not None
158 - src_block, dst_block = maybe_src_dst_block
159 - assert src_block != dst_block
156 + cows = block_manager.append_slots(child)
157 + assert cows
158 + for src_block, dst_blocks in cows.items():
159 + assert src_block not in dst_blocks
160
161 after_blocks = block_manager.get_num_free_gpu_blocks()
162 assert before_blocks - after_blocks == 1
@@ -184,7 +184,7 @@ def test_fork():
184 token_id = 4
185 # Append token to child. Block is shared so copy on write occurs.
186 child.append_token_id(token_id, {token_id: Logprob(0.0)})
187 - block_manager.append_slot(child)
187 + block_manager.append_slots(child)
188 assert block_manager.get_block_table(
189 prompt) != block_manager.get_block_table(child)
190
@@ -325,7 +325,7 @@ def test_sliding_window_multi_seq():
325 token_id = 4
326 # Append token to child. Block is shared so copy on write occurs.
327 child.append_token_id(token_id, {token_id: Logprob(0.0)})
328 - block_manager.append_slot(child)
328 + block_manager.append_slots(child)
329
330 # assert the number of blocks allocated is correct
331 # we will use now one block more. Each seq will use 2 blocks,
@@ -335,7 +335,7 @@ def test_sliding_window_multi_seq():
335
336 token_id = 5
337 parent.append_token_id(token_id, {token_id: Logprob(0.0)})
338 - block_manager.append_slot(parent)
338 + block_manager.append_slots(parent)
339
340 # assert the number of blocks allocated is correct
341 # no change, because both sequences are still just sharing one block
tests/core/utils.py
+2 -2
@@ -24,7 +24,7 @@ def create_dummy_prompt(
24
25
26 def create_seq_group(
27 - seq_prompt_lens=1024,
27 + seq_prompt_len=1024,
28 seq_output_lens=(128, ),
29 request_id='0',
30 seq_id_start=0,
@@ -32,7 +32,7 @@ def create_seq_group(
32
33 assert len(seq_output_lens) > 0
34
35 - prompt_token_ids = [0] * seq_prompt_lens
35 + prompt_token_ids = [0] * seq_prompt_len
36
37 seqs = []
38 for seq_id_offset, output_len in enumerate(seq_output_lens):
vllm/config.py
+16 -2
@@ -530,9 +530,13 @@ class SchedulerConfig:
530 iteration.
531 max_model_len: Maximum length of a sequence (including prompt
532 and generated text).
533 + use_v2_block_manager: Whether to use the BlockSpaceManagerV2 or not.
534 + num_lookahead_slots: The number of slots to allocate per sequence per
535 + step, beyond the known token ids. This is used in speculative
536 + decoding to store KV activations of tokens which may or may not be
537 + accepted.
538 delay_factor: Apply a delay (of delay factor multiplied by previous
539 prompt latency) before scheduling next prompt.
535 - use_v2_block_manager: Whether to use the BlockSpaceManagerV2 or not.
540 enable_chunked_prefill: If True, prefill requests can be chunked based
541 on the remaining max_num_batched_tokens.
542 """
@@ -543,6 +547,7 @@ class SchedulerConfig:
547 max_num_seqs: int,
548 max_model_len: int,
549 use_v2_block_manager: bool = False,
550 + num_lookahead_slots: int = 0,
551 delay_factor: float = 0.0,
552 enable_chunked_prefill: bool = False,
553 ) -> None:
@@ -554,9 +559,11 @@ class SchedulerConfig:
559 self.max_num_batched_tokens = max(max_model_len, 2048)
560 self.max_num_seqs = max_num_seqs
561 self.max_model_len = max_model_len
557 - self.delay_factor = delay_factor
562 self.use_v2_block_manager = use_v2_block_manager
563 + self.num_lookahead_slots = num_lookahead_slots
564 + self.delay_factor = delay_factor
565 self.chunked_prefill_enabled = enable_chunked_prefill
566 +
567 self._verify_args()
568
569 def _verify_args(self) -> None:
@@ -568,12 +575,19 @@ class SchedulerConfig:
575 "max_num_batched_tokens and makes vLLM reject longer "
576 "sequences. Please increase max_num_batched_tokens or "
577 "decrease max_model_len.")
578 +
579 if self.max_num_batched_tokens < self.max_num_seqs:
580 raise ValueError(
581 f"max_num_batched_tokens ({self.max_num_batched_tokens}) must "
582 "be greater than or equal to max_num_seqs "
583 f"({self.max_num_seqs}).")
584
585 + if self.num_lookahead_slots < 0:
586 + raise ValueError(
587 + "num_lookahead_slots "
588 + f"({self.num_lookahead_slots}) must be greater than or "
589 + "equal to 0.")
590 +
591
592 class DeviceConfig:
593
vllm/core/block/block_table.py
+52 -6
@@ -85,7 +85,9 @@ class BlockTable:
85 device=device)
86 self._num_full_slots = len(token_ids)
87
88 - def append_token_ids(self, token_ids: List[int]) -> None:
88 + def append_token_ids(self,
89 + token_ids: List[int],
90 + num_lookahead_slots: int = 0) -> None:
91 """Appends a sequence of token IDs to the existing blocks in the
92 BlockTable.
93
@@ -102,14 +104,13 @@ class BlockTable:
104 token_ids (List[int]): The sequence of token IDs to be appended.
105 """
106 assert self._is_allocated
107 + assert token_ids, "can't append empty token ids"
108
106 - self.ensure_num_empty_slots(num_empty_slots=len(token_ids))
109 + self.ensure_num_empty_slots(num_empty_slots=len(token_ids) +
110 + num_lookahead_slots)
111
112 blocks = self._blocks[self._num_full_slots // self._block_size:]
109 - first_chunk_size = self._block_size - (self._num_full_slots %
110 - self._block_size)
111 - token_blocks = [token_ids[:first_chunk_size]] + chunk_list(
112 - token_ids[first_chunk_size:], self._block_size)
113 + token_blocks = self._chunk_token_blocks_for_append(token_ids)
114
115 for block, token_block in zip(blocks, token_blocks):
116 block.append_token_ids(token_block)
@@ -195,6 +196,25 @@ class BlockTable:
196 assert self._is_allocated
197 return [block.block_id for block in self._blocks]
198
199 + def get_unseen_token_ids(self, sequence_token_ids: List[int]) -> List[int]:
200 + """Get the number of "unseen" tokens in the sequence.
201 +
202 + Unseen tokens are tokens in the sequence corresponding to this block
203 + table, but are not yet appended to this block table.
204 +
205 + Args:
206 + sequence_token_ids (List[int]): The list of token ids in the
207 + sequence.
208 +
209 + Returns:
210 + List[int]: The postfix of sequence_token_ids that has not yet been
211 + appended to the block table.
212 + """
213 +
214 + # Since the block table is append-only, the unseen token ids are the
215 + # ones after the appended ones.
216 + return sequence_token_ids[self.num_full_slots:]
217 +
218 def _allocate_blocks_for_token_ids(self, prev_block: Optional[Block],
219 token_ids: List[int],
220 device: Device) -> List[Block]:
@@ -243,3 +263,29 @@ class BlockTable:
263 int: The total number of tokens currently stored in the BlockTable.
264 """
265 return self._num_full_slots
266 +
267 + def get_num_blocks_touched_by_append_slots(
268 + self, token_ids: List[int], num_lookahead_slots: int) -> int:
269 + """Determine how many blocks will be "touched" by appending the token
270 + ids.
271 +
272 + This is required for the scheduler to determine whether a sequence can
273 + continue generation, or if it must be preempted.
274 + """
275 +
276 + all_token_ids = token_ids + [-1] * num_lookahead_slots
277 + token_blocks = self._chunk_token_blocks_for_append(all_token_ids)
278 + return len(token_blocks)
279 +
280 + def _chunk_token_blocks_for_append(
281 + self, token_ids: List[int]) -> List[List[int]]:
282 + """Split the token ids into block-sized chunks so they can be easily
283 + appended to blocks. The first such "token block" may have less token ids
284 + than the block size, since the last allocated block may be partially
285 + full.
286 + """
287 + first_chunk_size = self._block_size - (self._num_full_slots %
288 + self._block_size)
289 + token_blocks = [token_ids[:first_chunk_size]] + chunk_list(
290 + token_ids[first_chunk_size:], self._block_size)
291 + return token_blocks
vllm/core/block_manager_v1.py
+24 -9
@@ -2,7 +2,7 @@
2 from abc import ABC, abstractmethod
3 from itertools import count, takewhile
4 from os.path import commonprefix
5 -from typing import Dict, List, Optional, Set, Tuple
5 +from typing import Dict, List, Optional, Set
6
7 from vllm.block import BlockTable, PhysicalTokenBlock
8 from vllm.core.evictor import EvictionPolicy, Evictor, make_evictor
@@ -292,7 +292,12 @@ class BlockSpaceManagerV1(BlockSpaceManager):
292 for seq in seq_group.get_seqs(status=SequenceStatus.WAITING):
293 self.block_tables[seq.seq_id] = block_table.copy()
294
295 - def can_append_slot(self, seq_group: SequenceGroup) -> bool:
295 + def can_append_slots(self,
296 + seq_group: SequenceGroup,
297 + num_lookahead_slots: int = 0) -> bool:
298 + assert (num_lookahead_slots == 0
299 + ), "lookahead allocation not supported in BlockSpaceManagerV1"
300 +
301 # Simple heuristic: If there is at least one free block
302 # for each sequence, we can append.
303 num_free_gpu_blocks = self.gpu_allocator.get_num_free_blocks()
@@ -364,10 +369,11 @@ class BlockSpaceManagerV1(BlockSpaceManager):
369 assert new_block.ref_count == 1
370 return new_block
371
367 - def append_slot(
372 + def append_slots(
373 self,
374 seq: Sequence,
370 - ) -> Optional[Tuple[int, int]]:
375 + num_lookahead_slots: int = 0,
376 + ) -> Dict[int, List[int]]:
377 """Allocate a physical slot for a new token."""
378 logical_blocks = seq.logical_token_blocks
379 block_table = self.block_tables[seq.seq_id]
@@ -386,7 +392,7 @@ class BlockSpaceManagerV1(BlockSpaceManager):
392 # Allocate a new physical block.
393 new_block = self._allocate_last_physical_block(seq)
394 block_table.append(new_block)
389 - return None
395 + return {}
396
397 # We want to append the token to the last physical block.
398 last_block = block_table[-1]
@@ -399,7 +405,7 @@ class BlockSpaceManagerV1(BlockSpaceManager):
405 maybe_new_block = self._maybe_promote_last_block(
406 seq, last_block)
407 block_table[-1] = maybe_new_block
402 - return None
408 + return {}
409 else:
410 # The last block is shared with other sequences.
411 # Copy on Write: Allocate a new block and copy the tokens.
@@ -407,7 +413,7 @@ class BlockSpaceManagerV1(BlockSpaceManager):
413
414 block_table[-1] = new_block
415 self.gpu_allocator.free(last_block)
410 - return last_block.block_number, new_block.block_number
416 + return {last_block.block_number: [new_block.block_number]}
417
418 def fork(self, parent_seq: Sequence, child_seq: Sequence) -> None:
419 # NOTE: fork does not allocate a new physical block.
@@ -433,7 +439,11 @@ class BlockSpaceManagerV1(BlockSpaceManager):
439 blocks.update(self.block_tables[seq.seq_id])
440 return list(blocks)
441
436 - def can_swap_in(self, seq_group: SequenceGroup) -> bool:
442 + def can_swap_in(self,
443 + seq_group: SequenceGroup,
444 + num_lookahead_slots: int = 0) -> bool:
445 + assert (num_lookahead_slots == 0
446 + ), "BlockSpaceManagerV1 does not support lookahead allocation"
447 blocks = self._get_physical_blocks(seq_group)
448 num_swapped_seqs = seq_group.num_seqs(status=SequenceStatus.SWAPPED)
449 num_free_blocks = self.gpu_allocator.get_num_free_blocks()
@@ -443,7 +453,12 @@ class BlockSpaceManagerV1(BlockSpaceManager):
453 num_required_blocks = len(blocks) + num_swapped_seqs
454 return num_free_blocks - num_required_blocks >= self.watermark_blocks
455
446 - def swap_in(self, seq_group: SequenceGroup) -> Dict[int, int]:
456 + def swap_in(self,
457 + seq_group: SequenceGroup,
458 + num_lookahead_slots: int = 0) -> Dict[int, int]:
459 + assert (num_lookahead_slots == 0
460 + ), "BlockSpaceManagerV1 does not support lookahead allocation"
461 +
462 # CPU block -> GPU block.
463 mapping: Dict[PhysicalTokenBlock, PhysicalTokenBlock] = {}
464 for seq in seq_group.get_seqs(status=SequenceStatus.SWAPPED):
vllm/core/block_manager_v2.py
+59 -23
@@ -1,5 +1,5 @@
1 """A block manager that manages token blocks."""
2 -from typing import Dict, List, Optional, Tuple
2 +from typing import Dict, List, Optional
3
4 from vllm.core.block.block_table import BlockTable
5 from vllm.core.block.cpu_gpu_block_allocator import CpuGpuBlockAllocator
@@ -21,6 +21,24 @@ class BlockSpaceManagerV2(BlockSpaceManager):
21 sliding-window are not feature complete. This class implements the design
22 described in https://github.com/vllm-project/vllm/pull/3492.
23
24 + Lookahead slots
25 + The block manager has the notion of a "lookahead slot". These are slots
26 + in the KV cache that are allocated for a sequence. Unlike the other
27 + allocated slots, the content of these slots is undefined -- the worker
28 + may use the memory allocations in any way.
29 +
30 + In practice, a worker could use these lookahead slots to run multiple
31 + forward passes for a single scheduler invocation. Each successive
32 + forward pass would write KV activations to the corresponding lookahead
33 + slot. This allows low inter-token latency use-cases, where the overhead
34 + of continuous batching scheduling is amortized over >1 generated tokens.
35 +
36 + Speculative decoding uses lookahead slots to store KV activations of
37 + proposal tokens.
38 +
39 + See https://github.com/vllm-project/vllm/pull/3250 for more information
40 + on lookahead scheduling.
41 +
42 Args:
43 block_size (int): The size of each memory block.
44 num_gpu_blocks (int): The number of memory blocks allocated on GPU.
@@ -116,35 +134,51 @@ class BlockSpaceManagerV2(BlockSpaceManager):
134 for seq in waiting_seqs[1:]:
135 self.block_tables[seq.seq_id] = block_table.fork()
136
119 - def can_append_slot(self, seq_group: SequenceGroup) -> bool:
120 - # Simple heuristic: If there is at least one free block
121 - # for each sequence, we can append.
137 + def can_append_slots(self, seq_group: SequenceGroup,
138 + num_lookahead_slots: int) -> bool:
139 + """Determine if there is enough space in the GPU KV cache to continue
140 + generation of the specified sequence group.
141 +
142 + We use a worst-case heuristic: assume each touched block will require a
143 + new allocation (either via CoW or new block). We can append slots if the
144 + number of touched blocks is less than the number of free blocks.
145 +
146 + "Lookahead slots" are slots that are allocated in addition to the slots
147 + for known tokens. The contents of the lookahead slots are not defined.
148 + This is used by speculative decoding when speculating future tokens.
149 + """
150 +
151 + num_touched_blocks = 0
152 + for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
153 + block_table = self.block_tables[seq.seq_id]
154 +
155 + num_touched_blocks += (
156 + block_table.get_num_blocks_touched_by_append_slots(
157 + token_ids=block_table.get_unseen_token_ids(
158 + seq.get_token_ids()),
159 + num_lookahead_slots=num_lookahead_slots,
160 + ))
161 +
162 num_free_gpu_blocks = self.block_allocator.get_num_free_blocks(
163 Device.GPU)
124 - num_seqs = seq_group.num_seqs(status=SequenceStatus.RUNNING)
125 - return num_seqs <= num_free_gpu_blocks
164 + return num_touched_blocks <= num_free_gpu_blocks
165
127 - def append_slot(
166 + def append_slots(
167 self,
168 seq: Sequence,
130 - ) -> Optional[Tuple[int, int]]:
169 + num_lookahead_slots: int,
170 + ) -> Dict[int, List[int]]:
171
172 block_table = self.block_tables[seq.seq_id]
173
134 - # Get unseen token ids.
135 - num_full_slots = block_table.num_full_slots
136 - unseen_token_ids = seq.get_token_ids()[num_full_slots:]
137 - assert unseen_token_ids
138 -
139 - block_table.append_token_ids(unseen_token_ids)
140 -
141 - # Return any copy-on-writes.
142 - _ = self.block_allocator.clear_copy_on_writes()
143 -
144 - # TODO extend append_slot interface to append_slots
145 - # @cadedaniel will do in https://github.com/vllm-project/vllm/pull/3250
174 + block_table.append_token_ids(
175 + token_ids=block_table.get_unseen_token_ids(seq.get_token_ids()),
176 + num_lookahead_slots=num_lookahead_slots,
177 + )
178
147 - return None
179 + # Return any new copy-on-writes.
180 + new_cows = self.block_allocator.clear_copy_on_writes()
181 + return new_cows
182
183 def free(self, seq: Sequence) -> None:
184 if seq.seq_id not in self.block_tables:
@@ -191,10 +225,12 @@ class BlockSpaceManagerV2(BlockSpaceManager):
225 src_block_table = self.block_tables[parent_seq.seq_id]
226 self.block_tables[child_seq.seq_id] = src_block_table.fork()
227
194 - def can_swap_in(self, seq_group: SequenceGroup) -> bool:
228 + def can_swap_in(self, seq_group: SequenceGroup,
229 + num_lookahead_slots: int) -> bool:
230 return False
231
197 - def swap_in(self, seq_group: SequenceGroup) -> Dict[int, int]:
232 + def swap_in(self, seq_group: SequenceGroup,
233 + num_lookahead_slots: int) -> Dict[int, int]:
234 raise NotImplementedError
235
236 def can_swap_out(self, seq_group: SequenceGroup) -> bool:
vllm/core/interfaces.py
+10 -6
@@ -1,6 +1,6 @@
1 import enum
2 from abc import ABC, abstractmethod
3 -from typing import Dict, List, Optional, Tuple
3 +from typing import Dict, List
4
5 from vllm.sequence import Sequence, SequenceGroup
6
@@ -44,14 +44,16 @@ class BlockSpaceManager(ABC):
44 pass
45
46 @abstractmethod
47 - def can_append_slot(self, seq_group: SequenceGroup) -> bool:
47 + def can_append_slots(self, seq_group: SequenceGroup,
48 + num_lookahead_slots: int) -> bool:
49 pass
50
51 @abstractmethod
51 - def append_slot(
52 + def append_slots(
53 self,
54 seq: Sequence,
54 - ) -> Optional[Tuple[int, int]]:
55 + num_lookahead_slots: int,
56 + ) -> Dict[int, List[int]]:
57 pass
58
59 @abstractmethod
@@ -59,11 +61,13 @@ class BlockSpaceManager(ABC):
61 pass
62
63 @abstractmethod
62 - def can_swap_in(self, seq_group: SequenceGroup) -> bool:
64 + def can_swap_in(self, seq_group: SequenceGroup,
65 + num_lookahead_slots: int) -> bool:
66 pass
67
68 @abstractmethod
66 - def swap_in(self, seq_group: SequenceGroup) -> Dict[int, int]:
69 + def swap_in(self, seq_group: SequenceGroup,
70 + num_lookahead_slots: int) -> Dict[int, int]:
71 pass
72
73 @abstractmethod
vllm/core/scheduler.py
+63 -12
@@ -52,6 +52,7 @@ class SchedulerOutputs:
52 blocks_to_swap_out: Dict[int, int],
53 blocks_to_copy: Dict[int, List[int]],
54 ignored_seq_groups: List[SequenceGroup],
55 + num_lookahead_slots: int,
56 ) -> None:
57 """A list of sequence groups to be scheduled as a single batch.
58
@@ -86,6 +87,7 @@ class SchedulerOutputs:
87
88 # Swap in and swap out should never happen at the same time.
89 assert not (blocks_to_swap_in and blocks_to_swap_out)
90 + self.num_lookahead_slots = num_lookahead_slots
91
92 self.num_loras: int = len(self.lora_requests)
93 if self.num_loras > 0:
@@ -309,6 +311,8 @@ class Scheduler:
311 blocks_to_swap_out=blocks_to_swap_out,
312 blocks_to_copy=blocks_to_copy,
313 ignored_seq_groups=ignored_seq_groups,
314 + num_lookahead_slots=self._get_num_lookahead_slots(
315 + is_prefill=True),
316 )
317 return scheduler_outputs
318
@@ -323,7 +327,7 @@ class Scheduler:
327 preempted: List[SequenceGroup] = []
328 while self.running:
329 seq_group = self.running.popleft()
326 - while not self.block_manager.can_append_slot(seq_group):
330 + while not self._can_append_slots(seq_group):
331 if self.running:
332 # Preempt the lowest-priority sequence groups.
333 victim_seq_group = self.running.pop()
@@ -337,7 +341,7 @@ class Scheduler:
341 break
342 else:
343 # Append new slots to the sequence group.
340 - self._append_slot(seq_group, blocks_to_copy)
344 + self._append_slots(seq_group, blocks_to_copy)
345 running.append(seq_group)
346 self.running = running
347
@@ -366,7 +370,7 @@ class Scheduler:
370 continue
371
372 # If the sequence group cannot be swapped in, stop.
369 - if not self.block_manager.can_swap_in(seq_group):
373 + if not self._can_swap_in(seq_group):
374 break
375
376 # The total number of sequences in the RUNNING state should not
@@ -380,7 +384,7 @@ class Scheduler:
384 curr_loras.add(lora_int_id)
385 self.swapped.popleft()
386 self._swap_in(seq_group, blocks_to_swap_in)
383 - self._append_slot(seq_group, blocks_to_copy)
387 + self._append_slots(seq_group, blocks_to_copy)
388 num_curr_seqs += num_new_seqs
389 self.running.append(seq_group)
390
@@ -405,9 +409,32 @@ class Scheduler:
409 blocks_to_swap_out=blocks_to_swap_out,
410 blocks_to_copy=blocks_to_copy,
411 ignored_seq_groups=[],
412 + num_lookahead_slots=self._get_num_lookahead_slots(
413 + is_prefill=False),
414 )
415 return scheduler_outputs
416
417 + def _can_append_slots(self, seq_group: SequenceGroup) -> bool:
418 + """Determine whether or not we have enough space in the KV cache to
419 + continue generation of the sequence group.
420 + """
421 + # Appending slots only occurs in decoding.
422 + is_prefill = False
423 +
424 + return self.block_manager.can_append_slots(
425 + seq_group=seq_group,
426 + num_lookahead_slots=self._get_num_lookahead_slots(is_prefill),
427 + )
428 +
429 + def _can_swap_in(self, seq_group: SequenceGroup) -> bool:
430 + # Swapping in is considered decode.
431 + is_prefill = False
432 +
433 + return self.block_manager.can_swap_in(
434 + seq_group=seq_group,
435 + num_lookahead_slots=self._get_num_lookahead_slots(is_prefill),
436 + )
437 +
438 def schedule(self) -> Tuple[List[SequenceGroupMetadata], SchedulerOutputs]:
439 # Schedule sequence groups.
440 # This function call changes the internal states of the scheduler
@@ -482,19 +509,30 @@ class Scheduler:
509 for seq in seq_group.get_seqs(status=SequenceStatus.WAITING):
510 seq.status = SequenceStatus.RUNNING
511
485 - def _append_slot(
512 + def _append_slots(
513 self,
514 seq_group: SequenceGroup,
515 blocks_to_copy: Dict[int, List[int]],
516 ) -> None:
517 + """Appends new slots to the sequences in the given sequence group.
518 +
519 + Args:
520 + seq_group (SequenceGroup): The sequence group containing the
521 + sequences to append slots to.
522 + blocks_to_copy (Dict[int, List[int]]): A dictionary mapping source
523 + block indices to lists of destination block indices. This
524 + dictionary is updated with the new source and destination block
525 + indices for the appended slots.
526 + """
527 + num_lookahead_slots = self._get_num_lookahead_slots(is_prefill=False)
528 +
529 for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
491 - ret = self.block_manager.append_slot(seq)
492 - if ret is not None:
493 - src_block, dst_block = ret
494 - if src_block in blocks_to_copy:
495 - blocks_to_copy[src_block].append(dst_block)
496 - else:
497 - blocks_to_copy[src_block] = [dst_block]
530 + cows = self.block_manager.append_slots(seq, num_lookahead_slots)
531 +
532 + for src, dests in cows.items():
533 + if src not in blocks_to_copy:
534 + blocks_to_copy[src] = []
535 + blocks_to_copy[src].extend(dests)
536
537 def _preempt(
538 self,
@@ -588,3 +626,16 @@ class Scheduler:
626 else:
627 passed_delay = True
628 return passed_delay
629 +
630 + def _get_num_lookahead_slots(self, is_prefill: bool) -> int:
631 + """The number of slots to allocate per sequence per step, beyond known
632 + token ids. Speculative decoding uses these slots to store KV activations
633 + of tokens which may or may not be accepted.
634 +
635 + Speculative decoding does not yet support prefill, so we do not perform
636 + lookahead allocation for prefill.
637 + """
638 + if is_prefill:
639 + return 0
640 +
641 + return self.scheduler_config.num_lookahead_slots
vllm/engine/arg_utils.py
+10 -1
@@ -53,8 +53,8 @@ class EngineArgs:
53 max_cpu_loras: Optional[int] = None
54 device: str = 'auto'
55 ray_workers_use_nsight: bool = False
56 -
56 forced_num_gpu_blocks: Optional[int] = None
57 + num_lookahead_slots: int = 0
58
59 # Related to Vision-language models such as llava
60 image_input_type: Optional[str] = None
@@ -202,6 +202,14 @@ class EngineArgs:
202 parser.add_argument('--use-v2-block-manager',
203 action='store_true',
204 help='Use BlockSpaceMangerV2')
205 + parser.add_argument(
206 + '--num-lookahead-slots',
207 + type=int,
208 + default=EngineArgs.num_lookahead_slots,
209 + help='Experimental scheduling config necessary for '
210 + 'speculative decoding. This will be replaced by '
211 + 'speculative config in the future; it is present '
212 + 'to enable correctness tests until then.')
213
214 parser.add_argument('--seed',
215 type=int,
@@ -406,6 +414,7 @@ class EngineArgs:
414 self.max_num_seqs,
415 model_config.max_model_len,
416 self.use_v2_block_manager,
417 + num_lookahead_slots=self.num_lookahead_slots,
418 delay_factor=self.scheduler_delay_factor,
419 enable_chunked_prefill=self.enable_chunked_prefill,
420 )