A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Speculative decoding 4/9] Lookahead scheduling for speculative decoding (#3250)
Cade Daniel committed
Apr 1, 2024 at 15:55 UTC
93deb0b38f8c11a67fc559957661ceac7172dbfb
13 files changed
+579
-123
tests/core/block/e2e/test_correctness.py
+153
@@ -77,6 +77,159 @@ def test_v1_v2_greedy_equality_with_preemption(baseline_llm_generator,
77
assert baseline_token_ids == test_token_ids
78
79
80
+@pytest.mark.parametrize(
81
+ "common_llm_kwargs",
82
+ [{
83
+ # Use a small model for a fast test.
84
+ "model": "facebook/opt-125m",
85
+
86
+ # skip cuda graph creation for fast test.
87
+ "enforce_eager": True,
88
+
89
+ # Use a large block size to trigger more copy-on-writes.
90
+ "block_size": 32,
91
+ }])
92
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
93
+@pytest.mark.parametrize("baseline_llm_kwargs", [{
94
+ "use_v2_block_manager": False
95
+}])
96
+@pytest.mark.parametrize("test_llm_kwargs", [{"use_v2_block_manager": True}])
97
+@pytest.mark.parametrize("batch_size", [10])
98
+@pytest.mark.parametrize("seed", [1])
99
+def test_v1_v2_greedy_equality_with_cow(baseline_llm_generator,
100
+ test_llm_generator, batch_size):
101
+ """Verify beam search equality with block manager v1 and v2.
102
+
103
+ This requires copy-on-writes; if the v1 and v2 output is the same, then
104
+ we have some confidence cow is working.
105
+ """
106
+ output_len = 128
107
+ temperature = 0.0
108
+
109
+ prompts = [
110
+ "Hello, my name is",
111
+ "The president of the United States is",
112
+ "The capital of France is",
113
+ "The future of AI is",
114
+ ]
115
+
116
+ prompts = [prompt for prompt, _ in zip(cycle(prompts), range(batch_size))]
117
+
118
+ sampling_params = SamplingParams(
119
+ max_tokens=output_len,
120
+ ignore_eos=True,
121
+ temperature=temperature,
122
+ use_beam_search=True,
123
+ best_of=2,
124
+ )
125
+
126
+ print('Getting token ids from block manager v1')
127
+ baseline_token_ids = get_token_ids_from_llm_generator(
128
+ baseline_llm_generator, prompts, sampling_params)
129
+
130
+ print('Getting token ids from block manager v2')
131
+ test_token_ids = get_token_ids_from_llm_generator(test_llm_generator,
132
+ prompts, sampling_params)
133
+
134
+ for expected_token_ids, actual_token_ids in zip(baseline_token_ids,
135
+ test_token_ids):
136
+ assert expected_token_ids == actual_token_ids
137
+
138
+ assert baseline_token_ids == test_token_ids
139
+
140
+
141
+@pytest.mark.parametrize(
142
+ "common_llm_kwargs",
143
+ [{
144
+ # Use a small model for a fast test.
145
+ "model": "facebook/opt-125m",
146
+
147
+ # Our prompts will generate 128 tokens; since the prompts themselves are
148
+ # small, we don't need much KV space beyond 128.
149
+ "max_model_len": 160,
150
+
151
+ # skip cuda graph creation for fast test.
152
+ "enforce_eager": True,
153
+
154
+ # Lookahead scheduling only supported in v2 block manager.
155
+ "use_v2_block_manager": True,
156
+ }])
157
+@pytest.mark.parametrize(
158
+ "per_test_common_llm_kwargs",
159
+ [
160
+ {
161
+ "block_size": 16,
162
+
163
+ # Allow only 2 sequences of ~128 tokens in worst case.
164
+ # Note 8 = 128/block_size
165
+ "forced_num_gpu_blocks": 2 * (8 + 1),
166
+ },
167
+ {
168
+ "block_size": 8,
169
+
170
+ # Allow only 2 sequences of ~128 tokens in worst case.
171
+ # Note 16 = 128/block_size
172
+ "forced_num_gpu_blocks": 2 * (16 + 1),
173
+ }
174
+ ])
175
+@pytest.mark.parametrize("baseline_llm_kwargs", [{
176
+ "num_lookahead_slots": 0,
177
+}])
178
+@pytest.mark.parametrize(
179
+ "test_llm_kwargs",
180
+ [{
181
+ # We run one test with block_size < lookahead_slots, one test with
182
+ # block_size > lookahead_slots
183
+ "num_lookahead_slots": 10,
184
+ }])
185
+@pytest.mark.parametrize("batch_size", [4])
186
+@pytest.mark.parametrize("seed", [1])
187
+def test_lookahead_greedy_equality_with_preemption(baseline_llm_generator,
188
+ test_llm_generator,
189
+ batch_size):
190
+ """Verify vLLM produces the same output with greedy sampling, when lookahead
191
+ scheduling is used vs. not.
192
+
193
+ Lookahead scheduling is not expected to modify the output, as it simply
194
+ allocates empty slots ahead of the known token ids in a sliding fashion.
195
+
196
+ This test constrains the total number of blocks to force preemption. It also
197
+ varies the block size so that the lookahead size is less than and greater
198
+ than the block size.
199
+ """
200
+ output_len = 128
201
+ temperature = 0.0
202
+
203
+ prompts = [
204
+ "Hello, my name is",
205
+ "The president of the United States is",
206
+ "The capital of France is",
207
+ "The future of AI is",
208
+ ]
209
+
210
+ prompts = [prompt for prompt, _ in zip(cycle(prompts), range(batch_size))]
211
+
212
+ sampling_params = SamplingParams(
213
+ max_tokens=output_len,
214
+ ignore_eos=True,
215
+ temperature=temperature,
216
+ )
217
+
218
+ print('Getting token ids without lookahead scheduling')
219
+ baseline_token_ids = get_token_ids_from_llm_generator(
220
+ baseline_llm_generator, prompts, sampling_params)
221
+
222
+ print('Getting token ids with lookahead scheduling')
223
+ test_token_ids = get_token_ids_from_llm_generator(test_llm_generator,
224
+ prompts, sampling_params)
225
+
226
+ for expected_token_ids, actual_token_ids in zip(baseline_token_ids,
227
+ test_token_ids):
228
+ assert expected_token_ids == actual_token_ids
229
+
230
+ assert baseline_token_ids == test_token_ids
231
+
232
+
233
def get_token_ids_from_llm_generator(llm_generator, prompts, sampling_params):
234
for llm in llm_generator:
235
outputs = llm.generate(prompts, sampling_params, use_tqdm=True)
tests/core/block/test_block_manager_v2.py
new
+103
@@ -0,0 +1,103 @@
1
+import pytest
2
+
3
+from vllm.core.block_manager_v2 import BlockSpaceManagerV2
4
+from vllm.core.interfaces import AllocStatus
5
+from vllm.sequence import Logprob, SequenceStatus
6
+from vllm.utils import chunk_list
7
+
8
+from ..utils import create_seq_group
9
+
10
+
11
+@pytest.mark.parametrize("block_size", [16])
12
+@pytest.mark.parametrize("num_gpu_blocks", [8, 40, 80])
13
+@pytest.mark.parametrize("num_seqs_per_group", [1, 4])
14
+@pytest.mark.parametrize("watermark", [0.0, 0.5])
15
+def test_can_allocate_seq_group(block_size: int, num_seqs_per_group: int,
16
+ num_gpu_blocks: int, watermark: float):
17
+ block_manager = BlockSpaceManagerV2(
18
+ block_size=block_size,
19
+ num_gpu_blocks=num_gpu_blocks,
20
+ num_cpu_blocks=1024,
21
+ watermark=watermark,
22
+ )
23
+ num_watermark_blocks = int(watermark * num_gpu_blocks)
24
+
25
+ num_output_blocks_per_seq = 1
26
+
27
+ # NOTE: This should be num_output_blocks_per_seq * num_seqs_per_group, but
28
+ # the current implementation assumes all seqs are new prompts / don't have
29
+ # different output lens.
30
+ num_output_blocks = num_output_blocks_per_seq
31
+
32
+ for num_prompt_blocks in range(1, num_gpu_blocks - num_output_blocks):
33
+ seq_group = create_seq_group(
34
+ seq_prompt_len=block_size * num_prompt_blocks,
35
+ seq_output_lens=[
36
+ block_size * num_output_blocks_per_seq
37
+ for _ in range(num_seqs_per_group)
38
+ ],
39
+ )
40
+
41
+ assert num_prompt_blocks + num_output_blocks <= num_gpu_blocks
42
+
43
+ can_allocate_result = block_manager.can_allocate(seq_group)
44
+
45
+ num_required_blocks = num_prompt_blocks + num_output_blocks
46
+
47
+ if num_gpu_blocks - num_required_blocks < num_watermark_blocks:
48
+ assert can_allocate_result == AllocStatus.NEVER
49
+ elif num_gpu_blocks >= num_required_blocks:
50
+ assert can_allocate_result == AllocStatus.OK
51
+ else:
52
+ assert can_allocate_result == AllocStatus.LATER
53
+
54
+
55
+@pytest.mark.parametrize("block_size", [1, 8])
56
+@pytest.mark.parametrize("prompt_len", [1, 7, 8])
57
+@pytest.mark.parametrize("num_slots_to_append", [1, 8, 129])
58
+@pytest.mark.parametrize("num_lookahead_slots", [0, 10])
59
+def test_append_slots(block_size, prompt_len, num_slots_to_append,
60
+ num_lookahead_slots):
61
+ """Verify append_slots consumes the correct number of blocks from the block
62
+ table.
63
+ """
64
+
65
+ num_gpu_blocks = 1024
66
+ watermark = 0.1
67
+ block_manager = BlockSpaceManagerV2(
68
+ block_size=block_size,
69
+ num_gpu_blocks=num_gpu_blocks,
70
+ num_cpu_blocks=0,
71
+ watermark=watermark,
72
+ )
73
+
74
+ seq_group = create_seq_group(
75
+ seq_prompt_len=prompt_len,
76
+ seq_output_lens=[0],
77
+ )
78
+
79
+ # Allocate seq
80
+ assert block_manager.can_allocate(seq_group)
81
+ block_manager.allocate(seq_group)
82
+
83
+ # Seq seq to RUNNING
84
+ seq = seq_group.get_seqs()[0]
85
+ seq.status = SequenceStatus.RUNNING
86
+
87
+ # Append tokens to the sequeqnce
88
+ for token_id in range(num_slots_to_append):
89
+ seq.append_token_id(token_id, {token_id: Logprob(0.0)})
90
+
91
+ # Append slots for new tokens and lookahead slots.
92
+ free_blocks_before_append = block_manager.get_num_free_gpu_blocks()
93
+ block_manager.append_slots(seq, num_lookahead_slots)
94
+ num_consumed_blocks = (free_blocks_before_append -
95
+ block_manager.get_num_free_gpu_blocks())
96
+
97
+ # Expect consumed blocks to be new blocks required to support the new slots.
98
+ expected_consumed_blocks = len(
99
+ chunk_list(
100
+ list(
101
+ range(prompt_len + num_slots_to_append + num_lookahead_slots)),
102
+ block_size)) - len(chunk_list(list(range(prompt_len)), block_size))
103
+ assert num_consumed_blocks == expected_consumed_blocks
tests/core/block/test_block_space_manager.py
deleted
-50
@@ -1,50 +0,0 @@
1
-import pytest
2
-
3
-from vllm.core.block_manager_v2 import BlockSpaceManagerV2
4
-from vllm.core.interfaces import AllocStatus
5
-
6
-from ..utils import create_seq_group
7
-
8
-
9
-@pytest.mark.parametrize("block_size", [16])
10
-@pytest.mark.parametrize("num_gpu_blocks", [8, 40, 80])
11
-@pytest.mark.parametrize("num_seqs_per_group", [1, 4])
12
-@pytest.mark.parametrize("watermark", [0.0, 0.5])
13
-def test_can_allocate_seq_group(block_size: int, num_seqs_per_group: int,
14
- num_gpu_blocks: int, watermark: float):
15
- block_manager = BlockSpaceManagerV2(
16
- block_size=block_size,
17
- num_gpu_blocks=num_gpu_blocks,
18
- num_cpu_blocks=1024,
19
- watermark=watermark,
20
- )
21
- num_watermark_blocks = int(watermark * num_gpu_blocks)
22
-
23
- num_output_blocks_per_seq = 1
24
-
25
- # NOTE: This should be num_output_blocks_per_seq * num_seqs_per_group, but
26
- # the current implementation assumes all seqs are new prompts / don't have
27
- # different output lens.
28
- num_output_blocks = num_output_blocks_per_seq
29
-
30
- for num_prompt_blocks in range(1, num_gpu_blocks - num_output_blocks):
31
- seq_group = create_seq_group(
32
- seq_prompt_lens=block_size * num_prompt_blocks,
33
- seq_output_lens=[
34
- block_size * num_output_blocks_per_seq
35
- for _ in range(num_seqs_per_group)
36
- ],
37
- )
38
-
39
- assert num_prompt_blocks + num_output_blocks <= num_gpu_blocks
40
-
41
- can_allocate_result = block_manager.can_allocate(seq_group)
42
-
43
- num_required_blocks = num_prompt_blocks + num_output_blocks
44
-
45
- if num_gpu_blocks - num_required_blocks < num_watermark_blocks:
46
- assert can_allocate_result == AllocStatus.NEVER
47
- elif num_gpu_blocks >= num_required_blocks:
48
- assert can_allocate_result == AllocStatus.OK
49
- else:
50
- assert can_allocate_result == AllocStatus.LATER
tests/core/block/test_block_table.py
+75
@@ -498,3 +498,78 @@ def test_cow_lookahead_simple(block_size: int, sequence_len: int,
498
499
# After free, expect all blocks to be freed.
500
assert allocator.get_num_free_blocks(Device.GPU) == num_gpu_blocks
501
+
502
+
503
+@pytest.mark.parametrize("block_size", [1, 8])
504
+@pytest.mark.parametrize("sequence_len", [1, 16, 129])
505
+@pytest.mark.parametrize("num_new_tokens", [1, 16, 129])
506
+@pytest.mark.parametrize("num_lookahead_slots", [1, 7, 8])
507
+@pytest.mark.parametrize("allocator_type", ["naive", "prefix_caching"])
508
+def test_num_blocks_touched_by_append_slots(block_size: int, sequence_len: int,
509
+ num_new_tokens: int,
510
+ num_lookahead_slots: int,
511
+ allocator_type: str):
512
+ """Verify correct calculation of get_num_blocks_touched_by_append_slots.
513
+
514
+ This is done by using copy-on-write, which requires any modified block to
515
+ be copied before write if the refcount > 1. We set the refcount>1 by forking
516
+ a sequence, then measure the free blocks before and after an append. If the
517
+ number of consumed blocks equals what `get_num_blocks_touched_by_append_
518
+ slots` returns, then the calculation is correct.
519
+ """
520
+
521
+ num_gpu_blocks = 1024
522
+
523
+ allocator = CpuGpuBlockAllocator.create(
524
+ allocator_type=allocator_type,
525
+ num_gpu_blocks=num_gpu_blocks,
526
+ num_cpu_blocks=0,
527
+ block_size=block_size,
528
+ )
529
+
530
+ token_ids = list(range(sequence_len))
531
+ token_ids_to_append = list(range(num_new_tokens))
532
+
533
+ block_table = BlockTable(
534
+ block_size=block_size,
535
+ block_allocator=allocator,
536
+ )
537
+
538
+ block_table.allocate(token_ids=token_ids, device=Device.GPU)
539
+
540
+ # Add lookahead before fork so both sequences have the same lookahead
541
+ # blocks.
542
+ block_table.ensure_num_empty_slots(num_empty_slots=num_lookahead_slots)
543
+
544
+ # Fork sequence so that every block has refcount > 1.
545
+ _ = block_table.fork()
546
+
547
+ # Determine how many blocks should be touched.
548
+ expected_num_touched_blocks = (
549
+ block_table.get_num_blocks_touched_by_append_slots(
550
+ token_ids=token_ids_to_append,
551
+ num_lookahead_slots=num_lookahead_slots))
552
+
553
+ # Measure how many blocks are touched by measuring num_free_blocks before
554
+ # and after the append.
555
+ #
556
+ # We expect append_token_ids to CoW all mutated blocks that have refcount>1.
557
+ num_free_blocks_before_append = allocator.get_num_free_blocks(Device.GPU)
558
+ block_table.append_token_ids(token_ids_to_append, num_lookahead_slots)
559
+ num_consumed_blocks = (num_free_blocks_before_append -
560
+ allocator.get_num_free_blocks(Device.GPU))
561
+
562
+ # TODO(cade) ensure equality when num_lookahead_slots > 0.
563
+ # The reason we have < is because lookahead blocks are not copied eagerly;
564
+ # they are copied on first write. This will cause issues for beam search +
565
+ # speculative decoding. This is acceptable for now as it is a large effort
566
+ # to combine the two. To fix this, we can ensure single sequence ownership
567
+ # of lookahead blocks by appending empty slots to each block, which will
568
+ # trigger the CoW.
569
+ #
570
+ # Until then, we can accept that the consumed tokens are <= the expected
571
+ # tokens when appending with lookahead.
572
+ if num_lookahead_slots > 0:
573
+ assert num_consumed_blocks <= expected_num_touched_blocks
574
+ else:
575
+ assert num_consumed_blocks == expected_num_touched_blocks
tests/core/test_block_manager.py
+12
-12
@@ -103,9 +103,9 @@ def test_append_slot_single_seq():
103
block_manager.allocate(seq_group)
104
105
# Nothing to append. Sequence has no new logical blocks.
106
- assert block_manager.can_append_slot(seq_group)
106
+ assert block_manager.can_append_slots(seq_group)
107
before_blocks = block_manager.get_num_free_gpu_blocks()
108
- assert not block_manager.append_slot(prompt)
108
+ assert not block_manager.append_slots(prompt)
109
after_blocks = block_manager.get_num_free_gpu_blocks()
110
assert before_blocks == after_blocks
111
@@ -114,9 +114,9 @@ def test_append_slot_single_seq():
114
token_id = i + 5
115
prompt.append_token_id(token_id, {token_id: Logprob(0.0)})
116
117
- assert block_manager.can_append_slot(seq_group)
117
+ assert block_manager.can_append_slots(seq_group)
118
before_blocks = block_manager.get_num_free_gpu_blocks()
119
- assert not block_manager.append_slot(prompt)
119
+ assert not block_manager.append_slots(prompt)
120
after_blocks = block_manager.get_num_free_gpu_blocks()
121
assert before_blocks - after_blocks == 1
122
@@ -150,13 +150,13 @@ def test_append_slot_cow():
150
child.append_token_id(token_id, {token_id: Logprob(0.0)})
151
block_manager.fork(prompt, child)
152
153
- assert block_manager.can_append_slot(seq_group)
153
+ assert block_manager.can_append_slots(seq_group)
154
before_blocks = block_manager.get_num_free_gpu_blocks()
155
156
- maybe_src_dst_block = block_manager.append_slot(child)
157
- assert maybe_src_dst_block is not None
158
- src_block, dst_block = maybe_src_dst_block
159
- assert src_block != dst_block
156
+ cows = block_manager.append_slots(child)
157
+ assert cows
158
+ for src_block, dst_blocks in cows.items():
159
+ assert src_block not in dst_blocks
160
161
after_blocks = block_manager.get_num_free_gpu_blocks()
162
assert before_blocks - after_blocks == 1
@@ -184,7 +184,7 @@ def test_fork():
184
token_id = 4
185
# Append token to child. Block is shared so copy on write occurs.
186
child.append_token_id(token_id, {token_id: Logprob(0.0)})
187
- block_manager.append_slot(child)
187
+ block_manager.append_slots(child)
188
assert block_manager.get_block_table(
189
prompt) != block_manager.get_block_table(child)
190
@@ -325,7 +325,7 @@ def test_sliding_window_multi_seq():
325
token_id = 4
326
# Append token to child. Block is shared so copy on write occurs.
327
child.append_token_id(token_id, {token_id: Logprob(0.0)})
328
- block_manager.append_slot(child)
328
+ block_manager.append_slots(child)
329
330
# assert the number of blocks allocated is correct
331
# we will use now one block more. Each seq will use 2 blocks,
@@ -335,7 +335,7 @@ def test_sliding_window_multi_seq():
335
336
token_id = 5
337
parent.append_token_id(token_id, {token_id: Logprob(0.0)})
338
- block_manager.append_slot(parent)
338
+ block_manager.append_slots(parent)
339
340
# assert the number of blocks allocated is correct
341
# no change, because both sequences are still just sharing one block
tests/core/utils.py
+2
-2
@@ -24,7 +24,7 @@ def create_dummy_prompt(
24
25
26
def create_seq_group(
27
- seq_prompt_lens=1024,
27
+ seq_prompt_len=1024,
28
seq_output_lens=(128, ),
29
request_id='0',
30
seq_id_start=0,
@@ -32,7 +32,7 @@ def create_seq_group(
32
33
assert len(seq_output_lens) > 0
34
35
- prompt_token_ids = [0] * seq_prompt_lens
35
+ prompt_token_ids = [0] * seq_prompt_len
36
37
seqs = []
38
for seq_id_offset, output_len in enumerate(seq_output_lens):
vllm/config.py
+16
-2
@@ -530,9 +530,13 @@ class SchedulerConfig:
530
iteration.
531
max_model_len: Maximum length of a sequence (including prompt
532
and generated text).
533
+ use_v2_block_manager: Whether to use the BlockSpaceManagerV2 or not.
534
+ num_lookahead_slots: The number of slots to allocate per sequence per
535
+ step, beyond the known token ids. This is used in speculative
536
+ decoding to store KV activations of tokens which may or may not be
537
+ accepted.
538
delay_factor: Apply a delay (of delay factor multiplied by previous
539
prompt latency) before scheduling next prompt.
535
- use_v2_block_manager: Whether to use the BlockSpaceManagerV2 or not.
540
enable_chunked_prefill: If True, prefill requests can be chunked based
541
on the remaining max_num_batched_tokens.
542
"""
@@ -543,6 +547,7 @@ class SchedulerConfig:
547
max_num_seqs: int,
548
max_model_len: int,
549
use_v2_block_manager: bool = False,
550
+ num_lookahead_slots: int = 0,
551
delay_factor: float = 0.0,
552
enable_chunked_prefill: bool = False,
553
) -> None:
@@ -554,9 +559,11 @@ class SchedulerConfig:
559
self.max_num_batched_tokens = max(max_model_len, 2048)
560
self.max_num_seqs = max_num_seqs
561
self.max_model_len = max_model_len
557
- self.delay_factor = delay_factor
562
self.use_v2_block_manager = use_v2_block_manager
563
+ self.num_lookahead_slots = num_lookahead_slots
564
+ self.delay_factor = delay_factor
565
self.chunked_prefill_enabled = enable_chunked_prefill
566
+
567
self._verify_args()
568
569
def _verify_args(self) -> None:
@@ -568,12 +575,19 @@ class SchedulerConfig:
575
"max_num_batched_tokens and makes vLLM reject longer "
576
"sequences. Please increase max_num_batched_tokens or "
577
"decrease max_model_len.")
578
+
579
if self.max_num_batched_tokens < self.max_num_seqs:
580
raise ValueError(
581
f"max_num_batched_tokens ({self.max_num_batched_tokens}) must "
582
"be greater than or equal to max_num_seqs "
583
f"({self.max_num_seqs}).")
584
585
+ if self.num_lookahead_slots < 0:
586
+ raise ValueError(
587
+ "num_lookahead_slots "
588
+ f"({self.num_lookahead_slots}) must be greater than or "
589
+ "equal to 0.")
590
+
591
592
class DeviceConfig:
593
vllm/core/block/block_table.py
+52
-6
@@ -85,7 +85,9 @@ class BlockTable:
85
device=device)
86
self._num_full_slots = len(token_ids)
87
88
- def append_token_ids(self, token_ids: List[int]) -> None:
88
+ def append_token_ids(self,
89
+ token_ids: List[int],
90
+ num_lookahead_slots: int = 0) -> None:
91
"""Appends a sequence of token IDs to the existing blocks in the
92
BlockTable.
93
@@ -102,14 +104,13 @@ class BlockTable:
104
token_ids (List[int]): The sequence of token IDs to be appended.
105
"""
106
assert self._is_allocated
107
+ assert token_ids, "can't append empty token ids"
108
106
- self.ensure_num_empty_slots(num_empty_slots=len(token_ids))
109
+ self.ensure_num_empty_slots(num_empty_slots=len(token_ids) +
110
+ num_lookahead_slots)
111
112
blocks = self._blocks[self._num_full_slots // self._block_size:]
109
- first_chunk_size = self._block_size - (self._num_full_slots %
110
- self._block_size)
111
- token_blocks = [token_ids[:first_chunk_size]] + chunk_list(
112
- token_ids[first_chunk_size:], self._block_size)
113
+ token_blocks = self._chunk_token_blocks_for_append(token_ids)
114
115
for block, token_block in zip(blocks, token_blocks):
116
block.append_token_ids(token_block)
@@ -195,6 +196,25 @@ class BlockTable:
196
assert self._is_allocated
197
return [block.block_id for block in self._blocks]
198
199
+ def get_unseen_token_ids(self, sequence_token_ids: List[int]) -> List[int]:
200
+ """Get the number of "unseen" tokens in the sequence.
201
+
202
+ Unseen tokens are tokens in the sequence corresponding to this block
203
+ table, but are not yet appended to this block table.
204
+
205
+ Args:
206
+ sequence_token_ids (List[int]): The list of token ids in the
207
+ sequence.
208
+
209
+ Returns:
210
+ List[int]: The postfix of sequence_token_ids that has not yet been
211
+ appended to the block table.
212
+ """
213
+
214
+ # Since the block table is append-only, the unseen token ids are the
215
+ # ones after the appended ones.
216
+ return sequence_token_ids[self.num_full_slots:]
217
+
218
def _allocate_blocks_for_token_ids(self, prev_block: Optional[Block],
219
token_ids: List[int],
220
device: Device) -> List[Block]:
@@ -243,3 +263,29 @@ class BlockTable:
263
int: The total number of tokens currently stored in the BlockTable.
264
"""
265
return self._num_full_slots
266
+
267
+ def get_num_blocks_touched_by_append_slots(
268
+ self, token_ids: List[int], num_lookahead_slots: int) -> int:
269
+ """Determine how many blocks will be "touched" by appending the token
270
+ ids.
271
+
272
+ This is required for the scheduler to determine whether a sequence can
273
+ continue generation, or if it must be preempted.
274
+ """
275
+
276
+ all_token_ids = token_ids + [-1] * num_lookahead_slots
277
+ token_blocks = self._chunk_token_blocks_for_append(all_token_ids)
278
+ return len(token_blocks)
279
+
280
+ def _chunk_token_blocks_for_append(
281
+ self, token_ids: List[int]) -> List[List[int]]:
282
+ """Split the token ids into block-sized chunks so they can be easily
283
+ appended to blocks. The first such "token block" may have less token ids
284
+ than the block size, since the last allocated block may be partially
285
+ full.
286
+ """
287
+ first_chunk_size = self._block_size - (self._num_full_slots %
288
+ self._block_size)
289
+ token_blocks = [token_ids[:first_chunk_size]] + chunk_list(
290
+ token_ids[first_chunk_size:], self._block_size)
291
+ return token_blocks
vllm/core/block_manager_v1.py
+24
-9
@@ -2,7 +2,7 @@
2
from abc import ABC, abstractmethod
3
from itertools import count, takewhile
4
from os.path import commonprefix
5
-from typing import Dict, List, Optional, Set, Tuple
5
+from typing import Dict, List, Optional, Set
6
7
from vllm.block import BlockTable, PhysicalTokenBlock
8
from vllm.core.evictor import EvictionPolicy, Evictor, make_evictor
@@ -292,7 +292,12 @@ class BlockSpaceManagerV1(BlockSpaceManager):
292
for seq in seq_group.get_seqs(status=SequenceStatus.WAITING):
293
self.block_tables[seq.seq_id] = block_table.copy()
294
295
- def can_append_slot(self, seq_group: SequenceGroup) -> bool:
295
+ def can_append_slots(self,
296
+ seq_group: SequenceGroup,
297
+ num_lookahead_slots: int = 0) -> bool:
298
+ assert (num_lookahead_slots == 0
299
+ ), "lookahead allocation not supported in BlockSpaceManagerV1"
300
+
301
# Simple heuristic: If there is at least one free block
302
# for each sequence, we can append.
303
num_free_gpu_blocks = self.gpu_allocator.get_num_free_blocks()
@@ -364,10 +369,11 @@ class BlockSpaceManagerV1(BlockSpaceManager):
369
assert new_block.ref_count == 1
370
return new_block
371
367
- def append_slot(
372
+ def append_slots(
373
self,
374
seq: Sequence,
370
- ) -> Optional[Tuple[int, int]]:
375
+ num_lookahead_slots: int = 0,
376
+ ) -> Dict[int, List[int]]:
377
"""Allocate a physical slot for a new token."""
378
logical_blocks = seq.logical_token_blocks
379
block_table = self.block_tables[seq.seq_id]
@@ -386,7 +392,7 @@ class BlockSpaceManagerV1(BlockSpaceManager):
392
# Allocate a new physical block.
393
new_block = self._allocate_last_physical_block(seq)
394
block_table.append(new_block)
389
- return None
395
+ return {}
396
397
# We want to append the token to the last physical block.
398
last_block = block_table[-1]
@@ -399,7 +405,7 @@ class BlockSpaceManagerV1(BlockSpaceManager):
405
maybe_new_block = self._maybe_promote_last_block(
406
seq, last_block)
407
block_table[-1] = maybe_new_block
402
- return None
408
+ return {}
409
else:
410
# The last block is shared with other sequences.
411
# Copy on Write: Allocate a new block and copy the tokens.
@@ -407,7 +413,7 @@ class BlockSpaceManagerV1(BlockSpaceManager):
413
414
block_table[-1] = new_block
415
self.gpu_allocator.free(last_block)
410
- return last_block.block_number, new_block.block_number
416
+ return {last_block.block_number: [new_block.block_number]}
417
418
def fork(self, parent_seq: Sequence, child_seq: Sequence) -> None:
419
# NOTE: fork does not allocate a new physical block.
@@ -433,7 +439,11 @@ class BlockSpaceManagerV1(BlockSpaceManager):
439
blocks.update(self.block_tables[seq.seq_id])
440
return list(blocks)
441
436
- def can_swap_in(self, seq_group: SequenceGroup) -> bool:
442
+ def can_swap_in(self,
443
+ seq_group: SequenceGroup,
444
+ num_lookahead_slots: int = 0) -> bool:
445
+ assert (num_lookahead_slots == 0
446
+ ), "BlockSpaceManagerV1 does not support lookahead allocation"
447
blocks = self._get_physical_blocks(seq_group)
448
num_swapped_seqs = seq_group.num_seqs(status=SequenceStatus.SWAPPED)
449
num_free_blocks = self.gpu_allocator.get_num_free_blocks()
@@ -443,7 +453,12 @@ class BlockSpaceManagerV1(BlockSpaceManager):
453
num_required_blocks = len(blocks) + num_swapped_seqs
454
return num_free_blocks - num_required_blocks >= self.watermark_blocks
455
446
- def swap_in(self, seq_group: SequenceGroup) -> Dict[int, int]:
456
+ def swap_in(self,
457
+ seq_group: SequenceGroup,
458
+ num_lookahead_slots: int = 0) -> Dict[int, int]:
459
+ assert (num_lookahead_slots == 0
460
+ ), "BlockSpaceManagerV1 does not support lookahead allocation"
461
+
462
# CPU block -> GPU block.
463
mapping: Dict[PhysicalTokenBlock, PhysicalTokenBlock] = {}
464
for seq in seq_group.get_seqs(status=SequenceStatus.SWAPPED):
vllm/core/block_manager_v2.py
+59
-23
@@ -1,5 +1,5 @@
1
"""A block manager that manages token blocks."""
2
-from typing import Dict, List, Optional, Tuple
2
+from typing import Dict, List, Optional
3
4
from vllm.core.block.block_table import BlockTable
5
from vllm.core.block.cpu_gpu_block_allocator import CpuGpuBlockAllocator
@@ -21,6 +21,24 @@ class BlockSpaceManagerV2(BlockSpaceManager):
21
sliding-window are not feature complete. This class implements the design
22
described in https://github.com/vllm-project/vllm/pull/3492.
23
24
+ Lookahead slots
25
+ The block manager has the notion of a "lookahead slot". These are slots
26
+ in the KV cache that are allocated for a sequence. Unlike the other
27
+ allocated slots, the content of these slots is undefined -- the worker
28
+ may use the memory allocations in any way.
29
+
30
+ In practice, a worker could use these lookahead slots to run multiple
31
+ forward passes for a single scheduler invocation. Each successive
32
+ forward pass would write KV activations to the corresponding lookahead
33
+ slot. This allows low inter-token latency use-cases, where the overhead
34
+ of continuous batching scheduling is amortized over >1 generated tokens.
35
+
36
+ Speculative decoding uses lookahead slots to store KV activations of
37
+ proposal tokens.
38
+
39
+ See https://github.com/vllm-project/vllm/pull/3250 for more information
40
+ on lookahead scheduling.
41
+
42
Args:
43
block_size (int): The size of each memory block.
44
num_gpu_blocks (int): The number of memory blocks allocated on GPU.
@@ -116,35 +134,51 @@ class BlockSpaceManagerV2(BlockSpaceManager):
134
for seq in waiting_seqs[1:]:
135
self.block_tables[seq.seq_id] = block_table.fork()
136
119
- def can_append_slot(self, seq_group: SequenceGroup) -> bool:
120
- # Simple heuristic: If there is at least one free block
121
- # for each sequence, we can append.
137
+ def can_append_slots(self, seq_group: SequenceGroup,
138
+ num_lookahead_slots: int) -> bool:
139
+ """Determine if there is enough space in the GPU KV cache to continue
140
+ generation of the specified sequence group.
141
+
142
+ We use a worst-case heuristic: assume each touched block will require a
143
+ new allocation (either via CoW or new block). We can append slots if the
144
+ number of touched blocks is less than the number of free blocks.
145
+
146
+ "Lookahead slots" are slots that are allocated in addition to the slots
147
+ for known tokens. The contents of the lookahead slots are not defined.
148
+ This is used by speculative decoding when speculating future tokens.
149
+ """
150
+
151
+ num_touched_blocks = 0
152
+ for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
153
+ block_table = self.block_tables[seq.seq_id]
154
+
155
+ num_touched_blocks += (
156
+ block_table.get_num_blocks_touched_by_append_slots(
157
+ token_ids=block_table.get_unseen_token_ids(
158
+ seq.get_token_ids()),
159
+ num_lookahead_slots=num_lookahead_slots,
160
+ ))
161
+
162
num_free_gpu_blocks = self.block_allocator.get_num_free_blocks(
163
Device.GPU)
124
- num_seqs = seq_group.num_seqs(status=SequenceStatus.RUNNING)
125
- return num_seqs <= num_free_gpu_blocks
164
+ return num_touched_blocks <= num_free_gpu_blocks
165
127
- def append_slot(
166
+ def append_slots(
167
self,
168
seq: Sequence,
130
- ) -> Optional[Tuple[int, int]]:
169
+ num_lookahead_slots: int,
170
+ ) -> Dict[int, List[int]]:
171
172
block_table = self.block_tables[seq.seq_id]
173
134
- # Get unseen token ids.
135
- num_full_slots = block_table.num_full_slots
136
- unseen_token_ids = seq.get_token_ids()[num_full_slots:]
137
- assert unseen_token_ids
138
-
139
- block_table.append_token_ids(unseen_token_ids)
140
-
141
- # Return any copy-on-writes.
142
- _ = self.block_allocator.clear_copy_on_writes()
143
-
144
- # TODO extend append_slot interface to append_slots
145
- # @cadedaniel will do in https://github.com/vllm-project/vllm/pull/3250
174
+ block_table.append_token_ids(
175
+ token_ids=block_table.get_unseen_token_ids(seq.get_token_ids()),
176
+ num_lookahead_slots=num_lookahead_slots,
177
+ )
178
147
- return None
179
+ # Return any new copy-on-writes.
180
+ new_cows = self.block_allocator.clear_copy_on_writes()
181
+ return new_cows
182
183
def free(self, seq: Sequence) -> None:
184
if seq.seq_id not in self.block_tables:
@@ -191,10 +225,12 @@ class BlockSpaceManagerV2(BlockSpaceManager):
225
src_block_table = self.block_tables[parent_seq.seq_id]
226
self.block_tables[child_seq.seq_id] = src_block_table.fork()
227
194
- def can_swap_in(self, seq_group: SequenceGroup) -> bool:
228
+ def can_swap_in(self, seq_group: SequenceGroup,
229
+ num_lookahead_slots: int) -> bool:
230
return False
231
197
- def swap_in(self, seq_group: SequenceGroup) -> Dict[int, int]:
232
+ def swap_in(self, seq_group: SequenceGroup,
233
+ num_lookahead_slots: int) -> Dict[int, int]:
234
raise NotImplementedError
235
236
def can_swap_out(self, seq_group: SequenceGroup) -> bool:
vllm/core/interfaces.py
+10
-6
@@ -1,6 +1,6 @@
1
import enum
2
from abc import ABC, abstractmethod
3
-from typing import Dict, List, Optional, Tuple
3
+from typing import Dict, List
4
5
from vllm.sequence import Sequence, SequenceGroup
6
@@ -44,14 +44,16 @@ class BlockSpaceManager(ABC):
44
pass
45
46
@abstractmethod
47
- def can_append_slot(self, seq_group: SequenceGroup) -> bool:
47
+ def can_append_slots(self, seq_group: SequenceGroup,
48
+ num_lookahead_slots: int) -> bool:
49
pass
50
51
@abstractmethod
51
- def append_slot(
52
+ def append_slots(
53
self,
54
seq: Sequence,
54
- ) -> Optional[Tuple[int, int]]:
55
+ num_lookahead_slots: int,
56
+ ) -> Dict[int, List[int]]:
57
pass
58
59
@abstractmethod
@@ -59,11 +61,13 @@ class BlockSpaceManager(ABC):
61
pass
62
63
@abstractmethod
62
- def can_swap_in(self, seq_group: SequenceGroup) -> bool:
64
+ def can_swap_in(self, seq_group: SequenceGroup,
65
+ num_lookahead_slots: int) -> bool:
66
pass
67
68
@abstractmethod
66
- def swap_in(self, seq_group: SequenceGroup) -> Dict[int, int]:
69
+ def swap_in(self, seq_group: SequenceGroup,
70
+ num_lookahead_slots: int) -> Dict[int, int]:
71
pass
72
73
@abstractmethod
vllm/core/scheduler.py
+63
-12
@@ -52,6 +52,7 @@ class SchedulerOutputs:
52
blocks_to_swap_out: Dict[int, int],
53
blocks_to_copy: Dict[int, List[int]],
54
ignored_seq_groups: List[SequenceGroup],
55
+ num_lookahead_slots: int,
56
) -> None:
57
"""A list of sequence groups to be scheduled as a single batch.
58
@@ -86,6 +87,7 @@ class SchedulerOutputs:
87
88
# Swap in and swap out should never happen at the same time.
89
assert not (blocks_to_swap_in and blocks_to_swap_out)
90
+ self.num_lookahead_slots = num_lookahead_slots
91
92
self.num_loras: int = len(self.lora_requests)
93
if self.num_loras > 0:
@@ -309,6 +311,8 @@ class Scheduler:
311
blocks_to_swap_out=blocks_to_swap_out,
312
blocks_to_copy=blocks_to_copy,
313
ignored_seq_groups=ignored_seq_groups,
314
+ num_lookahead_slots=self._get_num_lookahead_slots(
315
+ is_prefill=True),
316
)
317
return scheduler_outputs
318
@@ -323,7 +327,7 @@ class Scheduler:
327
preempted: List[SequenceGroup] = []
328
while self.running:
329
seq_group = self.running.popleft()
326
- while not self.block_manager.can_append_slot(seq_group):
330
+ while not self._can_append_slots(seq_group):
331
if self.running:
332
# Preempt the lowest-priority sequence groups.
333
victim_seq_group = self.running.pop()
@@ -337,7 +341,7 @@ class Scheduler:
341
break
342
else:
343
# Append new slots to the sequence group.
340
- self._append_slot(seq_group, blocks_to_copy)
344
+ self._append_slots(seq_group, blocks_to_copy)
345
running.append(seq_group)
346
self.running = running
347
@@ -366,7 +370,7 @@ class Scheduler:
370
continue
371
372
# If the sequence group cannot be swapped in, stop.
369
- if not self.block_manager.can_swap_in(seq_group):
373
+ if not self._can_swap_in(seq_group):
374
break
375
376
# The total number of sequences in the RUNNING state should not
@@ -380,7 +384,7 @@ class Scheduler:
384
curr_loras.add(lora_int_id)
385
self.swapped.popleft()
386
self._swap_in(seq_group, blocks_to_swap_in)
383
- self._append_slot(seq_group, blocks_to_copy)
387
+ self._append_slots(seq_group, blocks_to_copy)
388
num_curr_seqs += num_new_seqs
389
self.running.append(seq_group)
390
@@ -405,9 +409,32 @@ class Scheduler:
409
blocks_to_swap_out=blocks_to_swap_out,
410
blocks_to_copy=blocks_to_copy,
411
ignored_seq_groups=[],
412
+ num_lookahead_slots=self._get_num_lookahead_slots(
413
+ is_prefill=False),
414
)
415
return scheduler_outputs
416
417
+ def _can_append_slots(self, seq_group: SequenceGroup) -> bool:
418
+ """Determine whether or not we have enough space in the KV cache to
419
+ continue generation of the sequence group.
420
+ """
421
+ # Appending slots only occurs in decoding.
422
+ is_prefill = False
423
+
424
+ return self.block_manager.can_append_slots(
425
+ seq_group=seq_group,
426
+ num_lookahead_slots=self._get_num_lookahead_slots(is_prefill),
427
+ )
428
+
429
+ def _can_swap_in(self, seq_group: SequenceGroup) -> bool:
430
+ # Swapping in is considered decode.
431
+ is_prefill = False
432
+
433
+ return self.block_manager.can_swap_in(
434
+ seq_group=seq_group,
435
+ num_lookahead_slots=self._get_num_lookahead_slots(is_prefill),
436
+ )
437
+
438
def schedule(self) -> Tuple[List[SequenceGroupMetadata], SchedulerOutputs]:
439
# Schedule sequence groups.
440
# This function call changes the internal states of the scheduler
@@ -482,19 +509,30 @@ class Scheduler:
509
for seq in seq_group.get_seqs(status=SequenceStatus.WAITING):
510
seq.status = SequenceStatus.RUNNING
511
485
- def _append_slot(
512
+ def _append_slots(
513
self,
514
seq_group: SequenceGroup,
515
blocks_to_copy: Dict[int, List[int]],
516
) -> None:
517
+ """Appends new slots to the sequences in the given sequence group.
518
+
519
+ Args:
520
+ seq_group (SequenceGroup): The sequence group containing the
521
+ sequences to append slots to.
522
+ blocks_to_copy (Dict[int, List[int]]): A dictionary mapping source
523
+ block indices to lists of destination block indices. This
524
+ dictionary is updated with the new source and destination block
525
+ indices for the appended slots.
526
+ """
527
+ num_lookahead_slots = self._get_num_lookahead_slots(is_prefill=False)
528
+
529
for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
491
- ret = self.block_manager.append_slot(seq)
492
- if ret is not None:
493
- src_block, dst_block = ret
494
- if src_block in blocks_to_copy:
495
- blocks_to_copy[src_block].append(dst_block)
496
- else:
497
- blocks_to_copy[src_block] = [dst_block]
530
+ cows = self.block_manager.append_slots(seq, num_lookahead_slots)
531
+
532
+ for src, dests in cows.items():
533
+ if src not in blocks_to_copy:
534
+ blocks_to_copy[src] = []
535
+ blocks_to_copy[src].extend(dests)
536
537
def _preempt(
538
self,
@@ -588,3 +626,16 @@ class Scheduler:
626
else:
627
passed_delay = True
628
return passed_delay
629
+
630
+ def _get_num_lookahead_slots(self, is_prefill: bool) -> int:
631
+ """The number of slots to allocate per sequence per step, beyond known
632
+ token ids. Speculative decoding uses these slots to store KV activations
633
+ of tokens which may or may not be accepted.
634
+
635
+ Speculative decoding does not yet support prefill, so we do not perform
636
+ lookahead allocation for prefill.
637
+ """
638
+ if is_prefill:
639
+ return 0
640
+
641
+ return self.scheduler_config.num_lookahead_slots
vllm/engine/arg_utils.py
+10
-1
@@ -53,8 +53,8 @@ class EngineArgs:
53
max_cpu_loras: Optional[int] = None
54
device: str = 'auto'
55
ray_workers_use_nsight: bool = False
56
-
56
forced_num_gpu_blocks: Optional[int] = None
57
+ num_lookahead_slots: int = 0
58
59
# Related to Vision-language models such as llava
60
image_input_type: Optional[str] = None
@@ -202,6 +202,14 @@ class EngineArgs:
202
parser.add_argument('--use-v2-block-manager',
203
action='store_true',
204
help='Use BlockSpaceMangerV2')
205
+ parser.add_argument(
206
+ '--num-lookahead-slots',
207
+ type=int,
208
+ default=EngineArgs.num_lookahead_slots,
209
+ help='Experimental scheduling config necessary for '
210
+ 'speculative decoding. This will be replaced by '
211
+ 'speculative config in the future; it is present '
212
+ 'to enable correctness tests until then.')
213
214
parser.add_argument('--seed',
215
type=int,
@@ -406,6 +414,7 @@ class EngineArgs:
414
self.max_num_seqs,
415
model_config.max_model_len,
416
self.use_v2_block_manager,
417
+ num_lookahead_slots=self.num_lookahead_slots,
418
delay_factor=self.scheduler_delay_factor,
419
enable_chunked_prefill=self.enable_chunked_prefill,
420
)