A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Core] Ignore infeasible swap requests. (#4557)
SangBin Cho committed
May 3, 2024 at 06:31 UTC
0f8a91401c89ac0a8018def3756829611b57727f
12 files changed
+187
-42
tests/basic_correctness/test_preemption.py
+85
@@ -7,6 +7,7 @@ pytest tests/basic_correctness/test_preemption.py`.
7
"""
8
import pytest
9
10
+from vllm import SamplingParams
11
from vllm.core.scheduler import (ARTIFICIAL_PREEMPTION_MAX_CNT,
12
ENABLE_ARTIFICIAL_PREEMPT)
13
@@ -136,3 +137,87 @@ def test_swap(
137
assert hf_output_ids[j] == vllm_output_ids[j], (
138
f"Test{i} output{j}:\nHF: {hf_output_ids}\n"
139
f"vLLM: {vllm_output_ids}")
140
+
141
+
142
+@pytest.mark.parametrize("model", MODELS)
143
+@pytest.mark.parametrize("dtype", ["float"])
144
+@pytest.mark.parametrize("max_tokens", [96])
145
+@pytest.mark.parametrize("beam_width", [4])
146
+def test_swap_infeasible(
147
+ vllm_runner,
148
+ example_prompts,
149
+ model: str,
150
+ dtype: str,
151
+ max_tokens: int,
152
+ beam_width: int,
153
+) -> None:
154
+ """Verify infeasible swap request will be ignored."""
155
+ BLOCK_SIZE = 16
156
+ prefill_blocks = 2
157
+ decode_blocks = max_tokens // BLOCK_SIZE
158
+ example_prompts = example_prompts[:1]
159
+
160
+ vllm_model = vllm_runner(
161
+ model,
162
+ dtype=dtype,
163
+ swap_space=10,
164
+ block_size=BLOCK_SIZE,
165
+ # Since beam search have more than 1 sequence, prefill + decode blocks
166
+ # are not enough to finish.
167
+ num_gpu_blocks_override=prefill_blocks + decode_blocks,
168
+ max_model_len=(prefill_blocks + decode_blocks) * BLOCK_SIZE,
169
+ )
170
+ sampling_params = SamplingParams(n=beam_width,
171
+ use_beam_search=True,
172
+ temperature=0.0,
173
+ max_tokens=max_tokens,
174
+ ignore_eos=True)
175
+ req_outputs = vllm_model.model.generate(
176
+ example_prompts,
177
+ sampling_params=sampling_params,
178
+ )
179
+ assert (vllm_model.model.llm_engine.scheduler.artificial_preempt_cnt <
180
+ ARTIFICIAL_PREEMPTION_MAX_CNT)
181
+ del vllm_model
182
+ # Verify the request is ignored and not hang.
183
+ assert req_outputs[0].outputs[0].finish_reason == "length"
184
+
185
+
186
+@pytest.mark.parametrize("model", MODELS)
187
+@pytest.mark.parametrize("dtype", ["float"])
188
+@pytest.mark.parametrize("max_tokens", [96])
189
+def test_preemption_infeasible(
190
+ vllm_runner,
191
+ example_prompts,
192
+ model: str,
193
+ dtype: str,
194
+ max_tokens: int,
195
+) -> None:
196
+ """Verify infeasible preemption request will be ignored."""
197
+ BLOCK_SIZE = 16
198
+ prefill_blocks = 2
199
+ decode_blocks = max_tokens // BLOCK_SIZE
200
+ vllm_model = vllm_runner(
201
+ model,
202
+ dtype=dtype,
203
+ block_size=BLOCK_SIZE,
204
+ # Not enough gpu blocks to complete a single sequence.
205
+ # preemption should happen, and the sequence should be
206
+ # ignored instead of hanging forever.
207
+ num_gpu_blocks_override=prefill_blocks + decode_blocks // 2,
208
+ max_model_len=((prefill_blocks + decode_blocks // 2) * BLOCK_SIZE),
209
+ )
210
+ sampling_params = SamplingParams(max_tokens=max_tokens, ignore_eos=True)
211
+ req_outputs = vllm_model.model.generate(
212
+ example_prompts,
213
+ sampling_params=sampling_params,
214
+ )
215
+
216
+ assert (vllm_model.model.llm_engine.scheduler.artificial_preempt_cnt <
217
+ ARTIFICIAL_PREEMPTION_MAX_CNT)
218
+ del vllm_model
219
+ # Verify the request is ignored and not hang.
220
+ for req_output in req_outputs:
221
+ outputs = req_output.outputs
222
+ assert len(outputs) == 1
223
+ assert outputs[0].finish_reason == "length"
tests/core/test_block_manager.py
+1
-1
@@ -224,7 +224,7 @@ def test_swap():
224
225
# Swap seq group from CPU -> GPU.
226
cpu_blocks = block_manager.get_block_table(prompt)
227
- assert block_manager.can_swap_in(seq_group)
227
+ assert block_manager.can_swap_in(seq_group) == AllocStatus.OK
228
before_cpu_blocks = block_manager.get_num_free_cpu_blocks()
229
before_gpu_blocks = block_manager.get_num_free_gpu_blocks()
230
mapping = block_manager.swap_in(seq_group)
tests/core/test_chunked_prefill_scheduler.py
+3
-2
@@ -4,6 +4,7 @@ from unittest.mock import MagicMock
4
import pytest # noqa
5
6
from vllm.config import CacheConfig, SchedulerConfig
7
+from vllm.core.interfaces import AllocStatus
8
from vllm.core.scheduler import Scheduler
9
from vllm.sequence import Logprob, SequenceGroup
10
@@ -410,7 +411,7 @@ def test_running_prefill_prioritized_over_swap():
411
412
# Add 1 more task. Swap is not possible, so prefill is running.
413
scheduler.block_manager.can_swap_in = MagicMock()
413
- scheduler.block_manager.can_swap_in.return_value = False
414
+ scheduler.block_manager.can_swap_in.return_value = AllocStatus.LATER
415
416
_, seq_group2 = create_dummy_prompt("2", prompt_length=60)
417
scheduler.add_seq_group(seq_group2)
@@ -423,7 +424,7 @@ def test_running_prefill_prioritized_over_swap():
424
assert out.scheduled_seq_groups[0].seq_group == seq_group2
425
426
# Now although swap is possible, running prefill is prioritized.
426
- scheduler.block_manager.can_swap_in.return_value = True
427
+ scheduler.block_manager.can_swap_in.return_value = AllocStatus.OK
428
_, out = schedule_and_update_computed_tokens(scheduler)
429
assert len(out.scheduled_seq_groups) == 1
430
# 3 decodes. It is swapped in.
tests/core/test_scheduler.py
+29
-1
@@ -791,7 +791,7 @@ def test_schedule_swapped_cannot_swap_in():
791
792
# The last request should be swapped out.
793
scheduler.block_manager.can_swap_in = MagicMock()
794
- scheduler.block_manager.can_swap_in.return_value = False
794
+ scheduler.block_manager.can_swap_in.return_value = AllocStatus.LATER
795
# Since we cannot swap in, none of the requests are swapped in.
796
budget = create_token_budget()
797
remaining_swapped, output = scheduler._schedule_swapped(
@@ -803,6 +803,34 @@ def test_schedule_swapped_cannot_swap_in():
803
assert len(output.prefill_seq_groups) == 0
804
805
806
+def test_infeasible_swap():
807
+ scheduler = initialize_scheduler()
808
+ swapped = deque()
809
+ policy = PolicyFactory.get_policy(policy_name="fcfs")
810
+ curr_loras = None
811
+ blocks_to_swap_out = {}
812
+ for _ in range(2):
813
+ _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
814
+ scheduler._allocate_and_set_running(seq_group)
815
+ append_new_token_seq_group(60, seq_group, 1)
816
+ scheduler._swap_out(seq_group, blocks_to_swap_out)
817
+ swapped.append(seq_group)
818
+
819
+ # The last request should be swapped out.
820
+ scheduler.block_manager.can_swap_in = MagicMock()
821
+ scheduler.block_manager.can_swap_in.return_value = AllocStatus.NEVER
822
+ # Since we cannot swap in, none of the requests are swapped in.
823
+ budget = create_token_budget()
824
+ remaining_swapped, output = scheduler._schedule_swapped(
825
+ swapped, budget, curr_loras, policy)
826
+ assert len(remaining_swapped) == 0
827
+ assert len(output.infeasible_seq_groups) == 2
828
+ assert budget.num_batched_tokens == 0
829
+ assert budget.num_curr_seqs == 0
830
+ assert len(output.decode_seq_groups) == 0
831
+ assert len(output.prefill_seq_groups) == 0
832
+
833
+
834
def test_schedule_swapped_blocks_to_copy():
835
scheduler = initialize_scheduler()
836
swapped = deque()
vllm/core/block/cpu_gpu_block_allocator.py
+8
-11
@@ -110,9 +110,8 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
110
for block_id in allocator.all_block_ids:
111
self._block_ids_to_allocator[block_id] = allocator
112
113
- def allocate_mutable(self,
114
- prev_block: Optional[Block],
115
- device: Optional[Device] = None) -> Block:
113
+ def allocate_mutable(self, prev_block: Optional[Block],
114
+ device: Device) -> Block:
115
"""Allocates a new mutable block on the specified device.
116
117
Args:
@@ -123,13 +122,10 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
122
Returns:
123
Block: The newly allocated mutable block.
124
"""
126
- assert device is not None
125
return self._allocators[device].allocate_mutable(prev_block)
126
129
- def allocate_immutable(self,
130
- prev_block: Optional[Block],
131
- token_ids: List[int],
132
- device: Optional[Device] = None) -> Block:
127
+ def allocate_immutable(self, prev_block: Optional[Block],
128
+ token_ids: List[int], device: Device) -> Block:
129
"""Allocates a new immutable block with the provided token IDs on the
130
specified device.
131
@@ -144,7 +140,6 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
140
Block: The newly allocated immutable block containing the provided
141
token IDs.
142
"""
147
- assert device is not None
143
return self._allocators[device].allocate_immutable(
144
prev_block, token_ids)
145
@@ -175,7 +170,7 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
170
allocator = self._block_ids_to_allocator[block_id]
171
return allocator.fork(last_block)
172
178
- def get_num_free_blocks(self, device: Optional[Device] = None) -> int:
173
+ def get_num_free_blocks(self, device: Device) -> int:
174
"""Returns the number of free blocks available on the specified device.
175
176
Args:
@@ -185,9 +180,11 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
180
Returns:
181
int: The number of free blocks available on the specified device.
182
"""
188
- assert device is not None
183
return self._allocators[device].get_num_free_blocks()
184
185
+ def get_num_total_blocks(self, device: Device) -> int:
186
+ return self._allocators[device].get_num_total_blocks()
187
+
188
def clear_copy_on_writes(self) -> Dict[int, List[int]]:
189
"""Clears the copy-on-write (CoW) state and returns the mapping of
190
source to destination block IDs.
vllm/core/block/interfaces.py
+13
-8
@@ -108,6 +108,10 @@ class BlockAllocator(ABC):
108
def fork(self, last_block: Block) -> List[Block]:
109
pass
110
111
+ @abstractmethod
112
+ def get_num_total_blocks(self) -> int:
113
+ pass
114
+
115
@abstractmethod
116
def get_num_free_blocks(self) -> int:
117
pass
@@ -152,20 +156,21 @@ class BlockAllocator(ABC):
156
class DeviceAwareBlockAllocator(ABC):
157
158
@abstractmethod
155
- def allocate_mutable(self,
156
- prev_block: Optional[Block],
157
- device: Optional[Device] = None) -> Block:
159
+ def allocate_mutable(self, prev_block: Optional[Block],
160
+ device: Device) -> Block:
161
+ pass
162
+
163
+ @abstractmethod
164
+ def allocate_immutable(self, prev_block: Optional[Block],
165
+ token_ids: List[int], device: Device) -> Block:
166
pass
167
168
@abstractmethod
161
- def allocate_immutable(self,
162
- prev_block: Optional[Block],
163
- token_ids: List[int],
164
- device: Optional[Device] = None) -> Block:
169
+ def get_num_free_blocks(self, device: Device) -> int:
170
pass
171
172
@abstractmethod
168
- def get_num_free_blocks(self, device: Optional[Device] = None) -> int:
173
+ def get_num_total_blocks(self, device: Device) -> int:
174
pass
175
176
@abstractmethod
vllm/core/block/naive_block.py
+4
-2
@@ -133,10 +133,12 @@ class NaiveBlockAllocator(BlockAllocator):
133
134
return forked_blocks
135
136
- def get_num_free_blocks(self, device: Optional[Device] = None) -> int:
137
- assert device is None
136
+ def get_num_free_blocks(self) -> int:
137
return len(self._free_block_indices)
138
139
+ def get_num_total_blocks(self) -> int:
140
+ return len(self._all_block_indices)
141
+
142
def _allocate_new_block_id(self) -> BlockId:
143
if not self._free_block_indices:
144
raise BlockAllocator.NoFreeBlocksError()
vllm/core/block/prefix_caching_block.py
+3
@@ -285,6 +285,9 @@ class PrefixCachingBlockAllocator(BlockAllocator):
285
return self._hashless_allocator.get_num_free_blocks(
286
) + self.evictor.num_blocks
287
288
+ def get_num_total_blocks(self) -> int:
289
+ return self._hashless_allocator.get_num_total_blocks()
290
+
291
@property
292
def all_block_ids(self) -> FrozenSet[int]:
293
return self._hashless_allocator.all_block_ids
vllm/core/block_manager_v1.py
+17
-2
@@ -47,6 +47,10 @@ class BlockAllocatorBase(ABC):
47
def get_num_free_blocks(self) -> int:
48
pass
49
50
+ @abstractmethod
51
+ def get_num_total_blocks(self) -> int:
52
+ pass
53
+
54
@abstractmethod
55
def contains_block(self, block_hash: int) -> bool:
56
pass
@@ -131,6 +135,9 @@ class CachedBlockAllocator(BlockAllocatorBase):
135
return (self.num_blocks - self.current_num_blocks +
136
self.evictor.num_blocks)
137
138
+ def get_num_total_blocks(self) -> int:
139
+ return self.num_blocks
140
+
141
def contains_block(self, block_hash: int) -> bool:
142
return block_hash in self.cached_blocks or block_hash in self.evictor
143
@@ -190,6 +197,9 @@ class UncachedBlockAllocator(BlockAllocatorBase):
197
def get_num_free_blocks(self) -> int:
198
return len(self.free_blocks)
199
200
+ def get_num_total_blocks(self) -> int:
201
+ return self.num_blocks
202
+
203
def contains_block(self, block_hash: int) -> bool:
204
raise NotImplementedError(
205
"Invalid codepath for uncached block allocator.")
@@ -444,7 +454,7 @@ class BlockSpaceManagerV1(BlockSpaceManager):
454
455
def can_swap_in(self,
456
seq_group: SequenceGroup,
447
- num_lookahead_slots: int = 0) -> bool:
457
+ num_lookahead_slots: int = 0) -> AllocStatus:
458
assert (num_lookahead_slots == 0
459
), "BlockSpaceManagerV1 does not support lookahead allocation"
460
blocks = self._get_physical_blocks(seq_group)
@@ -454,7 +464,12 @@ class BlockSpaceManagerV1(BlockSpaceManager):
464
# at least one free block right after the swap-in.
465
# NOTE: This should match the logic in can_append_slot().
466
num_required_blocks = len(blocks) + num_swapped_seqs
457
- return num_free_blocks - num_required_blocks >= self.watermark_blocks
467
+ if self.gpu_allocator.get_num_total_blocks() < num_required_blocks:
468
+ return AllocStatus.NEVER
469
+ elif num_free_blocks - num_required_blocks >= self.watermark_blocks:
470
+ return AllocStatus.OK
471
+ else:
472
+ return AllocStatus.LATER
473
474
def swap_in(self,
475
seq_group: SequenceGroup,
vllm/core/block_manager_v2.py
+2
-2
@@ -238,8 +238,8 @@ class BlockSpaceManagerV2(BlockSpaceManager):
238
self.block_tables[child_seq.seq_id] = src_block_table.fork()
239
240
def can_swap_in(self, seq_group: SequenceGroup,
241
- num_lookahead_slots: int) -> bool:
242
- return False
241
+ num_lookahead_slots: int) -> AllocStatus:
242
+ return AllocStatus.LATER
243
244
def swap_in(self, seq_group: SequenceGroup,
245
num_lookahead_slots: int) -> Dict[int, int]:
vllm/core/interfaces.py
+1
-1
@@ -63,7 +63,7 @@ class BlockSpaceManager(ABC):
63
64
@abstractmethod
65
def can_swap_in(self, seq_group: SequenceGroup,
66
- num_lookahead_slots: int) -> bool:
66
+ num_lookahead_slots: int) -> AllocStatus:
67
pass
68
69
@abstractmethod
vllm/core/scheduler.py
+21
-12
@@ -210,6 +210,8 @@ class SchedulerSwappedInOutputs:
210
blocks_to_copy: Dict[int, List[int]]
211
# The number of slots for lookahead decoding.
212
num_lookahead_slots: int
213
+ # Infeasible sequence groups.
214
+ infeasible_seq_groups: List[SequenceGroup]
215
216
@classmethod
217
def create_empty(cls) -> "SchedulerSwappedInOutputs":
@@ -219,6 +221,7 @@ class SchedulerSwappedInOutputs:
221
blocks_to_swap_in={},
222
blocks_to_copy={},
223
num_lookahead_slots=0,
224
+ infeasible_seq_groups=[],
225
)
226
227
@@ -511,14 +514,26 @@ class Scheduler:
514
prefill_seq_groups: List[ScheduledSequenceGroup] = []
515
now = time.time()
516
swapped_queue = policy.sort_by_priority(now, swapped_queue)
517
+ infeasible_seq_groups: List[SequenceGroup] = []
518
519
leftover_swapped: Deque[SequenceGroup] = deque()
520
while swapped_queue:
521
seq_group = swapped_queue[0]
522
523
# If the sequence group cannot be swapped in, stop.
520
- if not self.block_manager.can_swap_in(seq_group):
524
+ alloc_status = self.block_manager.can_swap_in(seq_group)
525
+ if alloc_status == AllocStatus.LATER:
526
break
527
+ elif alloc_status == AllocStatus.NEVER:
528
+ logger.warning(
529
+ "Failing the request %s because there's not enough kv "
530
+ "cache blocks to run the entire sequence.",
531
+ seq_group.request_id)
532
+ for seq in seq_group.get_seqs():
533
+ seq.status = SequenceStatus.FINISHED_IGNORED
534
+ infeasible_seq_groups.append(seq_group)
535
+ swapped_queue.popleft()
536
+ continue
537
538
lora_int_id = 0
539
if self.lora_enabled:
@@ -569,7 +584,9 @@ class Scheduler:
584
blocks_to_swap_in=blocks_to_swap_in,
585
blocks_to_copy=blocks_to_copy,
586
num_lookahead_slots=self._get_num_lookahead_slots(
572
- is_prefill=False))
587
+ is_prefill=False),
588
+ infeasible_seq_groups=infeasible_seq_groups,
589
+ )
590
591
def _schedule_prefills(
592
self,
@@ -777,7 +794,8 @@ class Scheduler:
794
blocks_to_swap_out=running_scheduled.blocks_to_swap_out,
795
blocks_to_copy=merge_dicts(running_scheduled.blocks_to_copy,
796
swapped_in.blocks_to_copy),
780
- ignored_seq_groups=prefills.ignored_seq_groups,
797
+ ignored_seq_groups=prefills.ignored_seq_groups +
798
+ swapped_in.infeasible_seq_groups,
799
num_lookahead_slots=running_scheduled.num_lookahead_slots,
800
)
801
@@ -893,15 +911,6 @@ class Scheduler:
911
num_lookahead_slots=self._get_num_lookahead_slots(is_prefill),
912
)
913
896
- def _can_swap_in(self, seq_group: SequenceGroup) -> bool:
897
- # Swapping in is considered decode.
898
- is_prefill = False
899
-
900
- return self.block_manager.can_swap_in(
901
- seq_group=seq_group,
902
- num_lookahead_slots=self._get_num_lookahead_slots(is_prefill),
903
- )
904
-
914
def schedule(self) -> Tuple[List[SequenceGroupMetadata], SchedulerOutputs]:
915
# Schedule sequence groups.
916
# This function call changes the internal states of the scheduler