@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0f8a91401c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Core] Ignore infeasible swap requests. (#4557)

SangBin Cho committed May 3, 2024 at 06:31 UTC 0f8a91401c89ac0a8018def3756829611b57727f
12 files changed +187 -42
tests/basic_correctness/test_preemption.py
+85
@@ -7,6 +7,7 @@ pytest tests/basic_correctness/test_preemption.py`.
7 """
8 import pytest
9
10 +from vllm import SamplingParams
11 from vllm.core.scheduler import (ARTIFICIAL_PREEMPTION_MAX_CNT,
12 ENABLE_ARTIFICIAL_PREEMPT)
13
@@ -136,3 +137,87 @@ def test_swap(
137 assert hf_output_ids[j] == vllm_output_ids[j], (
138 f"Test{i} output{j}:\nHF: {hf_output_ids}\n"
139 f"vLLM: {vllm_output_ids}")
140 +
141 +
142 +@pytest.mark.parametrize("model", MODELS)
143 +@pytest.mark.parametrize("dtype", ["float"])
144 +@pytest.mark.parametrize("max_tokens", [96])
145 +@pytest.mark.parametrize("beam_width", [4])
146 +def test_swap_infeasible(
147 + vllm_runner,
148 + example_prompts,
149 + model: str,
150 + dtype: str,
151 + max_tokens: int,
152 + beam_width: int,
153 +) -> None:
154 + """Verify infeasible swap request will be ignored."""
155 + BLOCK_SIZE = 16
156 + prefill_blocks = 2
157 + decode_blocks = max_tokens // BLOCK_SIZE
158 + example_prompts = example_prompts[:1]
159 +
160 + vllm_model = vllm_runner(
161 + model,
162 + dtype=dtype,
163 + swap_space=10,
164 + block_size=BLOCK_SIZE,
165 + # Since beam search have more than 1 sequence, prefill + decode blocks
166 + # are not enough to finish.
167 + num_gpu_blocks_override=prefill_blocks + decode_blocks,
168 + max_model_len=(prefill_blocks + decode_blocks) * BLOCK_SIZE,
169 + )
170 + sampling_params = SamplingParams(n=beam_width,
171 + use_beam_search=True,
172 + temperature=0.0,
173 + max_tokens=max_tokens,
174 + ignore_eos=True)
175 + req_outputs = vllm_model.model.generate(
176 + example_prompts,
177 + sampling_params=sampling_params,
178 + )
179 + assert (vllm_model.model.llm_engine.scheduler.artificial_preempt_cnt <
180 + ARTIFICIAL_PREEMPTION_MAX_CNT)
181 + del vllm_model
182 + # Verify the request is ignored and not hang.
183 + assert req_outputs[0].outputs[0].finish_reason == "length"
184 +
185 +
186 +@pytest.mark.parametrize("model", MODELS)
187 +@pytest.mark.parametrize("dtype", ["float"])
188 +@pytest.mark.parametrize("max_tokens", [96])
189 +def test_preemption_infeasible(
190 + vllm_runner,
191 + example_prompts,
192 + model: str,
193 + dtype: str,
194 + max_tokens: int,
195 +) -> None:
196 + """Verify infeasible preemption request will be ignored."""
197 + BLOCK_SIZE = 16
198 + prefill_blocks = 2
199 + decode_blocks = max_tokens // BLOCK_SIZE
200 + vllm_model = vllm_runner(
201 + model,
202 + dtype=dtype,
203 + block_size=BLOCK_SIZE,
204 + # Not enough gpu blocks to complete a single sequence.
205 + # preemption should happen, and the sequence should be
206 + # ignored instead of hanging forever.
207 + num_gpu_blocks_override=prefill_blocks + decode_blocks // 2,
208 + max_model_len=((prefill_blocks + decode_blocks // 2) * BLOCK_SIZE),
209 + )
210 + sampling_params = SamplingParams(max_tokens=max_tokens, ignore_eos=True)
211 + req_outputs = vllm_model.model.generate(
212 + example_prompts,
213 + sampling_params=sampling_params,
214 + )
215 +
216 + assert (vllm_model.model.llm_engine.scheduler.artificial_preempt_cnt <
217 + ARTIFICIAL_PREEMPTION_MAX_CNT)
218 + del vllm_model
219 + # Verify the request is ignored and not hang.
220 + for req_output in req_outputs:
221 + outputs = req_output.outputs
222 + assert len(outputs) == 1
223 + assert outputs[0].finish_reason == "length"
tests/core/test_block_manager.py
+1 -1
@@ -224,7 +224,7 @@ def test_swap():
224
225 # Swap seq group from CPU -> GPU.
226 cpu_blocks = block_manager.get_block_table(prompt)
227 - assert block_manager.can_swap_in(seq_group)
227 + assert block_manager.can_swap_in(seq_group) == AllocStatus.OK
228 before_cpu_blocks = block_manager.get_num_free_cpu_blocks()
229 before_gpu_blocks = block_manager.get_num_free_gpu_blocks()
230 mapping = block_manager.swap_in(seq_group)
tests/core/test_chunked_prefill_scheduler.py
+3 -2
@@ -4,6 +4,7 @@ from unittest.mock import MagicMock
4 import pytest # noqa
5
6 from vllm.config import CacheConfig, SchedulerConfig
7 +from vllm.core.interfaces import AllocStatus
8 from vllm.core.scheduler import Scheduler
9 from vllm.sequence import Logprob, SequenceGroup
10
@@ -410,7 +411,7 @@ def test_running_prefill_prioritized_over_swap():
411
412 # Add 1 more task. Swap is not possible, so prefill is running.
413 scheduler.block_manager.can_swap_in = MagicMock()
413 - scheduler.block_manager.can_swap_in.return_value = False
414 + scheduler.block_manager.can_swap_in.return_value = AllocStatus.LATER
415
416 _, seq_group2 = create_dummy_prompt("2", prompt_length=60)
417 scheduler.add_seq_group(seq_group2)
@@ -423,7 +424,7 @@ def test_running_prefill_prioritized_over_swap():
424 assert out.scheduled_seq_groups[0].seq_group == seq_group2
425
426 # Now although swap is possible, running prefill is prioritized.
426 - scheduler.block_manager.can_swap_in.return_value = True
427 + scheduler.block_manager.can_swap_in.return_value = AllocStatus.OK
428 _, out = schedule_and_update_computed_tokens(scheduler)
429 assert len(out.scheduled_seq_groups) == 1
430 # 3 decodes. It is swapped in.
tests/core/test_scheduler.py
+29 -1
@@ -791,7 +791,7 @@ def test_schedule_swapped_cannot_swap_in():
791
792 # The last request should be swapped out.
793 scheduler.block_manager.can_swap_in = MagicMock()
794 - scheduler.block_manager.can_swap_in.return_value = False
794 + scheduler.block_manager.can_swap_in.return_value = AllocStatus.LATER
795 # Since we cannot swap in, none of the requests are swapped in.
796 budget = create_token_budget()
797 remaining_swapped, output = scheduler._schedule_swapped(
@@ -803,6 +803,34 @@ def test_schedule_swapped_cannot_swap_in():
803 assert len(output.prefill_seq_groups) == 0
804
805
806 +def test_infeasible_swap():
807 + scheduler = initialize_scheduler()
808 + swapped = deque()
809 + policy = PolicyFactory.get_policy(policy_name="fcfs")
810 + curr_loras = None
811 + blocks_to_swap_out = {}
812 + for _ in range(2):
813 + _, seq_group = create_dummy_prompt("1", prompt_length=60, best_of=2)
814 + scheduler._allocate_and_set_running(seq_group)
815 + append_new_token_seq_group(60, seq_group, 1)
816 + scheduler._swap_out(seq_group, blocks_to_swap_out)
817 + swapped.append(seq_group)
818 +
819 + # The last request should be swapped out.
820 + scheduler.block_manager.can_swap_in = MagicMock()
821 + scheduler.block_manager.can_swap_in.return_value = AllocStatus.NEVER
822 + # Since we cannot swap in, none of the requests are swapped in.
823 + budget = create_token_budget()
824 + remaining_swapped, output = scheduler._schedule_swapped(
825 + swapped, budget, curr_loras, policy)
826 + assert len(remaining_swapped) == 0
827 + assert len(output.infeasible_seq_groups) == 2
828 + assert budget.num_batched_tokens == 0
829 + assert budget.num_curr_seqs == 0
830 + assert len(output.decode_seq_groups) == 0
831 + assert len(output.prefill_seq_groups) == 0
832 +
833 +
834 def test_schedule_swapped_blocks_to_copy():
835 scheduler = initialize_scheduler()
836 swapped = deque()
vllm/core/block/cpu_gpu_block_allocator.py
+8 -11
@@ -110,9 +110,8 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
110 for block_id in allocator.all_block_ids:
111 self._block_ids_to_allocator[block_id] = allocator
112
113 - def allocate_mutable(self,
114 - prev_block: Optional[Block],
115 - device: Optional[Device] = None) -> Block:
113 + def allocate_mutable(self, prev_block: Optional[Block],
114 + device: Device) -> Block:
115 """Allocates a new mutable block on the specified device.
116
117 Args:
@@ -123,13 +122,10 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
122 Returns:
123 Block: The newly allocated mutable block.
124 """
126 - assert device is not None
125 return self._allocators[device].allocate_mutable(prev_block)
126
129 - def allocate_immutable(self,
130 - prev_block: Optional[Block],
131 - token_ids: List[int],
132 - device: Optional[Device] = None) -> Block:
127 + def allocate_immutable(self, prev_block: Optional[Block],
128 + token_ids: List[int], device: Device) -> Block:
129 """Allocates a new immutable block with the provided token IDs on the
130 specified device.
131
@@ -144,7 +140,6 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
140 Block: The newly allocated immutable block containing the provided
141 token IDs.
142 """
147 - assert device is not None
143 return self._allocators[device].allocate_immutable(
144 prev_block, token_ids)
145
@@ -175,7 +170,7 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
170 allocator = self._block_ids_to_allocator[block_id]
171 return allocator.fork(last_block)
172
178 - def get_num_free_blocks(self, device: Optional[Device] = None) -> int:
173 + def get_num_free_blocks(self, device: Device) -> int:
174 """Returns the number of free blocks available on the specified device.
175
176 Args:
@@ -185,9 +180,11 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
180 Returns:
181 int: The number of free blocks available on the specified device.
182 """
188 - assert device is not None
183 return self._allocators[device].get_num_free_blocks()
184
185 + def get_num_total_blocks(self, device: Device) -> int:
186 + return self._allocators[device].get_num_total_blocks()
187 +
188 def clear_copy_on_writes(self) -> Dict[int, List[int]]:
189 """Clears the copy-on-write (CoW) state and returns the mapping of
190 source to destination block IDs.
vllm/core/block/interfaces.py
+13 -8
@@ -108,6 +108,10 @@ class BlockAllocator(ABC):
108 def fork(self, last_block: Block) -> List[Block]:
109 pass
110
111 + @abstractmethod
112 + def get_num_total_blocks(self) -> int:
113 + pass
114 +
115 @abstractmethod
116 def get_num_free_blocks(self) -> int:
117 pass
@@ -152,20 +156,21 @@ class BlockAllocator(ABC):
156 class DeviceAwareBlockAllocator(ABC):
157
158 @abstractmethod
155 - def allocate_mutable(self,
156 - prev_block: Optional[Block],
157 - device: Optional[Device] = None) -> Block:
159 + def allocate_mutable(self, prev_block: Optional[Block],
160 + device: Device) -> Block:
161 + pass
162 +
163 + @abstractmethod
164 + def allocate_immutable(self, prev_block: Optional[Block],
165 + token_ids: List[int], device: Device) -> Block:
166 pass
167
168 @abstractmethod
161 - def allocate_immutable(self,
162 - prev_block: Optional[Block],
163 - token_ids: List[int],
164 - device: Optional[Device] = None) -> Block:
169 + def get_num_free_blocks(self, device: Device) -> int:
170 pass
171
172 @abstractmethod
168 - def get_num_free_blocks(self, device: Optional[Device] = None) -> int:
173 + def get_num_total_blocks(self, device: Device) -> int:
174 pass
175
176 @abstractmethod
vllm/core/block/naive_block.py
+4 -2
@@ -133,10 +133,12 @@ class NaiveBlockAllocator(BlockAllocator):
133
134 return forked_blocks
135
136 - def get_num_free_blocks(self, device: Optional[Device] = None) -> int:
137 - assert device is None
136 + def get_num_free_blocks(self) -> int:
137 return len(self._free_block_indices)
138
139 + def get_num_total_blocks(self) -> int:
140 + return len(self._all_block_indices)
141 +
142 def _allocate_new_block_id(self) -> BlockId:
143 if not self._free_block_indices:
144 raise BlockAllocator.NoFreeBlocksError()
vllm/core/block/prefix_caching_block.py
+3
@@ -285,6 +285,9 @@ class PrefixCachingBlockAllocator(BlockAllocator):
285 return self._hashless_allocator.get_num_free_blocks(
286 ) + self.evictor.num_blocks
287
288 + def get_num_total_blocks(self) -> int:
289 + return self._hashless_allocator.get_num_total_blocks()
290 +
291 @property
292 def all_block_ids(self) -> FrozenSet[int]:
293 return self._hashless_allocator.all_block_ids
vllm/core/block_manager_v1.py
+17 -2
@@ -47,6 +47,10 @@ class BlockAllocatorBase(ABC):
47 def get_num_free_blocks(self) -> int:
48 pass
49
50 + @abstractmethod
51 + def get_num_total_blocks(self) -> int:
52 + pass
53 +
54 @abstractmethod
55 def contains_block(self, block_hash: int) -> bool:
56 pass
@@ -131,6 +135,9 @@ class CachedBlockAllocator(BlockAllocatorBase):
135 return (self.num_blocks - self.current_num_blocks +
136 self.evictor.num_blocks)
137
138 + def get_num_total_blocks(self) -> int:
139 + return self.num_blocks
140 +
141 def contains_block(self, block_hash: int) -> bool:
142 return block_hash in self.cached_blocks or block_hash in self.evictor
143
@@ -190,6 +197,9 @@ class UncachedBlockAllocator(BlockAllocatorBase):
197 def get_num_free_blocks(self) -> int:
198 return len(self.free_blocks)
199
200 + def get_num_total_blocks(self) -> int:
201 + return self.num_blocks
202 +
203 def contains_block(self, block_hash: int) -> bool:
204 raise NotImplementedError(
205 "Invalid codepath for uncached block allocator.")
@@ -444,7 +454,7 @@ class BlockSpaceManagerV1(BlockSpaceManager):
454
455 def can_swap_in(self,
456 seq_group: SequenceGroup,
447 - num_lookahead_slots: int = 0) -> bool:
457 + num_lookahead_slots: int = 0) -> AllocStatus:
458 assert (num_lookahead_slots == 0
459 ), "BlockSpaceManagerV1 does not support lookahead allocation"
460 blocks = self._get_physical_blocks(seq_group)
@@ -454,7 +464,12 @@ class BlockSpaceManagerV1(BlockSpaceManager):
464 # at least one free block right after the swap-in.
465 # NOTE: This should match the logic in can_append_slot().
466 num_required_blocks = len(blocks) + num_swapped_seqs
457 - return num_free_blocks - num_required_blocks >= self.watermark_blocks
467 + if self.gpu_allocator.get_num_total_blocks() < num_required_blocks:
468 + return AllocStatus.NEVER
469 + elif num_free_blocks - num_required_blocks >= self.watermark_blocks:
470 + return AllocStatus.OK
471 + else:
472 + return AllocStatus.LATER
473
474 def swap_in(self,
475 seq_group: SequenceGroup,
vllm/core/block_manager_v2.py
+2 -2
@@ -238,8 +238,8 @@ class BlockSpaceManagerV2(BlockSpaceManager):
238 self.block_tables[child_seq.seq_id] = src_block_table.fork()
239
240 def can_swap_in(self, seq_group: SequenceGroup,
241 - num_lookahead_slots: int) -> bool:
242 - return False
241 + num_lookahead_slots: int) -> AllocStatus:
242 + return AllocStatus.LATER
243
244 def swap_in(self, seq_group: SequenceGroup,
245 num_lookahead_slots: int) -> Dict[int, int]:
vllm/core/interfaces.py
+1 -1
@@ -63,7 +63,7 @@ class BlockSpaceManager(ABC):
63
64 @abstractmethod
65 def can_swap_in(self, seq_group: SequenceGroup,
66 - num_lookahead_slots: int) -> bool:
66 + num_lookahead_slots: int) -> AllocStatus:
67 pass
68
69 @abstractmethod
vllm/core/scheduler.py
+21 -12
@@ -210,6 +210,8 @@ class SchedulerSwappedInOutputs:
210 blocks_to_copy: Dict[int, List[int]]
211 # The number of slots for lookahead decoding.
212 num_lookahead_slots: int
213 + # Infeasible sequence groups.
214 + infeasible_seq_groups: List[SequenceGroup]
215
216 @classmethod
217 def create_empty(cls) -> "SchedulerSwappedInOutputs":
@@ -219,6 +221,7 @@ class SchedulerSwappedInOutputs:
221 blocks_to_swap_in={},
222 blocks_to_copy={},
223 num_lookahead_slots=0,
224 + infeasible_seq_groups=[],
225 )
226
227
@@ -511,14 +514,26 @@ class Scheduler:
514 prefill_seq_groups: List[ScheduledSequenceGroup] = []
515 now = time.time()
516 swapped_queue = policy.sort_by_priority(now, swapped_queue)
517 + infeasible_seq_groups: List[SequenceGroup] = []
518
519 leftover_swapped: Deque[SequenceGroup] = deque()
520 while swapped_queue:
521 seq_group = swapped_queue[0]
522
523 # If the sequence group cannot be swapped in, stop.
520 - if not self.block_manager.can_swap_in(seq_group):
524 + alloc_status = self.block_manager.can_swap_in(seq_group)
525 + if alloc_status == AllocStatus.LATER:
526 break
527 + elif alloc_status == AllocStatus.NEVER:
528 + logger.warning(
529 + "Failing the request %s because there's not enough kv "
530 + "cache blocks to run the entire sequence.",
531 + seq_group.request_id)
532 + for seq in seq_group.get_seqs():
533 + seq.status = SequenceStatus.FINISHED_IGNORED
534 + infeasible_seq_groups.append(seq_group)
535 + swapped_queue.popleft()
536 + continue
537
538 lora_int_id = 0
539 if self.lora_enabled:
@@ -569,7 +584,9 @@ class Scheduler:
584 blocks_to_swap_in=blocks_to_swap_in,
585 blocks_to_copy=blocks_to_copy,
586 num_lookahead_slots=self._get_num_lookahead_slots(
572 - is_prefill=False))
587 + is_prefill=False),
588 + infeasible_seq_groups=infeasible_seq_groups,
589 + )
590
591 def _schedule_prefills(
592 self,
@@ -777,7 +794,8 @@ class Scheduler:
794 blocks_to_swap_out=running_scheduled.blocks_to_swap_out,
795 blocks_to_copy=merge_dicts(running_scheduled.blocks_to_copy,
796 swapped_in.blocks_to_copy),
780 - ignored_seq_groups=prefills.ignored_seq_groups,
797 + ignored_seq_groups=prefills.ignored_seq_groups +
798 + swapped_in.infeasible_seq_groups,
799 num_lookahead_slots=running_scheduled.num_lookahead_slots,
800 )
801
@@ -893,15 +911,6 @@ class Scheduler:
911 num_lookahead_slots=self._get_num_lookahead_slots(is_prefill),
912 )
913
896 - def _can_swap_in(self, seq_group: SequenceGroup) -> bool:
897 - # Swapping in is considered decode.
898 - is_prefill = False
899 -
900 - return self.block_manager.can_swap_in(
901 - seq_group=seq_group,
902 - num_lookahead_slots=self._get_num_lookahead_slots(is_prefill),
903 - )
904 -
914 def schedule(self) -> Tuple[List[SequenceGroupMetadata], SchedulerOutputs]:
915 # Schedule sequence groups.
916 # This function call changes the internal states of the scheduler