@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3d4ceb292c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Fix hanging in the scheduler caused by long prompts (#1534)

陈序 committed Nov 21, 2023 at 08:06 UTC 3d4ceb292c1c03863daf2316b8123b17307050c3
2 files changed +36 -5
vllm/core/block_manager.py
+24 -3
@@ -1,4 +1,5 @@
1 """A block manager that manages token blocks."""
2 +import enum
3 from typing import Dict, List, Optional, Set, Tuple
4
5 from vllm.block import PhysicalTokenBlock
@@ -54,6 +55,20 @@ class BlockAllocator:
55 BlockTable = List[PhysicalTokenBlock]
56
57
58 +class AllocStatus(enum.Enum):
59 + """Result for BlockSpaceManager.can_allocate
60 +
61 + 1. Ok: seq_group can be allocated now.
62 + 2. Later: seq_group cannot be allocated.
63 + The capacity of allocator is larger than seq_group required.
64 + 3. Never: seq_group can never be allocated.
65 + The seq_group is too large to allocated in GPU.
66 + """
67 + OK = enum.auto()
68 + LATER = enum.auto()
69 + NEVER = enum.auto()
70 +
71 +
72 class BlockSpaceManager:
73 """Manages the mapping between logical and physical token blocks."""
74
@@ -86,7 +101,7 @@ class BlockSpaceManager:
101 # Mapping: seq_id -> BlockTable.
102 self.block_tables: Dict[int, BlockTable] = {}
103
89 - def can_allocate(self, seq_group: SequenceGroup) -> bool:
104 + def can_allocate(self, seq_group: SequenceGroup) -> AllocStatus:
105 # FIXME(woosuk): Here we assume that all sequences in the group share
106 # the same prompt. This may not be true for preempted sequences.
107 seq = seq_group.get_seqs()[0]
@@ -95,9 +110,15 @@ class BlockSpaceManager:
110 num_required_blocks = min(num_required_blocks,
111 self.block_sliding_window)
112 num_free_gpu_blocks = self.gpu_allocator.get_num_free_blocks()
113 +
114 # Use watermark to avoid frequent cache eviction.
99 - return (num_free_gpu_blocks - num_required_blocks >=
100 - self.watermark_blocks)
115 + if (self.num_total_gpu_blocks - num_required_blocks <
116 + self.watermark_blocks):
117 + return AllocStatus.NEVER
118 + if num_free_gpu_blocks - num_required_blocks >= self.watermark_blocks:
119 + return AllocStatus.OK
120 + else:
121 + return AllocStatus.LATER
122
123 def allocate(self, seq_group: SequenceGroup) -> None:
124 # NOTE: Here we assume that all sequences in the group have the same
vllm/core/scheduler.py
+12 -2
@@ -3,7 +3,7 @@ import time
3 from typing import Dict, Iterable, List, Optional, Tuple, Union
4
5 from vllm.config import CacheConfig, SchedulerConfig
6 -from vllm.core.block_manager import BlockSpaceManager
6 +from vllm.core.block_manager import AllocStatus, BlockSpaceManager
7 from vllm.core.policy import PolicyFactory
8 from vllm.logger import init_logger
9 from vllm.sequence import (Sequence, SequenceData, SequenceGroup,
@@ -154,8 +154,18 @@ class Scheduler:
154 continue
155
156 # If the sequence group cannot be allocated, stop.
157 - if not self.block_manager.can_allocate(seq_group):
157 + can_allocate = self.block_manager.can_allocate(seq_group)
158 + if can_allocate == AllocStatus.LATER:
159 break
160 + elif can_allocate == AllocStatus.NEVER:
161 + logger.warning(
162 + f"Input prompt ({num_prompt_tokens} tokens) is too long"
163 + f" and exceeds the capacity of block_manager")
164 + for seq in seq_group.get_seqs():
165 + seq.status = SequenceStatus.FINISHED_IGNORED
166 + ignored_seq_groups.append(seq_group)
167 + self.waiting.pop(0)
168 + continue
169
170 # If the number of batched tokens exceeds the limit, stop.
171 new_seq_lens = seq_lens + [num_prompt_tokens]