A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Fix hanging in the scheduler caused by long prompts (#1534)
陈序 committed
Nov 21, 2023 at 08:06 UTC
3d4ceb292c1c03863daf2316b8123b17307050c3
2 files changed
+36
-5
vllm/core/block_manager.py
+24
-3
@@ -1,4 +1,5 @@
1
"""A block manager that manages token blocks."""
2
+import enum
3
from typing import Dict, List, Optional, Set, Tuple
4
5
from vllm.block import PhysicalTokenBlock
@@ -54,6 +55,20 @@ class BlockAllocator:
55
BlockTable = List[PhysicalTokenBlock]
56
57
58
+class AllocStatus(enum.Enum):
59
+ """Result for BlockSpaceManager.can_allocate
60
+
61
+ 1. Ok: seq_group can be allocated now.
62
+ 2. Later: seq_group cannot be allocated.
63
+ The capacity of allocator is larger than seq_group required.
64
+ 3. Never: seq_group can never be allocated.
65
+ The seq_group is too large to allocated in GPU.
66
+ """
67
+ OK = enum.auto()
68
+ LATER = enum.auto()
69
+ NEVER = enum.auto()
70
+
71
+
72
class BlockSpaceManager:
73
"""Manages the mapping between logical and physical token blocks."""
74
@@ -86,7 +101,7 @@ class BlockSpaceManager:
101
# Mapping: seq_id -> BlockTable.
102
self.block_tables: Dict[int, BlockTable] = {}
103
89
- def can_allocate(self, seq_group: SequenceGroup) -> bool:
104
+ def can_allocate(self, seq_group: SequenceGroup) -> AllocStatus:
105
# FIXME(woosuk): Here we assume that all sequences in the group share
106
# the same prompt. This may not be true for preempted sequences.
107
seq = seq_group.get_seqs()[0]
@@ -95,9 +110,15 @@ class BlockSpaceManager:
110
num_required_blocks = min(num_required_blocks,
111
self.block_sliding_window)
112
num_free_gpu_blocks = self.gpu_allocator.get_num_free_blocks()
113
+
114
# Use watermark to avoid frequent cache eviction.
99
- return (num_free_gpu_blocks - num_required_blocks >=
100
- self.watermark_blocks)
115
+ if (self.num_total_gpu_blocks - num_required_blocks <
116
+ self.watermark_blocks):
117
+ return AllocStatus.NEVER
118
+ if num_free_gpu_blocks - num_required_blocks >= self.watermark_blocks:
119
+ return AllocStatus.OK
120
+ else:
121
+ return AllocStatus.LATER
122
123
def allocate(self, seq_group: SequenceGroup) -> None:
124
# NOTE: Here we assume that all sequences in the group have the same
vllm/core/scheduler.py
+12
-2
@@ -3,7 +3,7 @@ import time
3
from typing import Dict, Iterable, List, Optional, Tuple, Union
4
5
from vllm.config import CacheConfig, SchedulerConfig
6
-from vllm.core.block_manager import BlockSpaceManager
6
+from vllm.core.block_manager import AllocStatus, BlockSpaceManager
7
from vllm.core.policy import PolicyFactory
8
from vllm.logger import init_logger
9
from vllm.sequence import (Sequence, SequenceData, SequenceGroup,
@@ -154,8 +154,18 @@ class Scheduler:
154
continue
155
156
# If the sequence group cannot be allocated, stop.
157
- if not self.block_manager.can_allocate(seq_group):
157
+ can_allocate = self.block_manager.can_allocate(seq_group)
158
+ if can_allocate == AllocStatus.LATER:
159
break
160
+ elif can_allocate == AllocStatus.NEVER:
161
+ logger.warning(
162
+ f"Input prompt ({num_prompt_tokens} tokens) is too long"
163
+ f" and exceeds the capacity of block_manager")
164
+ for seq in seq_group.get_seqs():
165
+ seq.status = SequenceStatus.FINISHED_IGNORED
166
+ ignored_seq_groups.append(seq_group)
167
+ self.waiting.pop(0)
168
+ continue
169
170
# If the number of batched tokens exceeds the limit, stop.
171
new_seq_lens = seq_lens + [num_prompt_tokens]