A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
Raise error for long prompt (#273)
Lily Liu committed
Jun 30, 2023 at 18:48 UTC
dafd924c1f165b4478a9d7a3c915d2ecc2e148e2
5 files changed
+42
-11
vllm/config.py
+4
@@ -186,14 +186,18 @@ class SchedulerConfig:
186
a single iteration.
187
max_num_seqs: Maximum number of sequences to be processed in a single
188
iteration.
189
+ max_seq_len: Maximum length of a sequence (including prompt
190
+ and generated text).
191
"""
192
def __init__(
193
self,
194
max_num_batched_tokens: int,
195
max_num_seqs: int,
196
+ max_seq_len: int
197
) -> None:
198
self.max_num_batched_tokens = max_num_batched_tokens
199
self.max_num_seqs = max_num_seqs
200
+ self.max_seq_len = max_seq_len
201
202
203
_STR_DTYPE_TO_TORCH_DTYPE = {
vllm/core/scheduler.py
+20
-7
@@ -102,11 +102,12 @@ class Scheduler:
102
def get_num_unfinished_seq_groups(self) -> int:
103
return len(self.waiting) + len(self.running) + len(self.swapped)
104
105
- def _schedule(self) -> Tuple[SchedulerOutputs, List[str]]:
105
+ def _schedule(self) -> Tuple[SchedulerOutputs, List[str], List[SequenceGroup]]:
106
# Blocks that need to be swaped or copied before model execution.
107
blocks_to_swap_in: Dict[int, int] = {}
108
blocks_to_swap_out: Dict[int, int] = {}
109
blocks_to_copy: Dict[int, List[int]] = {}
110
+ ignored_seq_groups: List[SequenceGroup] = []
111
112
# Fix the current time.
113
now = time.time()
@@ -187,12 +188,24 @@ class Scheduler:
188
# If the sequence group has been preempted in this step, stop.
189
if seq_group in preempted:
190
break
191
+
192
+ num_prompt_tokens = seq_group.get_seqs()[0].get_len()
193
+ if num_prompt_tokens >= self.scheduler_config.max_seq_len:
194
+ logger.warn(
195
+ f"Input prompt ({num_prompt_tokens} tokens) is too long"
196
+ " and exceeds limit of "
197
+ f"{self.scheduler_config.max_seq_len}")
198
+ for seq in seq_group.get_seqs():
199
+ seq.status = SequenceStatus.FINISHED_IGNORED
200
+ ignored_seq_groups.append(seq_group)
201
+ self.waiting.pop(0)
202
+ break
203
+
204
# If the sequence group cannot be allocated, stop.
205
if not self.block_manager.can_allocate(seq_group):
206
break
207
208
# If the number of batched tokens exceeds the limit, stop.
195
- num_prompt_tokens = seq_group.get_seqs()[0].get_len()
209
if (num_batched_tokens + num_prompt_tokens
210
> self.scheduler_config.max_num_batched_tokens):
211
break
@@ -218,7 +231,7 @@ class Scheduler:
231
blocks_to_copy=blocks_to_copy,
232
)
233
if not self.log_stats:
221
- return scheduler_outputs, prompt_group_ids
234
+ return scheduler_outputs, prompt_group_ids, ignored_seq_groups
235
236
# TODO(woosuk): Move the below code to the engine.
237
now = time.time()
@@ -258,13 +271,13 @@ class Scheduler:
271
f"Pending: {len(self.waiting)} reqs, "
272
f"GPU KV cache usage: {gpu_cache_usage * 100:.1f}%, "
273
f"CPU KV cache usage: {cpu_cache_usage * 100:.1f}%")
261
- return scheduler_outputs, prompt_group_ids
274
+ return scheduler_outputs, prompt_group_ids, ignored_seq_groups
275
263
- def schedule(self) -> Tuple[List[SequenceGroupMetadata], SchedulerOutputs]:
276
+ def schedule(self) -> Tuple[List[SequenceGroupMetadata], SchedulerOutputs, List[SequenceGroup]]:
277
# Schedule sequence groups.
278
# This function call changes the internal states of the scheduler
279
# such as self.running, self.swapped, and self.waiting.
267
- scheduler_outputs, prompt_group_ids = self._schedule()
280
+ scheduler_outputs, prompt_group_ids, ignored_seq_groups = self._schedule()
281
282
# Create input data structures.
283
seq_group_metadata_list: List[SequenceGroupMetadata] = []
@@ -286,7 +299,7 @@ class Scheduler:
299
block_tables=block_tables,
300
)
301
seq_group_metadata_list.append(seq_group_metadata)
289
- return seq_group_metadata_list, scheduler_outputs
302
+ return seq_group_metadata_list, scheduler_outputs, ignored_seq_groups
303
304
def update(
305
self,
vllm/engine/arg_utils.py
+5
-1
@@ -123,8 +123,12 @@ class EngineArgs:
123
parallel_config = ParallelConfig(self.pipeline_parallel_size,
124
self.tensor_parallel_size,
125
self.worker_use_ray)
126
+ max_seq_len = min(
127
+ self.max_num_batched_tokens,
128
+ getattr(model_config.hf_config, "max_position_embeddings",
129
+ float("inf")))
130
scheduler_config = SchedulerConfig(self.max_num_batched_tokens,
127
- self.max_num_seqs)
131
+ self.max_num_seqs, max_seq_len)
132
return model_config, cache_config, parallel_config, scheduler_config
133
134
vllm/engine/llm_engine.py
+9
-3
@@ -226,8 +226,8 @@ class LLMEngine:
226
and updates the scheduler with the model outputs. Finally, it decodes
227
the sequences and returns the newly generated results.
228
"""
229
- seq_group_metadata_list, scheduler_outputs = self.scheduler.schedule()
230
- if (not seq_group_metadata_list) and scheduler_outputs.is_empty():
229
+ seq_group_metadata_list, scheduler_outputs, ignored_seq_groups = self.scheduler.schedule()
230
+ if (not seq_group_metadata_list) and scheduler_outputs.is_empty() and (not ignored_seq_groups):
231
# Nothing to do.
232
return []
233
@@ -251,7 +251,7 @@ class LLMEngine:
251
252
# Create the outputs.
253
request_outputs: List[RequestOutput] = []
254
- for seq_group in seq_groups:
254
+ for seq_group in seq_groups + ignored_seq_groups:
255
request_output = RequestOutput.from_seq_group(seq_group)
256
request_outputs.append(request_output)
257
return request_outputs
@@ -288,6 +288,12 @@ class LLMEngine:
288
if stopped:
289
continue
290
291
+ # Check if the sequence has reached max_seq_len.
292
+ if (seq.get_len() >=
293
+ self.scheduler.scheduler_config.max_seq_len):
294
+ self.scheduler.free_seq(
295
+ seq, SequenceStatus.FINISHED_LENGTH_CAPPED)
296
+ continue
297
# Check if the sequence has reached max_tokens.
298
if seq.get_output_len() == sampling_params.max_tokens:
299
self.scheduler.free_seq(
vllm/sequence.py
+4
@@ -13,6 +13,7 @@ class SequenceStatus(enum.Enum):
13
FINISHED_STOPPED = enum.auto()
14
FINISHED_LENGTH_CAPPED = enum.auto()
15
FINISHED_ABORTED = enum.auto()
16
+ FINISHED_IGNORED = enum.auto()
17
18
@staticmethod
19
def is_finished(status: "SequenceStatus") -> bool:
@@ -20,6 +21,7 @@ class SequenceStatus(enum.Enum):
21
SequenceStatus.FINISHED_STOPPED,
22
SequenceStatus.FINISHED_LENGTH_CAPPED,
23
SequenceStatus.FINISHED_ABORTED,
24
+ SequenceStatus.FINISHED_IGNORED
25
]
26
27
@staticmethod
@@ -30,6 +32,8 @@ class SequenceStatus(enum.Enum):
32
finish_reason = "length"
33
elif status == SequenceStatus.FINISHED_ABORTED:
34
finish_reason = "abort"
35
+ elif status == SequenceStatus.FINISHED_IGNORED:
36
+ finish_reason = "length"
37
else:
38
finish_reason = None
39
return finish_reason