@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / dafd924c1f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

Raise error for long prompt (#273)

Lily Liu committed Jun 30, 2023 at 18:48 UTC dafd924c1f165b4478a9d7a3c915d2ecc2e148e2
5 files changed +42 -11
vllm/config.py
+4
@@ -186,14 +186,18 @@ class SchedulerConfig:
186 a single iteration.
187 max_num_seqs: Maximum number of sequences to be processed in a single
188 iteration.
189 + max_seq_len: Maximum length of a sequence (including prompt
190 + and generated text).
191 """
192 def __init__(
193 self,
194 max_num_batched_tokens: int,
195 max_num_seqs: int,
196 + max_seq_len: int
197 ) -> None:
198 self.max_num_batched_tokens = max_num_batched_tokens
199 self.max_num_seqs = max_num_seqs
200 + self.max_seq_len = max_seq_len
201
202
203 _STR_DTYPE_TO_TORCH_DTYPE = {
vllm/core/scheduler.py
+20 -7
@@ -102,11 +102,12 @@ class Scheduler:
102 def get_num_unfinished_seq_groups(self) -> int:
103 return len(self.waiting) + len(self.running) + len(self.swapped)
104
105 - def _schedule(self) -> Tuple[SchedulerOutputs, List[str]]:
105 + def _schedule(self) -> Tuple[SchedulerOutputs, List[str], List[SequenceGroup]]:
106 # Blocks that need to be swaped or copied before model execution.
107 blocks_to_swap_in: Dict[int, int] = {}
108 blocks_to_swap_out: Dict[int, int] = {}
109 blocks_to_copy: Dict[int, List[int]] = {}
110 + ignored_seq_groups: List[SequenceGroup] = []
111
112 # Fix the current time.
113 now = time.time()
@@ -187,12 +188,24 @@ class Scheduler:
188 # If the sequence group has been preempted in this step, stop.
189 if seq_group in preempted:
190 break
191 +
192 + num_prompt_tokens = seq_group.get_seqs()[0].get_len()
193 + if num_prompt_tokens >= self.scheduler_config.max_seq_len:
194 + logger.warn(
195 + f"Input prompt ({num_prompt_tokens} tokens) is too long"
196 + " and exceeds limit of "
197 + f"{self.scheduler_config.max_seq_len}")
198 + for seq in seq_group.get_seqs():
199 + seq.status = SequenceStatus.FINISHED_IGNORED
200 + ignored_seq_groups.append(seq_group)
201 + self.waiting.pop(0)
202 + break
203 +
204 # If the sequence group cannot be allocated, stop.
205 if not self.block_manager.can_allocate(seq_group):
206 break
207
208 # If the number of batched tokens exceeds the limit, stop.
195 - num_prompt_tokens = seq_group.get_seqs()[0].get_len()
209 if (num_batched_tokens + num_prompt_tokens
210 > self.scheduler_config.max_num_batched_tokens):
211 break
@@ -218,7 +231,7 @@ class Scheduler:
231 blocks_to_copy=blocks_to_copy,
232 )
233 if not self.log_stats:
221 - return scheduler_outputs, prompt_group_ids
234 + return scheduler_outputs, prompt_group_ids, ignored_seq_groups
235
236 # TODO(woosuk): Move the below code to the engine.
237 now = time.time()
@@ -258,13 +271,13 @@ class Scheduler:
271 f"Pending: {len(self.waiting)} reqs, "
272 f"GPU KV cache usage: {gpu_cache_usage * 100:.1f}%, "
273 f"CPU KV cache usage: {cpu_cache_usage * 100:.1f}%")
261 - return scheduler_outputs, prompt_group_ids
274 + return scheduler_outputs, prompt_group_ids, ignored_seq_groups
275
263 - def schedule(self) -> Tuple[List[SequenceGroupMetadata], SchedulerOutputs]:
276 + def schedule(self) -> Tuple[List[SequenceGroupMetadata], SchedulerOutputs, List[SequenceGroup]]:
277 # Schedule sequence groups.
278 # This function call changes the internal states of the scheduler
279 # such as self.running, self.swapped, and self.waiting.
267 - scheduler_outputs, prompt_group_ids = self._schedule()
280 + scheduler_outputs, prompt_group_ids, ignored_seq_groups = self._schedule()
281
282 # Create input data structures.
283 seq_group_metadata_list: List[SequenceGroupMetadata] = []
@@ -286,7 +299,7 @@ class Scheduler:
299 block_tables=block_tables,
300 )
301 seq_group_metadata_list.append(seq_group_metadata)
289 - return seq_group_metadata_list, scheduler_outputs
302 + return seq_group_metadata_list, scheduler_outputs, ignored_seq_groups
303
304 def update(
305 self,
vllm/engine/arg_utils.py
+5 -1
@@ -123,8 +123,12 @@ class EngineArgs:
123 parallel_config = ParallelConfig(self.pipeline_parallel_size,
124 self.tensor_parallel_size,
125 self.worker_use_ray)
126 + max_seq_len = min(
127 + self.max_num_batched_tokens,
128 + getattr(model_config.hf_config, "max_position_embeddings",
129 + float("inf")))
130 scheduler_config = SchedulerConfig(self.max_num_batched_tokens,
127 - self.max_num_seqs)
131 + self.max_num_seqs, max_seq_len)
132 return model_config, cache_config, parallel_config, scheduler_config
133
134
vllm/engine/llm_engine.py
+9 -3
@@ -226,8 +226,8 @@ class LLMEngine:
226 and updates the scheduler with the model outputs. Finally, it decodes
227 the sequences and returns the newly generated results.
228 """
229 - seq_group_metadata_list, scheduler_outputs = self.scheduler.schedule()
230 - if (not seq_group_metadata_list) and scheduler_outputs.is_empty():
229 + seq_group_metadata_list, scheduler_outputs, ignored_seq_groups = self.scheduler.schedule()
230 + if (not seq_group_metadata_list) and scheduler_outputs.is_empty() and (not ignored_seq_groups):
231 # Nothing to do.
232 return []
233
@@ -251,7 +251,7 @@ class LLMEngine:
251
252 # Create the outputs.
253 request_outputs: List[RequestOutput] = []
254 - for seq_group in seq_groups:
254 + for seq_group in seq_groups + ignored_seq_groups:
255 request_output = RequestOutput.from_seq_group(seq_group)
256 request_outputs.append(request_output)
257 return request_outputs
@@ -288,6 +288,12 @@ class LLMEngine:
288 if stopped:
289 continue
290
291 + # Check if the sequence has reached max_seq_len.
292 + if (seq.get_len() >=
293 + self.scheduler.scheduler_config.max_seq_len):
294 + self.scheduler.free_seq(
295 + seq, SequenceStatus.FINISHED_LENGTH_CAPPED)
296 + continue
297 # Check if the sequence has reached max_tokens.
298 if seq.get_output_len() == sampling_params.max_tokens:
299 self.scheduler.free_seq(
vllm/sequence.py
+4
@@ -13,6 +13,7 @@ class SequenceStatus(enum.Enum):
13 FINISHED_STOPPED = enum.auto()
14 FINISHED_LENGTH_CAPPED = enum.auto()
15 FINISHED_ABORTED = enum.auto()
16 + FINISHED_IGNORED = enum.auto()
17
18 @staticmethod
19 def is_finished(status: "SequenceStatus") -> bool:
@@ -20,6 +21,7 @@ class SequenceStatus(enum.Enum):
21 SequenceStatus.FINISHED_STOPPED,
22 SequenceStatus.FINISHED_LENGTH_CAPPED,
23 SequenceStatus.FINISHED_ABORTED,
24 + SequenceStatus.FINISHED_IGNORED
25 ]
26
27 @staticmethod
@@ -30,6 +32,8 @@ class SequenceStatus(enum.Enum):
32 finish_reason = "length"
33 elif status == SequenceStatus.FINISHED_ABORTED:
34 finish_reason = "abort"
35 + elif status == SequenceStatus.FINISHED_IGNORED:
36 + finish_reason = "length"
37 else:
38 finish_reason = None
39 return finish_reason