@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e21d7687a9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Fix hanging when prompt exceeds limit (#1029)

陈序 committed Sep 17, 2023 at 16:48 UTC e21d7687a9528ce6c1a7123b6223c53e65f37d8a
2 files changed +10 -13
vllm/core/scheduler.py
+1 -1
@@ -175,7 +175,7 @@ class Scheduler:
175 num_curr_seqs += num_new_seqs
176 scheduled.append(seq_group)
177
178 - if scheduled:
178 + if scheduled or ignored_seq_groups:
179 scheduler_outputs = SchedulerOutputs(
180 scheduled_seq_groups=scheduled,
181 prompt_run=True,
vllm/engine/llm_engine.py
+9 -12
@@ -294,14 +294,12 @@ class LLMEngine:
294 def _schedule(
295 self
296 ) -> Tuple[List[SequenceGroupMetadata], SchedulerOutputs,
297 - Optional[List[RequestOutput]]]:
297 + List[RequestOutput]]:
298 seq_group_metadata_list, scheduler_outputs = self.scheduler.schedule()
299 - if scheduler_outputs.is_empty():
300 - return seq_group_metadata_list, scheduler_outputs, [
301 - RequestOutput.from_seq_group(seq_group)
302 - for seq_group in scheduler_outputs.ignored_seq_groups
303 - ]
304 - return seq_group_metadata_list, scheduler_outputs, None
299 + return seq_group_metadata_list, scheduler_outputs, [
300 + RequestOutput.from_seq_group(seq_group)
301 + for seq_group in scheduler_outputs.ignored_seq_groups
302 + ]
303
304 def _check_beam_search_early_stopping(
305 self,
@@ -545,10 +543,9 @@ class LLMEngine:
543 and updates the scheduler with the model outputs. Finally, it decodes
544 the sequences and returns the newly generated results.
545 """
548 - (seq_group_metadata_list, scheduler_outputs,
549 - early_return) = self._schedule()
550 - if early_return is not None:
551 - return early_return
546 + seq_group_metadata_list, scheduler_outputs, ignored = self._schedule()
547 + if scheduler_outputs.is_empty():
548 + return ignored
549
550 # Execute the model.
551 output = self._run_workers(
@@ -559,7 +556,7 @@ class LLMEngine:
556 blocks_to_copy=scheduler_outputs.blocks_to_copy,
557 )
558
562 - return self._process_model_outputs(output, scheduler_outputs)
559 + return self._process_model_outputs(output, scheduler_outputs) + ignored
560
561 def _log_system_stats(
562 self,