A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Fix hanging when prompt exceeds limit (#1029)
陈序 committed
Sep 17, 2023 at 16:48 UTC
e21d7687a9528ce6c1a7123b6223c53e65f37d8a
2 files changed
+10
-13
vllm/core/scheduler.py
+1
-1
@@ -175,7 +175,7 @@ class Scheduler:
175
num_curr_seqs += num_new_seqs
176
scheduled.append(seq_group)
177
178
- if scheduled:
178
+ if scheduled or ignored_seq_groups:
179
scheduler_outputs = SchedulerOutputs(
180
scheduled_seq_groups=scheduled,
181
prompt_run=True,
vllm/engine/llm_engine.py
+9
-12
@@ -294,14 +294,12 @@ class LLMEngine:
294
def _schedule(
295
self
296
) -> Tuple[List[SequenceGroupMetadata], SchedulerOutputs,
297
- Optional[List[RequestOutput]]]:
297
+ List[RequestOutput]]:
298
seq_group_metadata_list, scheduler_outputs = self.scheduler.schedule()
299
- if scheduler_outputs.is_empty():
300
- return seq_group_metadata_list, scheduler_outputs, [
301
- RequestOutput.from_seq_group(seq_group)
302
- for seq_group in scheduler_outputs.ignored_seq_groups
303
- ]
304
- return seq_group_metadata_list, scheduler_outputs, None
299
+ return seq_group_metadata_list, scheduler_outputs, [
300
+ RequestOutput.from_seq_group(seq_group)
301
+ for seq_group in scheduler_outputs.ignored_seq_groups
302
+ ]
303
304
def _check_beam_search_early_stopping(
305
self,
@@ -545,10 +543,9 @@ class LLMEngine:
543
and updates the scheduler with the model outputs. Finally, it decodes
544
the sequences and returns the newly generated results.
545
"""
548
- (seq_group_metadata_list, scheduler_outputs,
549
- early_return) = self._schedule()
550
- if early_return is not None:
551
- return early_return
546
+ seq_group_metadata_list, scheduler_outputs, ignored = self._schedule()
547
+ if scheduler_outputs.is_empty():
548
+ return ignored
549
550
# Execute the model.
551
output = self._run_workers(
@@ -559,7 +556,7 @@ class LLMEngine:
556
blocks_to_copy=scheduler_outputs.blocks_to_copy,
557
)
558
562
- return self._process_model_outputs(output, scheduler_outputs)
559
+ return self._process_model_outputs(output, scheduler_outputs) + ignored
560
561
def _log_system_stats(
562
self,